sam/obmp-docker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Back AS Relationship Map with a materialized view

Browse Source 
The AS map previously exploded ~4.4M base_attrs AS_PATH rows live,
three times per load (one per panel), ~1.8s each — slow enough that
navigating away cancelled the queries mid-flight.

Add mv_as_adjacency: undirected consecutive-AS pairs with occurrence
counts over the full RIB (17k rows), refreshed hourly by pg_cron via
REFRESH ... CONCURRENTLY. The dashboard panels now read the view in
~1ms. Min-occurrence options rescaled for full-RIB counts
(2000/5000/10000/50000, default 2000 -> ~63-node graph).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
sam 2026-05-19 07:04:38 -07:00
parent 0190ef5fb8
commit cc0d20bf9e
2 changed files with 33 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
obmp-grafana/dashboards/obmp/Maps-1006/as_relationship_map.json
View File

@ -1,6 +1,6 @@
{ {
"annotations": {"list": [{"builtIn": 1,"datasource": {"type": "datasource","uid": "grafana"},"enable": true,"hide": true,"iconColor": "rgba(0, 211, 255, 1)","name": "Annotations & Alerts","type": "dashboard"}]}, "annotations": {"list": [{"builtIn": 1,"datasource": {"type": "datasource","uid": "grafana"},"enable": true,"hide": true,"iconColor": "rgba(0, 211, 255, 1)","name": "Annotations & Alerts","type": "dashboard"}]},
"description": "AS adjacency graph derived from consecutive AS pairs in observed AS_PATHs. Edge label = how many times that adjacency appears in a 200k-route sample. Raise 'Min occurrences' to thin the graph; set 'Focus AS' to a 1-hop view around one AS. Manual refresh — the query explodes ~200k AS_PATH arrays.", "description": "AS adjacency graph derived from consecutive AS pairs in observed AS_PATHs. Reads the mv_as_adjacency materialized view (full RIB, refreshed hourly by pg_cron) so panels load instantly. Edge label = how many times that adjacency appears. Raise 'Min occurrences' to thin the graph; set 'Focus AS' for a 1-hop view around one AS.",
"editable": true, "editable": true,
"fiscalYearStartMonth": 0, "fiscalYearStartMonth": 0,
"graphTooltip": 0, "graphTooltip": 0,
@ -10,7 +10,7 @@
"panels": [ "panels": [
{ {
"datasource": {"type": "postgres","uid": "obmp_postgres"}, "datasource": {"type": "postgres","uid": "obmp_postgres"},
"description": "Each node is an AS (enriched from info_asn whois data); each edge is an adjacency seen in the AS_PATH sample. Edge label is the occurrence count.", "description": "Each node is an AS (enriched from info_asn whois data); each edge is an adjacency seen in the AS_PATH data. Edge label is the occurrence count.",
"fieldConfig": {"defaults": {},"overrides": []}, "fieldConfig": {"defaults": {},"overrides": []},
"gridPos": {"h": 24,"w": 24,"x": 0,"y": 0}, "gridPos": {"h": 24,"w": 24,"x": 0,"y": 0},
"id": 1, "id": 1,
@ -19,13 +19,13 @@
{ {
"datasource": {"type": "postgres","uid": "obmp_postgres"}, "datasource": {"type": "postgres","uid": "obmp_postgres"},
"format": "table", "format": "table",
"rawSql": "WITH sample AS (SELECT as_path FROM base_attrs WHERE as_path_count >= 2 LIMIT 200000),\npairs AS (\n SELECT a.asn AS src, b.asn AS dst\n FROM sample ba\n CROSS JOIN LATERAL unnest(ba.as_path) WITH ORDINALITY a(asn,ord)\n JOIN LATERAL unnest(ba.as_path) WITH ORDINALITY b(asn,ord) ON b.ord = a.ord + 1\n WHERE a.asn <> b.asn\n),\nedges AS (\n SELECT LEAST(src,dst) AS x, GREATEST(src,dst) AS y, COUNT(*) AS occ\n FROM pairs GROUP BY 1,2 HAVING COUNT(*) >= $min_occ ORDER BY occ DESC LIMIT 300\n),\nfedges AS (\n SELECT * FROM edges\n WHERE '$focus_as' = '' OR x::text = '$focus_as' OR y::text = '$focus_as'\n),\nnlist AS (SELECT x AS asn FROM fedges UNION SELECT y FROM fedges),\ndeg AS (SELECT asn, COUNT(*) AS d FROM (SELECT x AS asn FROM fedges UNION ALL SELECT y FROM fedges) z GROUP BY asn)\nSELECT n.asn::text AS id,\n COALESCE(NULLIF(ia.as_name,''),'AS'||n.asn) AS title,\n 'AS ' || n.asn AS mainstat,\n COALESCE(NULLIF(ia.country,''),'?') || ' · ' || dg.d::text || ' links' AS secondarystat,\n COALESCE(NULLIF(ia.org_name,''),'—') AS detail__org,\n COALESCE(NULLIF(ia.country,''),'—') AS detail__country,\n dg.d::text AS detail__degree\nFROM nlist n\nLEFT JOIN info_asn ia ON ia.asn = n.asn\nLEFT JOIN deg dg ON dg.asn = n.asn\nORDER BY dg.d DESC", "rawSql": "WITH edges AS (\n SELECT asn_a, asn_b, occ FROM mv_as_adjacency\n WHERE occ >= $min_occ\n AND ('$focus_as' = '' OR asn_a::text = '$focus_as' OR asn_b::text = '$focus_as')\n ORDER BY occ DESC LIMIT 300\n),\nnlist AS (SELECT asn_a AS asn FROM edges UNION SELECT asn_b FROM edges),\ndeg AS (SELECT asn, COUNT(*) AS d FROM (SELECT asn_a AS asn FROM edges UNION ALL SELECT asn_b FROM edges) z GROUP BY asn)\nSELECT n.asn::text AS id,\n COALESCE(NULLIF(ia.as_name,''),'AS'||n.asn) AS title,\n 'AS ' || n.asn AS mainstat,\n COALESCE(NULLIF(ia.country,''),'?') || ' · ' || dg.d::text || ' links' AS secondarystat,\n COALESCE(NULLIF(ia.org_name,''),'—') AS detail__org,\n COALESCE(NULLIF(ia.country,''),'—') AS detail__country,\n dg.d::text AS detail__degree\nFROM nlist n\nLEFT JOIN info_asn ia ON ia.asn = n.asn\nLEFT JOIN deg dg ON dg.asn = n.asn\nORDER BY dg.d DESC",
"refId": "nodes" "refId": "nodes"
}, },
{ {
"datasource": {"type": "postgres","uid": "obmp_postgres"}, "datasource": {"type": "postgres","uid": "obmp_postgres"},
"format": "table", "format": "table",
"rawSql": "WITH sample AS (SELECT as_path FROM base_attrs WHERE as_path_count >= 2 LIMIT 200000),\npairs AS (\n SELECT a.asn AS src, b.asn AS dst\n FROM sample ba\n CROSS JOIN LATERAL unnest(ba.as_path) WITH ORDINALITY a(asn,ord)\n JOIN LATERAL unnest(ba.as_path) WITH ORDINALITY b(asn,ord) ON b.ord = a.ord + 1\n WHERE a.asn <> b.asn\n),\nedges AS (\n SELECT LEAST(src,dst) AS x, GREATEST(src,dst) AS y, COUNT(*) AS occ\n FROM pairs GROUP BY 1,2 HAVING COUNT(*) >= $min_occ ORDER BY occ DESC LIMIT 300\n)\nSELECT x::text || '-' || y::text AS id,\n x::text AS source, y::text AS target,\n occ AS mainstat,\n occ::text || ' paths' AS detail__occurrences\nFROM edges\nWHERE '$focus_as' = '' OR x::text = '$focus_as' OR y::text = '$focus_as'", "rawSql": "SELECT asn_a::text || '-' || asn_b::text AS id,\n asn_a::text AS source, asn_b::text AS target,\n occ AS mainstat,\n occ::text || ' paths' AS detail__occurrences\nFROM mv_as_adjacency\nWHERE occ >= $min_occ\n AND ('$focus_as' = '' OR asn_a::text = '$focus_as' OR asn_b::text = '$focus_as')\nORDER BY occ DESC LIMIT 300",
"refId": "edges" "refId": "edges"
} }
], ],
@ -34,7 +34,7 @@
}, },
{ {
"datasource": {"type": "postgres","uid": "obmp_postgres"}, "datasource": {"type": "postgres","uid": "obmp_postgres"},
"description": "The strongest AS adjacencies in the sample, with whois names for both endpoints.", "description": "The strongest AS adjacencies, with whois names for both endpoints.",
"fieldConfig": {"defaults": {"custom": {"align": "auto","displayMode": "auto"}},"overrides": [{"matcher": {"id": "byName","options": "Occurrences"},"properties": [{"id": "custom.displayMode","value": "gradient-gauge"},{"id": "color","value": {"mode": "continuous-BlPu"}}]}]}, "fieldConfig": {"defaults": {"custom": {"align": "auto","displayMode": "auto"}},"overrides": [{"matcher": {"id": "byName","options": "Occurrences"},"properties": [{"id": "custom.displayMode","value": "gradient-gauge"},{"id": "color","value": {"mode": "continuous-BlPu"}}]}]},
"gridPos": {"h": 10,"w": 24,"x": 0,"y": 24}, "gridPos": {"h": 10,"w": 24,"x": 0,"y": 24},
"id": 2, "id": 2,
@ -43,7 +43,7 @@
{ {
"datasource": {"type": "postgres","uid": "obmp_postgres"}, "datasource": {"type": "postgres","uid": "obmp_postgres"},
"format": "table", "format": "table",
"rawSql": "WITH sample AS (SELECT as_path FROM base_attrs WHERE as_path_count >= 2 LIMIT 200000),\npairs AS (\n SELECT a.asn AS src, b.asn AS dst\n FROM sample ba\n CROSS JOIN LATERAL unnest(ba.as_path) WITH ORDINALITY a(asn,ord)\n JOIN LATERAL unnest(ba.as_path) WITH ORDINALITY b(asn,ord) ON b.ord = a.ord + 1\n WHERE a.asn <> b.asn\n),\nedges AS (\n SELECT LEAST(src,dst) AS x, GREATEST(src,dst) AS y, COUNT(*) AS occ\n FROM pairs GROUP BY 1,2 HAVING COUNT(*) >= $min_occ ORDER BY occ DESC LIMIT 300\n)\nSELECT e.x AS \"AS A\",\n COALESCE(NULLIF(ax.as_name,''),'—') AS \"Name A\",\n e.y AS \"AS B\",\n COALESCE(NULLIF(ay.as_name,''),'—') AS \"Name B\",\n e.occ AS \"Occurrences\"\nFROM edges e\nLEFT JOIN info_asn ax ON ax.asn = e.x\nLEFT JOIN info_asn ay ON ay.asn = e.y\nWHERE '$focus_as' = '' OR e.x::text = '$focus_as' OR e.y::text = '$focus_as'\nORDER BY e.occ DESC", "rawSql": "SELECT e.asn_a AS \"AS A\",\n COALESCE(NULLIF(ax.as_name,''),'—') AS \"Name A\",\n e.asn_b AS \"AS B\",\n COALESCE(NULLIF(ay.as_name,''),'—') AS \"Name B\",\n e.occ AS \"Occurrences\"\nFROM (\n SELECT asn_a, asn_b, occ FROM mv_as_adjacency\n WHERE occ >= $min_occ\n AND ('$focus_as' = '' OR asn_a::text = '$focus_as' OR asn_b::text = '$focus_as')\n ORDER BY occ DESC LIMIT 300\n) e\nLEFT JOIN info_asn ax ON ax.asn = e.asn_a\nLEFT JOIN info_asn ay ON ay.asn = e.asn_b\nORDER BY e.occ DESC",
"refId": "A" "refId": "A"
} }
], ],
@ -56,7 +56,7 @@
"tags": ["obmp", "obmp-nav", "obmp-maps", "asn"], "tags": ["obmp", "obmp-nav", "obmp-maps", "asn"],
"templating": { "templating": {
"list": [ "list": [
{"name": "min_occ","type": "custom","label": "Min occurrences","description": "Only draw adjacencies seen at least this many times in the 200k-route sample. Raise it to thin a cluttered graph.","query": "5,20,100,500","current": {"text": "20","value": "20"},"options": [{"text": "5","value": "5","selected": false},{"text": "20","value": "20","selected": true},{"text": "100","value": "100","selected": false},{"text": "500","value": "500","selected": false}],"hide": 0}, {"name": "min_occ","type": "custom","label": "Min occurrences","description": "Only draw adjacencies seen at least this many times across the RIB. Raise it to thin a cluttered graph.","query": "2000,5000,10000,50000","current": {"text": "2000","value": "2000"},"options": [{"text": "2000","value": "2000","selected": true},{"text": "5000","value": "5000","selected": false},{"text": "10000","value": "10000","selected": false},{"text": "50000","value": "50000","selected": false}],"hide": 0},
{"name": "focus_as","type": "textbox","label": "Focus AS","description": "Optional. Enter an ASN to show only adjacencies touching that AS (1-hop view). Leave blank for the full graph.","query": "","current": {"text": "","value": ""},"options": [{"text": "","value": "","selected": true}],"hide": 0} {"name": "focus_as","type": "textbox","label": "Focus AS","description": "Optional. Enter an ASN to show only adjacencies touching that AS (1-hop view). Leave blank for the full graph.","query": "","current": {"text": "","value": ""},"options": [{"text": "","value": "","selected": true}],"hide": 0}
] ]
}, },

26
postgres/scripts/006_obmp_matviews.sql Normal file
View File

@ -0,0 +1,26 @@
-- OBMP analytics materialized views
--
-- Applied to the live database and version-controlled here for reproducibility.
-- Safe to run against an empty database at init time: the view is created empty
-- and the pg_cron job below populates/refreshes it hourly.
-- AS adjacency: every undirected pair of consecutive ASes in an observed AS_PATH,
-- with an occurrence count. Backs the AS Relationship Map dashboard so its panels
-- read a 17k-row view instead of exploding ~4.4M base_attrs rows live.
CREATE MATERIALIZED VIEW IF NOT EXISTS mv_as_adjacency AS
SELECT LEAST(a.asn, b.asn) AS asn_a,
GREATEST(a.asn, b.asn) AS asn_b,
COUNT(*) AS occ
FROM base_attrs ba
CROSS JOIN LATERAL unnest(ba.as_path) WITH ORDINALITY a(asn, ord)
JOIN LATERAL unnest(ba.as_path) WITH ORDINALITY b(asn, ord) ON b.ord = a.ord + 1
WHERE ba.as_path_count >= 2 AND a.asn <> b.asn
GROUP BY 1, 2;
-- Unique index required for REFRESH ... CONCURRENTLY (no read lock during refresh).
CREATE UNIQUE INDEX IF NOT EXISTS mv_as_adjacency_pair_idx ON mv_as_adjacency (asn_a, asn_b);
CREATE INDEX IF NOT EXISTS mv_as_adjacency_occ_idx ON mv_as_adjacency (occ DESC);
-- Refresh hourly at minute 23.
SELECT cron.schedule('refresh-mv-as-adjacency', '23 * * * *',
'REFRESH MATERIALIZED VIEW CONCURRENTLY mv_as_adjacency');