Compare commits

..

No commits in common. "26dea47a55c98999d2ad55f63d7424eb027b2b80" and "6d3387dfe57592e43760b75a7341c246fa7cb1c2" have entirely different histories.

6 changed files with 30 additions and 297 deletions

View File

@ -24,8 +24,6 @@ OBMP_COOKIE_DOMAIN=example.com
PSQL_MEM_LIMIT=6g PSQL_MEM_LIMIT=6g
PSQL_APP_MEM_LIMIT=4g PSQL_APP_MEM_LIMIT=4g
KAFKA_MEM_LIMIT=4g KAFKA_MEM_LIMIT=4g
# ExaBGP — the full-table feature holds up to 900K route objects in memory.
EXABGP_MEM_LIMIT=6g
# gNMI streaming telemetry (telegraf, test profile). GNMI_ADDRESSES is a # gNMI streaming telemetry (telegraf, test profile). GNMI_ADDRESSES is a
# quoted, comma-separated host:port list — add a router here once gNMI/grpc # quoted, comma-separated host:port list — add a router here once gNMI/grpc

View File

@ -17,12 +17,6 @@ services:
zookeeper: zookeeper:
restart: unless-stopped restart: unless-stopped
container_name: obmp-zookeeper container_name: obmp-zookeeper
healthcheck:
test: ["CMD-SHELL", "bash -c 'echo > /dev/tcp/localhost/2181'"]
interval: 30s
timeout: 10s
retries: 3
start_period: 60s
image: confluentinc/cp-zookeeper:7.1.1 image: confluentinc/cp-zookeeper:7.1.1
mem_limit: 1g mem_limit: 1g
volumes: volumes:
@ -35,12 +29,6 @@ services:
kafka: kafka:
restart: unless-stopped restart: unless-stopped
container_name: obmp-kafka container_name: obmp-kafka
healthcheck:
test: ["CMD-SHELL", "bash -c 'echo > /dev/tcp/localhost/9092'"]
interval: 30s
timeout: 10s
retries: 3
start_period: 90s
image: confluentinc/cp-kafka:7.1.1 image: confluentinc/cp-kafka:7.1.1
# Raise KAFKA_MEM_LIMIT for production (full-table initial dumps are bursty). # Raise KAFKA_MEM_LIMIT for production (full-table initial dumps are bursty).
mem_limit: ${KAFKA_MEM_LIMIT:-4g} mem_limit: ${KAFKA_MEM_LIMIT:-4g}
@ -99,12 +87,6 @@ services:
grafana: grafana:
restart: unless-stopped restart: unless-stopped
container_name: obmp-grafana container_name: obmp-grafana
healthcheck:
test: ["CMD-SHELL", "wget -q --spider http://localhost:3000/api/health || exit 1"]
interval: 30s
timeout: 10s
retries: 3
start_period: 40s
image: grafana/grafana:9.1.7 image: grafana/grafana:9.1.7
mem_limit: 1g mem_limit: 1g
ports: ports:
@ -146,12 +128,6 @@ services:
psql: psql:
restart: unless-stopped restart: unless-stopped
container_name: obmp-psql container_name: obmp-psql
healthcheck:
test: ["CMD-SHELL", "pg_isready -U openbmp -d openbmp"]
interval: 30s
timeout: 10s
retries: 3
start_period: 60s
image: openbmp/postgres:2.2.1 image: openbmp/postgres:2.2.1
# Raise PSQL_MEM_LIMIT for production (see docs/production-sizing.md). # Raise PSQL_MEM_LIMIT for production (see docs/production-sizing.md).
mem_limit: ${PSQL_MEM_LIMIT:-6g} mem_limit: ${PSQL_MEM_LIMIT:-6g}
@ -177,12 +153,6 @@ services:
collector: collector:
restart: unless-stopped restart: unless-stopped
container_name: obmp-collector container_name: obmp-collector
healthcheck:
test: ["CMD-SHELL", "bash -c 'echo > /dev/tcp/localhost/5000'"]
interval: 30s
timeout: 10s
retries: 3
start_period: 40s
image: openbmp/collector:2.2.3 image: openbmp/collector:2.2.3
mem_limit: 2g mem_limit: 2g
sysctls: sysctls:
@ -199,8 +169,6 @@ services:
psql-app: psql-app:
restart: unless-stopped restart: unless-stopped
container_name: obmp-psql-app container_name: obmp-psql-app
# No healthcheck — the consumer exposes no health port; Docker's
# restart-on-exit covers process death.
image: openbmp/psql-app:2.2.2 image: openbmp/psql-app:2.2.2
# mem_limit must exceed the MEM (JVM heap) env below. Raise both for # mem_limit must exceed the MEM (JVM heap) env below. Raise both for
# production — see docs/production-sizing.md. # production — see docs/production-sizing.md.
@ -248,16 +216,8 @@ services:
exabgp: exabgp:
restart: unless-stopped restart: unless-stopped
container_name: obmp-exabgp container_name: obmp-exabgp
healthcheck:
test: ["CMD-SHELL", "bash -c 'echo > /dev/tcp/localhost/5050'"]
interval: 30s
timeout: 10s
retries: 3
start_period: 40s
profiles: ["test"] profiles: ["test"]
# The full-table feature generates up to 900K route objects in memory; mem_limit: 512m
# 512m OOM-killed it. Raise EXABGP_MEM_LIMIT in .env for larger tables.
mem_limit: ${EXABGP_MEM_LIMIT:-6g}
build: build:
context: ./exabgp context: ./exabgp
dockerfile: Dockerfile dockerfile: Dockerfile
@ -281,12 +241,6 @@ services:
exabgp-ui: exabgp-ui:
restart: unless-stopped restart: unless-stopped
container_name: obmp-exabgp-ui container_name: obmp-exabgp-ui
healthcheck:
test: ["CMD-SHELL", "wget -q --spider http://localhost:5001/ || exit 1"]
interval: 30s
timeout: 10s
retries: 3
start_period: 30s
profiles: ["test"] profiles: ["test"]
mem_limit: 256m mem_limit: 256m
build: build:
@ -301,12 +255,6 @@ services:
influxdb: influxdb:
restart: unless-stopped restart: unless-stopped
container_name: obmp-influxdb container_name: obmp-influxdb
healthcheck:
test: ["CMD-SHELL", "curl -fsS http://localhost:8086/health || exit 1"]
interval: 30s
timeout: 10s
retries: 3
start_period: 40s
profiles: ["test"] profiles: ["test"]
image: influxdb:2.7 image: influxdb:2.7
mem_limit: 2g mem_limit: 2g
@ -332,13 +280,6 @@ services:
context: ./telegraf context: ./telegraf
dockerfile: Dockerfile dockerfile: Dockerfile
network_mode: host network_mode: host
# Run telegraf as root and override the image entrypoint (which otherwise
# drops back to the telegraf user) so [[inputs.docker]] can read the
# Docker daemon socket for container resource metrics.
user: root
entrypoint: ["telegraf"]
volumes:
- /var/run/docker.sock:/var/run/docker.sock
depends_on: depends_on:
- influxdb - influxdb
environment: environment:
@ -354,12 +295,6 @@ services:
traffic-gen: traffic-gen:
restart: unless-stopped restart: unless-stopped
container_name: obmp-traffic-gen container_name: obmp-traffic-gen
healthcheck:
test: ["CMD-SHELL", "bash -c 'echo > /dev/tcp/localhost/5051'"]
interval: 30s
timeout: 10s
retries: 3
start_period: 30s
profiles: ["test"] profiles: ["test"]
mem_limit: 1g mem_limit: 1g
build: build:
@ -377,12 +312,6 @@ services:
traffic-gen-ui: traffic-gen-ui:
restart: unless-stopped restart: unless-stopped
container_name: obmp-traffic-gen-ui container_name: obmp-traffic-gen-ui
healthcheck:
test: ["CMD-SHELL", "wget -q --spider http://localhost:5002/ || exit 1"]
interval: 30s
timeout: 10s
retries: 3
start_period: 30s
profiles: ["test"] profiles: ["test"]
mem_limit: 256m mem_limit: 256m
build: build:
@ -394,12 +323,6 @@ services:
traffic-gen-responder: traffic-gen-responder:
restart: unless-stopped restart: unless-stopped
container_name: obmp-traffic-gen-responder container_name: obmp-traffic-gen-responder
healthcheck:
test: ["CMD-SHELL", "bash -c 'echo > /dev/tcp/localhost/5053'"]
interval: 30s
timeout: 10s
retries: 3
start_period: 30s
profiles: ["test"] profiles: ["test"]
mem_limit: 1g mem_limit: 1g
build: build:
@ -422,12 +345,6 @@ services:
whois: whois:
restart: unless-stopped restart: unless-stopped
container_name: obmp-whois container_name: obmp-whois
healthcheck:
test: ["CMD-SHELL", "bash -c 'echo > /dev/tcp/localhost/43'"]
interval: 30s
timeout: 10s
retries: 3
start_period: 30s
image: openbmp/whois:2.2.0 image: openbmp/whois:2.2.0
mem_limit: 1g mem_limit: 1g
sysctls: sysctls:
@ -461,12 +378,6 @@ services:
portal: portal:
restart: unless-stopped restart: unless-stopped
container_name: obmp-portal container_name: obmp-portal
healthcheck:
test: ["CMD-SHELL", "wget -q --spider http://localhost:80/ || exit 1"]
interval: 30s
timeout: 10s
retries: 3
start_period: 20s
profiles: ["auth"] profiles: ["auth"]
mem_limit: 128m mem_limit: 128m
image: nginx:alpine image: nginx:alpine

View File

@ -52,7 +52,6 @@ neighbor ${p_ip} {
family { family {
ipv4 unicast; ipv4 unicast;
ipv6 unicast;
} }
api { api {

View File

@ -1,78 +0,0 @@
{
"annotations": {"list": [{"builtIn": 1,"datasource": {"type": "datasource","uid": "grafana"},"enable": true,"hide": true,"iconColor": "rgba(0, 211, 255, 1)","name": "Annotations & Alerts","type": "dashboard"}]},
"description": "Per-container CPU, memory, and I/O for the OpenBMP stack — collected by the Telegraf docker input. Watch memory % to catch a container approaching its mem_limit before it OOM-crashes.",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [{"asDropdown": true,"icon": "external link","includeVars": true,"keepTime": true,"tags": ["obmp-nav"],"title": "OBMP Dashboards","type": "dashboards"}],
"liveNow": false,
"panels": [
{
"datasource": {"type": "influxdb","uid": "obmp_influxdb"},
"description": "Memory usage as a percentage of each container's mem_limit. Sustained values near 100% precede an OOM kill.",
"fieldConfig": {
"defaults": {"color": {"mode": "palette-classic"},"custom": {"axisPlacement": "auto","drawStyle": "line","fillOpacity": 10,"lineInterpolation": "smooth","lineWidth": 1,"pointSize": 5,"showPoints": "never","spanNulls": false,"stacking": {"group": "A","mode": "none"}},"unit": "percent","min": 0,"thresholds": {"mode": "absolute","steps": [{"color": "green","value": null},{"color": "orange","value": 80},{"color": "red","value": 95}]}},
"overrides": []
},
"gridPos": {"h": 9,"w": 12,"x": 0,"y": 0},
"id": 1,
"options": {"legend": {"calcs": ["max"],"displayMode": "table","placement": "right","showLegend": true},"tooltip": {"mode": "multi","sort": "desc"}},
"targets": [{"datasource": {"type": "influxdb","uid": "obmp_influxdb"},"query": "from(bucket: \"telemetry\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"docker_container_mem\" and r._field == \"usage_percent\")\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> keep(columns: [\"_time\", \"_value\", \"container_name\"])\n |> group(columns: [\"container_name\"])","refId": "A"}],
"title": "Container Memory %",
"type": "timeseries"
},
{
"datasource": {"type": "influxdb","uid": "obmp_influxdb"},
"description": "CPU usage per container (cpu-total). Can exceed 100% — that is multiple cores.",
"fieldConfig": {
"defaults": {"color": {"mode": "palette-classic"},"custom": {"axisPlacement": "auto","drawStyle": "line","fillOpacity": 10,"lineInterpolation": "smooth","lineWidth": 1,"pointSize": 5,"showPoints": "never","spanNulls": false,"stacking": {"group": "A","mode": "none"}},"unit": "percent","min": 0},
"overrides": []
},
"gridPos": {"h": 9,"w": 12,"x": 12,"y": 0},
"id": 2,
"options": {"legend": {"calcs": ["max"],"displayMode": "table","placement": "right","showLegend": true},"tooltip": {"mode": "multi","sort": "desc"}},
"targets": [{"datasource": {"type": "influxdb","uid": "obmp_influxdb"},"query": "from(bucket: \"telemetry\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"docker_container_cpu\" and r._field == \"usage_percent\" and r.cpu == \"cpu-total\")\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> keep(columns: [\"_time\", \"_value\", \"container_name\"])\n |> group(columns: [\"container_name\"])","refId": "A"}],
"title": "Container CPU %",
"type": "timeseries"
},
{
"datasource": {"type": "influxdb","uid": "obmp_influxdb"},
"description": "Absolute memory usage per container.",
"fieldConfig": {
"defaults": {"color": {"mode": "palette-classic"},"custom": {"axisPlacement": "auto","drawStyle": "line","fillOpacity": 10,"lineInterpolation": "smooth","lineWidth": 1,"pointSize": 5,"showPoints": "never","spanNulls": false,"stacking": {"group": "A","mode": "none"}},"unit": "bytes","min": 0},
"overrides": []
},
"gridPos": {"h": 9,"w": 12,"x": 0,"y": 9},
"id": 3,
"options": {"legend": {"calcs": ["max"],"displayMode": "table","placement": "right","showLegend": true},"tooltip": {"mode": "multi","sort": "desc"}},
"targets": [{"datasource": {"type": "influxdb","uid": "obmp_influxdb"},"query": "from(bucket: \"telemetry\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"docker_container_mem\" and r._field == \"usage\")\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> keep(columns: [\"_time\", \"_value\", \"container_name\"])\n |> group(columns: [\"container_name\"])","refId": "A"}],
"title": "Container Memory Usage",
"type": "timeseries"
},
{
"datasource": {"type": "influxdb","uid": "obmp_influxdb"},
"description": "Current memory pressure per container. Anything in orange/red is close to its mem_limit.",
"fieldConfig": {
"defaults": {"custom": {"align": "auto","displayMode": "auto"},"unit": "percent"},
"overrides": [{"matcher": {"id": "byName","options": "Memory %"},"properties": [{"id": "custom.displayMode","value": "gradient-gauge"},{"id": "max","value": 100},{"id": "thresholds","value": {"mode": "absolute","steps": [{"color": "green","value": null},{"color": "orange","value": 80},{"color": "red","value": 95}]}}]}]
},
"gridPos": {"h": 9,"w": 12,"x": 12,"y": 9},
"id": 4,
"options": {"showHeader": true,"sortBy": [{"desc": true,"displayName": "Memory %"}]},
"targets": [{"datasource": {"type": "influxdb","uid": "obmp_influxdb"},"query": "from(bucket: \"telemetry\")\n |> range(start: -5m)\n |> filter(fn: (r) => r._measurement == \"docker_container_mem\" and r._field == \"usage_percent\")\n |> last()\n |> keep(columns: [\"container_name\", \"_value\"])\n |> group()\n |> rename(columns: {_value: \"Memory %\", container_name: \"Container\"})\n |> sort(columns: [\"Memory %\"], desc: true)","refId": "A"}],
"title": "Current Memory % by Container",
"type": "table"
}
],
"refresh": "30s",
"schemaVersion": 36,
"style": "dark",
"tags": ["obmp", "obmp-nav", "telemetry", "resources"],
"time": {"from": "now-1h","to": "now"},
"timepicker": {},
"timezone": "browser",
"title": "Stack Resources",
"uid": "obmp-stack-resources",
"version": 1
}

View File

@ -27,17 +27,7 @@
"id": null, "id": null,
"iteration": 1654876929746, "iteration": 1654876929746,
"links": [ "links": [
{ {"asDropdown": true,"icon": "external link","includeVars": true,"keepTime": true,"tags": ["obmp-nav"],"title": "OBMP Dashboards","type": "dashboards"}
"asDropdown": true,
"icon": "external link",
"includeVars": true,
"keepTime": true,
"tags": [
"obmp-nav"
],
"title": "OBMP Dashboards",
"type": "dashboards"
}
], ],
"liveNow": false, "liveNow": false,
"panels": [ "panels": [
@ -49,16 +39,8 @@
"description": "IPv4 vs IPv6 prefix count advertised by this ASN.", "description": "IPv4 vs IPv6 prefix count advertised by this ASN.",
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {
"color": { "color": {"mode": "palette-classic"},
"mode": "palette-classic" "custom": {"hideFrom": {"legend": false,"tooltip": false,"viz": false}},
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
},
"decimals": 0, "decimals": 0,
"mappings": [], "mappings": [],
"unit": "none" "unit": "none"
@ -74,48 +56,23 @@
"id": 6, "id": 6,
"links": [], "links": [],
"options": { "options": {
"displayLabels": [ "displayLabels": ["value"],
"value" "legend": {"calcs": [],"displayMode": "table","placement": "bottom","values": ["value","percent"]},
],
"legend": {
"calcs": [],
"displayMode": "table",
"placement": "bottom",
"values": [
"value",
"percent"
]
},
"pieType": "pie", "pieType": "pie",
"reduceOptions": { "reduceOptions": {"calcs": ["lastNotNull"],"fields": "","values": false},
"calcs": [ "tooltip": {"mode": "single","sort": "none"}
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
}, },
"pluginVersion": "9.1.7", "pluginVersion": "9.1.7",
"targets": [ "targets": [
{ {
"datasource": { "datasource": {"type": "postgres","uid": "obmp_postgres"},
"type": "postgres",
"uid": "obmp_postgres"
},
"alias": "", "alias": "",
"format": "time_series", "format": "time_series",
"rawSql": "SELECT\n max(timestamp) as time,\n count(*) as \"ipv4\"\nFROM\n global_ip_rib\nWHERE\n recv_origin_as = [[asn_num]]\n and family(prefix) = 4\nGROUP BY prefix\n", "rawSql": "SELECT\n max(timestamp) as time,\n count(*) as \"ipv4\"\nFROM\n global_ip_rib\nWHERE\n recv_origin_as = [[asn_num]]\n and family(prefix) = 4\nGROUP BY prefix\n",
"refId": "A" "refId": "A"
}, },
{ {
"datasource": { "datasource": {"type": "postgres","uid": "obmp_postgres"},
"type": "postgres",
"uid": "obmp_postgres"
},
"alias": "", "alias": "",
"format": "time_series", "format": "time_series",
"rawSql": "SELECT\n max(timestamp) as time,\n count(*) as \"ipv6\"\nFROM\n global_ip_rib\nWHERE\n recv_origin_as = [[asn_num]]\n and family(prefix) = 6\nGROUP BY prefix\n", "rawSql": "SELECT\n max(timestamp) as time,\n count(*) as \"ipv6\"\nFROM\n global_ip_rib\nWHERE\n recv_origin_as = [[asn_num]]\n and family(prefix) = 6\nGROUP BY prefix\n",
@ -229,39 +186,8 @@
"description": "IPv4/IPv6 prefixes originated by this ASN over time, with RPKI/IRR coverage (from stats_ip_origins).", "description": "IPv4/IPv6 prefixes originated by this ASN over time, with RPKI/IRR coverage (from stats_ip_origins).",
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {
"color": { "color": {"mode": "palette-classic"},
"mode": "palette-classic" "custom": {"axisCenteredZero": false,"axisColorMode": "text","axisLabel": "","axisPlacement": "auto","barAlignment": 0,"drawStyle": "line","fillOpacity": 10,"gradientMode": "none","hideFrom": {"legend": false,"tooltip": false,"viz": false},"lineInterpolation": "linear","lineWidth": 1,"pointSize": 5,"scaleDistribution": {"type": "linear"},"showPoints": "auto","spanNulls": false,"stacking": {"group": "A","mode": "none"},"thresholdsStyle": {"mode": "off"}},
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 0, "decimals": 0,
"mappings": [], "mappings": [],
"unit": "none" "unit": "none"
@ -277,28 +203,13 @@
"id": 14, "id": 14,
"links": [], "links": [],
"options": { "options": {
"legend": { "legend": {"calcs": ["min","max","mean"],"displayMode": "table","placement": "right","showLegend": true},
"calcs": [ "tooltip": {"mode": "multi","sort": "none"}
"min",
"max",
"mean"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
}, },
"pluginVersion": "9.1.7", "pluginVersion": "9.1.7",
"targets": [ "targets": [
{ {
"datasource": { "datasource": {"type": "postgres","uid": "obmp_postgres"},
"type": "postgres",
"uid": "obmp_postgres"
},
"alias": "", "alias": "",
"format": "time_series", "format": "time_series",
"rawSql": "SELECT\n $__time(interval_time),\n v4_prefixes,v6_prefixes,v4_with_rpki,v6_with_rpki,v4_with_irr,v6_with_irr\nFROM\n stats_ip_origins\nWHERE\n $__timeFilter(interval_time) and asn = [[asn_num]]\nORDER BY interval_time asc\n", "rawSql": "SELECT\n $__time(interval_time),\n v4_prefixes,v6_prefixes,v4_with_rpki,v6_with_rpki,v4_with_irr,v6_with_irr\nFROM\n stats_ip_origins\nWHERE\n $__timeFilter(interval_time) and asn = [[asn_num]]\nORDER BY interval_time asc\n",
@ -1030,24 +941,27 @@
"templating": { "templating": {
"list": [ "list": [
{ {
"name": "asn_num",
"type": "textbox",
"label": "Origin AS",
"description": "Enter an origin AS number \u2014 every panel shows that AS's prefixes, upstreams, and downstreams from the BMP RIB.",
"query": "13335",
"current": { "current": {
"text": "13335", "selected": false,
"value": "13335" "text": "714",
"value": "714"
}, },
"hide": 0,
"includeAll": false,
"label": "ASN",
"multi": false,
"name": "asn_num",
"options": [ "options": [
{ {
"text": "13335", "selected": true,
"value": "13335", "text": "109",
"selected": true "value": "109"
} }
], ],
"hide": 0, "query": "109",
"skipUrlSync": false "queryValue": "714",
"skipUrlSync": false,
"type": "custom"
} }
] ]
}, },

View File

@ -53,17 +53,6 @@
subscription_mode = "sample" subscription_mode = "sample"
sample_interval = "30s" sample_interval = "30s"
## Docker container resource metrics — CPU, memory (incl. limit + %), network,
## and block IO for every obmp-* container. Surfaces resource pressure (e.g. a
## container approaching its mem_limit) before it OOM-crashes.
[[inputs.docker]]
endpoint = "unix:///var/run/docker.sock"
gather_services = false
container_name_include = ["obmp-*"]
perdevice = false
total = true
timeout = "10s"
############################################################################### ###############################################################################
# OUTPUT PLUGINS # # OUTPUT PLUGINS #
############################################################################### ###############################################################################