Add psql-app conatainer and docker compose
This commit is contained in:
Tim Evens
parent
8b3356086b
commit
574bf5e8a9
29
README.md
29
README.md
@ -30,7 +30,7 @@ Each docker file contains a readme file, see below:
|
|||||||
|
|
||||||
* [Collector](collector/README.md)
|
* [Collector](collector/README.md)
|
||||||
* [PostgreSQL](postgres/README.md)
|
* [PostgreSQL](postgres/README.md)
|
||||||
* [PSQL Consumer](psql-consumer/README.md)
|
* [PSQL Consumer](psql-app/README.md)
|
||||||
|
|
||||||
|
|
||||||
## Using Docker Compose to run everything
|
## Using Docker Compose to run everything
|
||||||
@ -39,8 +39,33 @@ Each docker file contains a readme file, see below:
|
|||||||
You will need docker-compose. You can install that via [Docker Compose](https://docs.docker.com/compose/install/)
|
You will need docker-compose. You can install that via [Docker Compose](https://docs.docker.com/compose/install/)
|
||||||
instructions. Docker compose will run everything, including handling restarts of containers.
|
instructions. Docker compose will run everything, including handling restarts of containers.
|
||||||
|
|
||||||
|
#### (1) Mount/Make persistent directories
|
||||||
|
Create expected directories. You can choose to mount these as well or update the compose file to change them.
|
||||||
|
|
||||||
|
> **NOTE**
|
||||||
|
> If you are using OSX/Mac, then you will need to update your docker preferences to allow ```/var/openbmp```
|
||||||
|
|
||||||
|
Make sure to create the **OBMP_DATA_ROOT** directory first.
|
||||||
```
|
```
|
||||||
docker-compose -p obmp up
|
export OBMP_DATA_ROOT=/var/openbmp
|
||||||
|
sudo mkdir -p $OBMP_DATA_ROOT
|
||||||
|
```
|
||||||
|
|
||||||
|
Create sub directories
|
||||||
|
```
|
||||||
|
mkdir -p ${OBMP_DATA_ROOT}/config
|
||||||
|
mkdir -p ${OBMP_DATA_ROOT``}/kafka-data
|
||||||
|
mkdir -p ${OBMP_DATA_ROOT}/postgres/data
|
||||||
|
mkdir -p ${OBMP_DATA_ROOT}/postgres/ts
|
||||||
|
mkdir -p ${OBMP_DATA_ROOT}/grafana
|
||||||
|
|
||||||
|
sudo chmod -R 7777 $OBMP_DATA_ROOT
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
Change ```OBMP_DATA_ROOT=<path>``` to where you created the directories above. The default is ```/var/openbmp```
|
||||||
|
|
||||||
|
```
|
||||||
|
OBMP_DATA_ROOT=/var/openbmp docker-compose -p obmp up -d
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@ -54,7 +54,7 @@ fi
|
|||||||
# Update openbmpd config file
|
# Update openbmpd config file
|
||||||
OPENBMP_CFG_FILE=/usr/etc/openbmp/openbmpd.conf
|
OPENBMP_CFG_FILE=/usr/etc/openbmp/openbmpd.conf
|
||||||
sed -r -i "s/admin_id:.*/admin_id: ${ADMIN_ID}/" /usr/etc/openbmp/openbmpd.conf
|
sed -r -i "s/admin_id:.*/admin_id: ${ADMIN_ID}/" /usr/etc/openbmp/openbmpd.conf
|
||||||
sed -r -i "s/localhost:9092/${KAFKA_FQDN}:9092/" /usr/etc/openbmp/openbmpd.conf
|
sed -r -i "s/localhost:9092/${KAFKA_FQDN}/" /usr/etc/openbmp/openbmpd.conf
|
||||||
|
|
||||||
if [[ -f /config/openbmpd.conf ]]; then
|
if [[ -f /config/openbmpd.conf ]]; then
|
||||||
OPENBMP_CFG_FILE=/config/openbmpd.conf
|
OPENBMP_CFG_FILE=/config/openbmpd.conf
|
||||||
|
|||||||
111
docker-compose.yml
Normal file
111
docker-compose.yml
Normal file
@ -0,0 +1,111 @@
|
|||||||
|
---
|
||||||
|
version: '3'
|
||||||
|
services:
|
||||||
|
|
||||||
|
zookeeper:
|
||||||
|
restart: unless-stopped
|
||||||
|
container_name: zookeeper
|
||||||
|
image: confluentinc/cp-zookeeper:6.0.2
|
||||||
|
environment:
|
||||||
|
ZOOKEEPER_CLIENT_PORT: 2181
|
||||||
|
ZOOKEEPER_TICK_TIME: 2000
|
||||||
|
|
||||||
|
kafka:
|
||||||
|
restart: unless-stopped
|
||||||
|
container_name: kafka
|
||||||
|
image: confluentinc/cp-kafka:6.0.2
|
||||||
|
|
||||||
|
# Change the mount point to where you want to store Kafka data.
|
||||||
|
# Normally 80GB or more
|
||||||
|
volumes:
|
||||||
|
- ${OBMP_DATA_ROOT}/kafka-data:/var/lib/kafka/data
|
||||||
|
depends_on:
|
||||||
|
- zookeeper
|
||||||
|
ports:
|
||||||
|
- 9092:9092
|
||||||
|
environment:
|
||||||
|
KAFKA_BROKER_ID: 1
|
||||||
|
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
|
||||||
|
|
||||||
|
# Change/add listeners based on your FQDN that the host and other containers can access. You can use
|
||||||
|
# an IP address as well. By default, only within the compose/containers can Kafka be accesssed
|
||||||
|
# using port 29092. Outside access can be enabled, but you should use an FQDN listener.
|
||||||
|
#KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://<FQDN>:9092
|
||||||
|
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092
|
||||||
|
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
|
||||||
|
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
|
||||||
|
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
||||||
|
KAFKA_NUM_PARTITIONS: 8
|
||||||
|
KAFKA_LOG_RETENTION_MINUTES: 90
|
||||||
|
KAFKA_LOG_ROLL_MS: 3600000
|
||||||
|
KAFKA_LOG_SEGMENT_BYTES: 1073741824
|
||||||
|
KAFKA_MESSAGE_MAX_BYTES: 100000000
|
||||||
|
KAFKA_LOG_CLEANER_THREADS: 2
|
||||||
|
|
||||||
|
grafana:
|
||||||
|
restart: unless-stopped
|
||||||
|
container_name: grafana
|
||||||
|
image: grafana/grafana:latest
|
||||||
|
ports:
|
||||||
|
- 3000:3000
|
||||||
|
volumes:
|
||||||
|
- ${OBMP_DATA_ROOT}/grafana:/var/lib/grafana
|
||||||
|
- ${OBMP_DATA_ROOT}/grafana-provisioning/:/etc/grafana/provisioning/
|
||||||
|
environment:
|
||||||
|
- GF_SECURITY_ADMIN_PASSWORD=openbmp
|
||||||
|
- GF_AUTH_ANONYMOUS_ENABLED=true
|
||||||
|
- GF_SERVER_ROOT_URL=/ # Change this if you have grafana behind URL/paths, such as /grafana/
|
||||||
|
- GF_INSTALL_PLUGINS=agenty-flowcharting-panel,briangann-datatable-panel,digrich-bubblechart-panel,grafana-piechart-panel,grafana-worldmap-panel,vonage-status-panel
|
||||||
|
|
||||||
|
psql:
|
||||||
|
restart: unless-stopped
|
||||||
|
container_name: psql
|
||||||
|
image: openbmp/postgres:build-50
|
||||||
|
ports:
|
||||||
|
- 5432:5432
|
||||||
|
volumes:
|
||||||
|
- ${OBMP_DATA_ROOT}/postgres/data:/var/lib/postgresql/data # change this to 80GB SSD slice/partition
|
||||||
|
- ${OBMP_DATA_ROOT}/postgres/ts:/var/lib/postgresql/ts # Chnage this to 500GB SSD slice/partition
|
||||||
|
environment:
|
||||||
|
- MEM=2 # Set MEM to at least 2GB but ideally >16GB
|
||||||
|
- POSTGRES_PASSWORD=openbmp
|
||||||
|
- POSTGRES_USER=openbmp
|
||||||
|
- POSTGRES_DB=openbmp
|
||||||
|
|
||||||
|
collector:
|
||||||
|
restart: unless-stopped
|
||||||
|
container_name: collector
|
||||||
|
image: openbmp/collector:build-50
|
||||||
|
sysctls:
|
||||||
|
- net.ipv4.tcp_keepalive_intvl=30
|
||||||
|
- net.ipv4.tcp_keepalive_probes=5
|
||||||
|
- net.ipv4.tcp_keepalive_time=180
|
||||||
|
ports:
|
||||||
|
- 5000:5000
|
||||||
|
volumes:
|
||||||
|
- ${OBMP_DATA_ROOT}/config:/config
|
||||||
|
environment:
|
||||||
|
- KAFKA_FQDN=kafka:29092
|
||||||
|
|
||||||
|
psql-app:
|
||||||
|
restart: unless-stopped
|
||||||
|
container_name: psql-app
|
||||||
|
image: openbmp/psql-app:build-50
|
||||||
|
sysctls:
|
||||||
|
- net.ipv4.tcp_keepalive_intvl=30
|
||||||
|
- net.ipv4.tcp_keepalive_probes=5
|
||||||
|
- net.ipv4.tcp_keepalive_time=180
|
||||||
|
ports:
|
||||||
|
- 9005:9005
|
||||||
|
- 8080:8080
|
||||||
|
volumes:
|
||||||
|
- ${OBMP_DATA_ROOT}/config:/config
|
||||||
|
environment:
|
||||||
|
- MEM=2 # Set memory to at least 2GB but ideally 4GB
|
||||||
|
- KAFKA_FQDN=kafka:29092
|
||||||
|
- ENABLE_RPKI=0 # 1 enables, 0 disables RPKI sync
|
||||||
|
- ENABLE_IRR=0 # 1 enables, 0 disables IRR sync
|
||||||
|
- POSTGRES_PASSWORD=openbmp
|
||||||
|
- POSTGRES_USER=openbmp
|
||||||
|
- POSTGRES_DB=openbmp
|
||||||
|
- POSTGRES_HOST=psql:5432
|
||||||
@ -18,6 +18,11 @@ FROM timescale/timescaledb:2.1.0-pg13
|
|||||||
VOLUME ["/ws"]
|
VOLUME ["/ws"]
|
||||||
WORKDIR /ws
|
WORKDIR /ws
|
||||||
|
|
||||||
|
# Expected data locations for base tables and timeseries
|
||||||
|
#
|
||||||
|
VOLUME ["/var/lib/postgresql/data"]
|
||||||
|
VOLUME ["/var/lib/postgresql/ts"]
|
||||||
|
|
||||||
|
|
||||||
RUN apk update \
|
RUN apk update \
|
||||||
&& apk add openssl \
|
&& apk add openssl \
|
||||||
@ -31,10 +36,12 @@ RUN apk update \
|
|||||||
&& sed -i -e "s/^\#*listen_addresses.*=.*/listen_addresses = '*'/" /usr/local/share/postgresql/postgresql.conf.sample \
|
&& sed -i -e "s/^\#*listen_addresses.*=.*/listen_addresses = '*'/" /usr/local/share/postgresql/postgresql.conf.sample \
|
||||||
&& sed -i -e "s/^\#*ssl[ ]*=.*/ssl = on/" /usr/local/share/postgresql/postgresql.conf.sample \
|
&& sed -i -e "s/^\#*ssl[ ]*=.*/ssl = on/" /usr/local/share/postgresql/postgresql.conf.sample \
|
||||||
&& sed -i -e "s/^\#*ssl_cert_file.*=.*/ssl_cert_file = \'\/psql_server.crt\'/" /usr/local/share/postgresql/postgresql.conf.sample \
|
&& sed -i -e "s/^\#*ssl_cert_file.*=.*/ssl_cert_file = \'\/psql_server.crt\'/" /usr/local/share/postgresql/postgresql.conf.sample \
|
||||||
&& sed -i -e "s/^\#*ssl_key_file.*=.*/ssl_key_file = \'\/psql_server.key\'/" /usr/local/share/postgresql/postgresql.conf.sample
|
&& sed -i -e "s/^\#*ssl_key_file.*=.*/ssl_key_file = \'\/psql_server.key\'/" /usr/local/share/postgresql/postgresql.conf.sample \
|
||||||
|
&& sed -i -e "s/^\#*shared_buffers.*=.*/shared_buffers = ${MEM:-'1'}GB/" /usr/local/share/postgresql/postgresql.conf.sample \
|
||||||
|
&& sed -i -e "s/^\#*work_mem.*=.*/work_mem = $(( (${MEM:-1} * 1024) * 5 / 100))MB/" /usr/local/share/postgresql/postgresql.conf.sample
|
||||||
|
|
||||||
|
# Init timesries location
|
||||||
|
RUN echo 'psql -U $POSTGRES_USER -c "CREATE TABLESPACE timeseries LOCATION '\''/var/lib/postgresql/ts'\'';" $POSTGRES_DB' > /docker-entrypoint-initdb.d/0_obmp_init.sh
|
||||||
|
|
||||||
|
|
||||||
#RUN echo 'psql -U $POSTGRES_USER -c "CREATE TABLESPACE timeseries LOCATION '\''/var/lib/postgresql/ts'\'';" $POSTGRES_DB' > /docker-entrypoint-initdb.d/0_cnis_schema.sh \
|
|
||||||
# && echo 'psql -U $POSTGRES_USER $POSTGRES_DB < /tmp/cnis-mq.sql' >> /docker-entrypoint-initdb.d/0_cnis_schema.sh \
|
|
||||||
# && echo 'psql -U $POSTGRES_USER $POSTGRES_DB < /tmp/cnis-mq-functions.sql' >> /docker-entrypoint-initdb.d/0_cnis_schema.sh \
|
|
||||||
# && echo 'psql -U $POSTGRES_USER $POSTGRES_DB < /tmp/dev-psql.sql' >> /docker-entrypoint-initdb.d/0_cnis_schema.sh
|
|
||||||
|
|
||||||
|
|||||||
@ -20,3 +20,70 @@ See both [Postgres](https://hub.docker.com/_/postgres) and
|
|||||||
[TimescaleDB](https://hub.docker.com/r/timescale/timescaledb) documentation for more
|
[TimescaleDB](https://hub.docker.com/r/timescale/timescaledb) documentation for more
|
||||||
information on how to configure/run the docker container.
|
information on how to configure/run the docker container.
|
||||||
|
|
||||||
|
### PostgreSQL Related
|
||||||
|
|
||||||
|
#### Postgres can be killed by the Linux OOM-Killer
|
||||||
|
This is very bad as it causes Postgres to restart. This will happen because postgres uses a large shared buffer,
|
||||||
|
which causes the OOM to believe it's using a lot of VM.
|
||||||
|
|
||||||
|
It is suggested to run the postgres server with the following Linux settings:
|
||||||
|
|
||||||
|
# Update runtime
|
||||||
|
sysctl -w vm.vfs_cache_pressure=500
|
||||||
|
sysctl -w vm.swappiness=10
|
||||||
|
sysctl -w vm.min_free_kbytes=1000000
|
||||||
|
sysctl -w vm.overcommit_memory=2
|
||||||
|
sysctl -w vm.overcommit_ratio=95
|
||||||
|
|
||||||
|
# Update startup
|
||||||
|
echo "vm.vfs_cache_pressure=500" >> /etc/sysctl.conf
|
||||||
|
echo "vm.min_free_kbytes=1000000" >> /etc/sysctl.conf
|
||||||
|
echo "vm.swappiness=10" >> /etc/sysctl.conf
|
||||||
|
echo "vm.overcommit_memory=2" >> /etc/sysctl.conf
|
||||||
|
echo "vm.overcommit_ratio=95" >> /etc/sysctl.conf
|
||||||
|
|
||||||
|
|
||||||
|
See Postgres [hugepages](https://www.postgresql.org/docs/current/static/kernel-resources.html#LINUX-HUGE-PAGES) for
|
||||||
|
details on how to enable and use hugepages. Some Linux distributions enable **transparent hugepages** which
|
||||||
|
will prevent the ability to configure ```vm.nr_hugepages```. If you find that you cannot set ```vm.nr_hugepages```,
|
||||||
|
then try the below:
|
||||||
|
|
||||||
|
echo never > /sys/kernel/mm/transparent_hugepage/enabled
|
||||||
|
echo never > /sys/kernel/mm/transparent_hugepage/defrag
|
||||||
|
sync && echo 3 > /proc/sys/vm/drop_caches
|
||||||
|
|
||||||
|
|
||||||
|
#### Postgres Vacuum (reclaim disk space)
|
||||||
|
Postgres reclaims deleted/updated records using the vacuum process. You can run this manually/cron via the
|
||||||
|
```VACUUM``` command. **autovacuum** is used to do this periodically. Careful tuning of this
|
||||||
|
is required. Checkout [autovacuum-tuning-basics](https://blog.2ndquadrant.com/autovacuum-tuning-basics/),
|
||||||
|
[Routine Vacuuming](https://www.postgresql.org/docs/current/static/routine-vacuuming.html), and
|
||||||
|
[VACUUM](https://www.postgresql.org/docs/current/static/sql-vacuum.html) for more details.
|
||||||
|
|
||||||
|
#### Create persistent postgres locations
|
||||||
|
|
||||||
|
*You should use fast SSD and/or ZFS.* Size of these locations/mount points are directly related to the
|
||||||
|
number of NLRI's maintained and number of changes/updates per second.
|
||||||
|
|
||||||
|
> TODO: Will post numbers of how to determine the disk size needed. For now, if you have less
|
||||||
|
> than 50,000,00 prefixes, then you can use 1TB. If you have more than that, you should consider
|
||||||
|
> multiple disks. ZFS can make your life easier as you can easily add disks and it supports compression.
|
||||||
|
|
||||||
|
- **postgres/main** - This location will be used for the main postgres data
|
||||||
|
files and tables.
|
||||||
|
|
||||||
|
> This really should be a mount point to a dedicated filesystem
|
||||||
|
|
||||||
|
```
|
||||||
|
mkdir -p /var/openbmp/postgres/main
|
||||||
|
chmod 7777 /var/openbmp/postgres/main
|
||||||
|
```
|
||||||
|
|
||||||
|
- **postgres/ts** - This location will be used for the time series postgres tables
|
||||||
|
|
||||||
|
> This really should be a mount point to a dedicated filesystem
|
||||||
|
|
||||||
|
```
|
||||||
|
mkdir -p /var/openbmp/postgres/ts
|
||||||
|
chmod 7777 /var/openbmp/postgres/ts
|
||||||
|
```
|
||||||
|
|||||||
115
psql-app/Dockerfile
Normal file
115
psql-app/Dockerfile
Normal file
@ -0,0 +1,115 @@
|
|||||||
|
# Postgres Backend: openbmp/psql-consumer
|
||||||
|
#
|
||||||
|
# Copyright (c) 2021 Cisco Systems, Inc. and Tim Evens. All rights reserved.
|
||||||
|
#
|
||||||
|
# Author: Tim Evens <tim@openbmp.org>
|
||||||
|
#
|
||||||
|
# Docker context does not support multiple paths or mounting volumes for builds.
|
||||||
|
# In effort to build the container from local git clones, we dynamically build a context
|
||||||
|
#
|
||||||
|
# Clone the obmp-psql, obmp-java-api-message, and obmp-docker repos into the same directory.
|
||||||
|
# Change directories to obmp-docker/psql-app and run the below from that
|
||||||
|
# directory.
|
||||||
|
#
|
||||||
|
# Example docker build:
|
||||||
|
# tar -cL -C ../../ ./obmp-psql ./obmp-docker/psql-app ./obmp-java-api-message \
|
||||||
|
# | docker build --build-arg BUILD_NUMBER=50 \
|
||||||
|
# -f obmp-docker/psql-app/Dockerfile -t openbmp/psql-app:build-50 -
|
||||||
|
|
||||||
|
# -----------------------------------------------
|
||||||
|
# stage: Build
|
||||||
|
# -----------------------------------------------
|
||||||
|
FROM openbmp/dev-image:latest AS build
|
||||||
|
|
||||||
|
ARG BUILD_NUMBER=0
|
||||||
|
|
||||||
|
# Proxy servers
|
||||||
|
#ENV http_proxy http://proxy:80
|
||||||
|
#ENV https_proxy http://proxy:80
|
||||||
|
#ENV no_proxy "domain.com"
|
||||||
|
|
||||||
|
COPY obmp-psql/ /ws
|
||||||
|
COPY obmp-java-api-message/ /tmp/obmp-java-api-message
|
||||||
|
WORKDIR /ws
|
||||||
|
|
||||||
|
RUN cd /tmp/obmp-java-api-message \
|
||||||
|
&& mvn clean install \
|
||||||
|
&& cd /ws \
|
||||||
|
&& mvn clean package
|
||||||
|
|
||||||
|
# -----------------------------------------------
|
||||||
|
# stage: Final container
|
||||||
|
# -----------------------------------------------
|
||||||
|
FROM openjdk:17-slim
|
||||||
|
|
||||||
|
# Copy files from previous stages
|
||||||
|
COPY --from=build /ws/target/obmp-psql-consumer-0.1.0-SNAPSHOT.jar /usr/local/openbmp/obmp-psql-consumer.jar
|
||||||
|
COPY --from=build /ws/database/ /usr/local/openbmp/database
|
||||||
|
COPY --from=build /ws/cron_scripts/gen-whois/*.py /usr/local/openbmp/
|
||||||
|
COPY --from=build /ws/cron_scripts/rpki/*.py /usr/local/openbmp/
|
||||||
|
COPY --from=build /ws/scripts/dbip-to-psql.py /usr/local/openbmp/
|
||||||
|
|
||||||
|
# Add files
|
||||||
|
ADD obmp-docker/psql-app/scripts/run /usr/sbin/
|
||||||
|
|
||||||
|
#----------------------------------
|
||||||
|
# Define persistent data volumes
|
||||||
|
VOLUME ["/config"]
|
||||||
|
|
||||||
|
#----------------------------------
|
||||||
|
# Expose ports.
|
||||||
|
|
||||||
|
# Consumer JMX console
|
||||||
|
EXPOSE 9005
|
||||||
|
|
||||||
|
# RPKI Validator port
|
||||||
|
EXPOSE 8080
|
||||||
|
|
||||||
|
#----------------------------------
|
||||||
|
# Define working directory.
|
||||||
|
WORKDIR /tmp
|
||||||
|
|
||||||
|
# Base setup tasks
|
||||||
|
RUN touch /usr/local/build-${BUILD_NUMBER} \
|
||||||
|
&& chmod 755 /usr/local/openbmp/*.py
|
||||||
|
|
||||||
|
#----------------------------------
|
||||||
|
# Install depends
|
||||||
|
# Depends are installed in layers so that this doesn't have to be done each time
|
||||||
|
# the image is built.
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install --allow-unauthenticated -y unzip curl wget whois vim rsyslog cron rsync kafkacat \
|
||||||
|
procps python3-minimal python3-distutils python3-psycopg2 python3-dnspython postgresql-client \
|
||||||
|
&& ln -s /usr/bin/python3 /usr/bin/python
|
||||||
|
|
||||||
|
RUN cd /tmp && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \
|
||||||
|
&& python3 get-pip.py
|
||||||
|
|
||||||
|
RUN pip install ipaddr
|
||||||
|
|
||||||
|
# Install latest postgres client
|
||||||
|
#RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' \
|
||||||
|
# && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - \
|
||||||
|
# && apt-get update
|
||||||
|
# && apt-get install postgresql-13-client
|
||||||
|
|
||||||
|
# Cleanup
|
||||||
|
RUN apt-get autoremove && apt-get clean
|
||||||
|
|
||||||
|
#----------------------------------
|
||||||
|
# Install RPKI validator (https://github.com/RIPE-NCC/rpki-validator-3/wiki)
|
||||||
|
RUN mkdir /usr/local/rpki && cd /tmp \
|
||||||
|
&& wget https://ftp.ripe.net/tools/rpki/validator3/prod/generic/rpki-validator-3-latest-dist.tar.gz \
|
||||||
|
&& tar xzf rpki-validator-3-latest-dist.tar.gz \
|
||||||
|
&& cd rpki-validator-*/ \
|
||||||
|
&& mv * /usr/local/rpki/ \
|
||||||
|
&& rm -rf /tmp/rpki-* \
|
||||||
|
&& cd /usr/local/rpki \
|
||||||
|
&& sed -i -r 's/.*server.address=.*/server.address=0.0.0.0/' /usr/local/rpki/conf/application.properties \
|
||||||
|
&& sed -i -r 's/jvm.mem.maximum=.*/jvm.mem.maximum=2g/' /usr/local/rpki/conf/application.properties
|
||||||
|
|
||||||
|
|
||||||
|
#----------------------------------
|
||||||
|
# Define default command.
|
||||||
|
CMD ["/usr/sbin/run"]
|
||||||
|
|
||||||
129
psql-app/README.md
Normal file
129
psql-app/README.md
Normal file
@ -0,0 +1,129 @@
|
|||||||
|
# OpenBMP Postgres Application Container
|
||||||
|
This container is the main application container for OpenBMP and PostgreSQL.
|
||||||
|
|
||||||
|
It provides:
|
||||||
|
|
||||||
|
* PostgreSQL consumer
|
||||||
|
* RPKI validator improt/sync
|
||||||
|
* IRR and peering DB import/sync
|
||||||
|
* Schedules and runs the metric DB functions
|
||||||
|
* Schedules and runs the DB timescale DB chunk drops
|
||||||
|
|
||||||
|
## Building
|
||||||
|
See the [Dockerfile](Dockerfile) notes for build instructions.
|
||||||
|
|
||||||
|
## Running
|
||||||
|
|
||||||
|
### Kafka Validation Testing
|
||||||
|
The Kafka setup can be tricky due to docker networking between containers and remote systems. Kafka clustering
|
||||||
|
makes use of a bootstrap server which will advertise each broker ```hostname:port``` that the consumer/producer
|
||||||
|
will use. Each consumer/producer will connect to the brokers using these **advertised** hostnames and ports. The
|
||||||
|
setting in Kafka to configure the broker hostname is ```advertised.listeners```.
|
||||||
|
|
||||||
|
The postgres container (**this container**) uses the **KAFKA_FQDN** as the bootstrap server,
|
||||||
|
syntax is ```<HOSTNAME or IP:PORT>```. This will work with an
|
||||||
|
IP or hostname. When using a hostname, the hostname *MUST* resolve within the container. While this may work for
|
||||||
|
bootstrap server conection, the advertised hostnames need to also resolve in the container.
|
||||||
|
|
||||||
|
**Kafka Validation is a 3 step process**
|
||||||
|
|
||||||
|
1. Successfully connect to the bootstrap server and retrieve metadata (e.g. broker hostname:port)
|
||||||
|
2. Successfully produce a test message to ```openbmp.parsed.test``` topic
|
||||||
|
3. Successfully consume a test message from ```openbmp.parsed.test``` topic
|
||||||
|
|
||||||
|
> **IMPORTANT**
|
||||||
|
> If using your own Kafka install, make sure you allow producing/consuming to/from **openbmp.parsed.test**
|
||||||
|
> for the consumer validation.
|
||||||
|
|
||||||
|
### Hostnames in Container
|
||||||
|
You can map the Kafka hostname and each broker if they are different using two methods:
|
||||||
|
|
||||||
|
1. add ```--add-host HOSTNAME:IP``` to **docker run** command. Make sure to add one for the bootstrap and each broker.
|
||||||
|
2. Create a **/var/openbmp/config/hosts** file and add the Kafka bootstrap and broker hostname to IP mappings.
|
||||||
|
|
||||||
|
### VM Specifications
|
||||||
|
|
||||||
|
#### Storage
|
||||||
|
|
||||||
|
You will need to dedicate space for the postgres instance. Normally two partitions are used. A good
|
||||||
|
starting size for postgres main is 500GB and postgres ts (timescaleDB) is 1TB. Both disks
|
||||||
|
should be fast SSD. ZFS can be used on either of them to add compression. The size you need will depend
|
||||||
|
on the number of NLRI's and updates per second.
|
||||||
|
|
||||||
|
#### Memory & CPU
|
||||||
|
|
||||||
|
The size of memory will depend on the type of queries and number of NLRI's. A good starting point for
|
||||||
|
memory is a server with more than 48GB RAM. You can run on as little as 4GB RAM but that will only
|
||||||
|
scale to about 10,000,000 NLRI's. 64BG of RAM should scale to 150,000,000 NLRI's.
|
||||||
|
|
||||||
|
The number of vCPU's also varies by the number of concurrent connections and how many threads you use for
|
||||||
|
the postgres consumer. A good starting point is at least 8 vCPU's.
|
||||||
|
|
||||||
|
|
||||||
|
### 1) Install docker
|
||||||
|
Follow the [Docker Instructions](https://docs.docker.com/install) to install docker CE.
|
||||||
|
|
||||||
|
### 2) Add persistent volumes
|
||||||
|
|
||||||
|
Persistent volumes make it possible for upgrades without loosing any data.
|
||||||
|
|
||||||
|
#### (a) Create persistent config location
|
||||||
|
|
||||||
|
mkdir -p /var/openbmp/config
|
||||||
|
chmod 777 /var/openbmp/config
|
||||||
|
|
||||||
|
##### config/hosts
|
||||||
|
You can add custom host entries so that the collector will reverse lookup IP addresses
|
||||||
|
using a persistent hosts file.
|
||||||
|
|
||||||
|
Run docker with ```-v /var/openbmp/config:/config``` to make use of the persistent config files.
|
||||||
|
|
||||||
|
##### config/obmp-psql.yml
|
||||||
|
If the [obmp-psql.yml](https://github.com/OpenBMP/obmp-postgres/blob/master/src/main/resources/obmp-psql.yml) file
|
||||||
|
does not exist, a default one will be created. You should update this based on your settings. This file
|
||||||
|
is inline documented.
|
||||||
|
|
||||||
|
|
||||||
|
### 3) Run docker container
|
||||||
|
|
||||||
|
> Running the docker container for the first time will download the container image.
|
||||||
|
|
||||||
|
#### Environment Variables
|
||||||
|
Below table lists the environment variables that can be used with ``docker run -e <name=value>``
|
||||||
|
|
||||||
|
NAME | Value | Details
|
||||||
|
:---- | ----- |:-------
|
||||||
|
KAFKA\_FQDN | hostanme or IP | Kafka broker hostname. Hostname can be an IP address.
|
||||||
|
ENABLE_RPKI | 1 | Set to 1 to eanble RPKI. RPKI is disabled by default
|
||||||
|
ENABLE_IRR | 1 | Set to 1 to enable IRR. IRR is disabled by default
|
||||||
|
MEM | number | Number value in GB to allocate to Postgres. This will be the shared_buffers value.
|
||||||
|
PGUSER | username | Postgres username, default is **openbmp**
|
||||||
|
PGPASSWORD | password | Postgres password, default is **openbmp**
|
||||||
|
PGDATABASE | database | Name of postgres database, default is **openbmp**
|
||||||
|
|
||||||
|
#### Docker Run obmp-psql-app
|
||||||
|
> **NOTE:**
|
||||||
|
> If the container fails to start, it's likely due to the configuration. Check using
|
||||||
|
> ```docker logs obmp-psql-app```
|
||||||
|
|
||||||
|
```
|
||||||
|
docker run --rm -d --name obmp-psql-app \
|
||||||
|
-h obmp-psql-app \
|
||||||
|
-e ENABLE_RPKI=1 \
|
||||||
|
-e ENABLE_IRR=1 \
|
||||||
|
-e KAFKA_FQDN=kafka \
|
||||||
|
-e MEM=16 \
|
||||||
|
-v /var/openbmp/config:/config \
|
||||||
|
-p 9005:9005 -p 8080:8080 \
|
||||||
|
openbmp/psql-app:build-50
|
||||||
|
```
|
||||||
|
|
||||||
|
### Monitoring/Troubleshooting
|
||||||
|
|
||||||
|
Useful commands:
|
||||||
|
|
||||||
|
- docker logs obmp-psql-app
|
||||||
|
- docker exec obmp-psql-app tail -f /var/log/obmp-psql.log
|
||||||
|
- docker exec obmp-psql-app tail -f /var/log/postgresql/postgresql-10-main.log
|
||||||
|
- docker exec -it obmp-psql-app bash
|
||||||
|
|
||||||
243
psql-app/scripts/run
Executable file
243
psql-app/scripts/run
Executable file
@ -0,0 +1,243 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Postgres Backend: Run script
|
||||||
|
#
|
||||||
|
# Copyright (c) 2021 Cisco Systems, Inc. and Tim Evens. All rights reserved.
|
||||||
|
#
|
||||||
|
# Author: Tim Evens <tim@evensweb.com>
|
||||||
|
#
|
||||||
|
|
||||||
|
# Postgres details - Can be set using docker -e
|
||||||
|
export POSTGRES_USER=${POSTGRES_USER:="openbmp"}
|
||||||
|
export POSTGRES_PASSWORD=${POSTGRES_PASSWORD:="openbmp"}
|
||||||
|
export POSTGRES_HOST=${POSTGRES_HOST:="127.0.0.1"}
|
||||||
|
export POSTGRES_DB=${POSTGRES_DB:="openbmp"}
|
||||||
|
export MEM=${MEM:="1"} # mem in gigabytes
|
||||||
|
export PGCONNECT_TIMEOUT=15
|
||||||
|
|
||||||
|
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
# Functions
|
||||||
|
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
|
||||||
|
# -----------------------------------------------
|
||||||
|
# Check Kafka to make sure it's valid
|
||||||
|
# -----------------------------------------------
|
||||||
|
check_kafka() {
|
||||||
|
echo "===> Performing Kafka check"
|
||||||
|
|
||||||
|
if [[ ${KAFKA_FQDN:-""} == "" ]]; then
|
||||||
|
echo "ERROR: Missing ENV KAFKA_FQDN. Cannot proceed until you add that in docker run -e KAFKA_FQDN=<...>"
|
||||||
|
exit 1
|
||||||
|
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "===> Checking Kafka bootstrap server connection"
|
||||||
|
kafkacat -u -b $KAFKA_FQDN -L | grep broker
|
||||||
|
|
||||||
|
if [[ $? -ne 0 ]]; then
|
||||||
|
echo "ERROR: Failed to connect to Kafka at $KAFKA_FQDN, check the docker run -e KAFKA_FQDN= value"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "testing" | timeout 5 kafkacat -b $KAFKA_FQDN -P -t openbmp.parsed.test
|
||||||
|
echo "===> Checking if we can successfully consume messages"
|
||||||
|
timeout 5 kafkacat -u -b $KAFKA_FQDN -C -c 1 -o beginning -t openbmp.parsed.test > /dev/null
|
||||||
|
|
||||||
|
if [[ $? -ne 0 ]]; then
|
||||||
|
echo "ERROR: Failed to connect to Kafka broker, check the Kafka 'advertised.listeners' configuration."
|
||||||
|
echo " Advertised hostname must be reachable within the container. You can run this container"
|
||||||
|
echo " with --add-host <hostname>:<ip> to map the ip address within the container."
|
||||||
|
echo " You can also add/update the persistent /config/hosts file with the broker hostname/ip."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# -----------------------------------------------
|
||||||
|
# Configure Postgres shell profile
|
||||||
|
# -----------------------------------------------
|
||||||
|
config_postgres_profile() {
|
||||||
|
echo "===> Configuring PostgreSQL Shell Profile"
|
||||||
|
|
||||||
|
echo "export PGUSER=$POSTGRES_USER" > /usr/local/openbmp/pg_profile
|
||||||
|
echo "export PGPASSWORD=$POSTGRES_PASSWORD" >> /usr/local/openbmp/pg_profile
|
||||||
|
echo "export PGHOST=$POSTGRES_HOST" >> /usr/local/openbmp/pg_profile
|
||||||
|
echo "export PGDATABASE=$POSTGRES_DB" >> /usr/local/openbmp/pg_profile
|
||||||
|
}
|
||||||
|
|
||||||
|
# -----------------------------------------------
|
||||||
|
# Initdb Postgres
|
||||||
|
# -----------------------------------------------
|
||||||
|
initdb_postgres() {
|
||||||
|
echo " ===> Initializing the DB"
|
||||||
|
|
||||||
|
# Load the schema files
|
||||||
|
echo " ===> Loading Schemas"
|
||||||
|
|
||||||
|
echo "------" > /var/log/db_schema_load.log
|
||||||
|
for file in $(ls -v /usr/local/openbmp/db_schema/*.sql); do
|
||||||
|
echo " ===[ $file ] ========================================" >> /var/log/db_schema_load.log
|
||||||
|
su - -c "psql -U $POSTGRES_USER $POSTGRES_DB < $file" >> /var/log/db_schema_load.log 2>&1
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
# -----------------------------------------------
|
||||||
|
# Update hosts file
|
||||||
|
# -----------------------------------------------
|
||||||
|
update_hosts() {
|
||||||
|
echo "===> Updating /etc/hosts"
|
||||||
|
|
||||||
|
# Update the etc hosts file
|
||||||
|
if [[ -f /config/hosts ]]; then
|
||||||
|
cat /config/hosts >> /etc/hosts
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# -----------------------------------------------
|
||||||
|
# Enable RPKI
|
||||||
|
# -----------------------------------------------
|
||||||
|
enable_rpki() {
|
||||||
|
echo "===> Enabling RPKI"
|
||||||
|
|
||||||
|
cat > /etc/cron.d/openbmp-rpki <<SETVAR
|
||||||
|
MAILTO=""
|
||||||
|
|
||||||
|
# Update RPKI
|
||||||
|
31 */2 * * * root . /usr/local/openbmp/pg_profile && /usr/local/openbmp/rpki_validator.py -u $PGUSER -p $PGPASSWORD -s 127.0.0.1:8080 $PGHOST
|
||||||
|
|
||||||
|
SETVAR
|
||||||
|
|
||||||
|
echo "===> Downloading and Installing ARIN TAL"
|
||||||
|
wget -q https://www.arin.net/resources/manage/rpki/arin-rfc7730.tal -O /usr/local/rpki/preconfigured-tals/arin.tal
|
||||||
|
|
||||||
|
# start RPKI validator
|
||||||
|
/usr/local/rpki/rpki-validator-3.sh > /var/log/rpki-validator.log &
|
||||||
|
}
|
||||||
|
|
||||||
|
# -----------------------------------------------
|
||||||
|
# Enable IRR
|
||||||
|
# -----------------------------------------------
|
||||||
|
enable_irr() {
|
||||||
|
echo "===> Enabling IRR"
|
||||||
|
|
||||||
|
cat > /etc/cron.d/openbmp-irr <<SETVAR
|
||||||
|
MAILTO=""
|
||||||
|
|
||||||
|
# Update IRR
|
||||||
|
1 1 * * * root . /usr/local/openbmp/pg_profile && /usr/local/openbmp/gen_whois_route.py -u $PGUSER -p $PGPASSWORD $PGHOST > /var/log/irr_load.log
|
||||||
|
|
||||||
|
SETVAR
|
||||||
|
|
||||||
|
# Load IRR data
|
||||||
|
echo "Loading IRR data"
|
||||||
|
/usr/local/openbmp/gen_whois_route.py -u $PGUSER -p $PGPASSWORD $PGHOST > /var/log/irr_load.log &
|
||||||
|
}
|
||||||
|
|
||||||
|
# -----------------------------------------------
|
||||||
|
# config_cron
|
||||||
|
# -----------------------------------------------
|
||||||
|
config_cron() {
|
||||||
|
cat > /etc/cron.d/openbmp <<SETVAR
|
||||||
|
MAILTO=""
|
||||||
|
|
||||||
|
# Update ASN info
|
||||||
|
6 */2 * * * root . /usr/local/openbmp/pg_profile && /usr/local/openbmp/gen_whois_asn.py -u $PGUSER -p $PGPASSWORD $PGHOST >> /var/log/asn_load.log
|
||||||
|
|
||||||
|
# Update aggregation table stats
|
||||||
|
*/3 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_chg_stats('8 minute')"
|
||||||
|
|
||||||
|
# Update peer rib counts
|
||||||
|
*/15 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_peer_rib_counts()"
|
||||||
|
|
||||||
|
# Update peer update counts
|
||||||
|
*/30 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_peer_update_counts(1800)"
|
||||||
|
|
||||||
|
# Update origin stats
|
||||||
|
21 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_global_ip_rib();"
|
||||||
|
|
||||||
|
# Purge time series data that is older than desired retention
|
||||||
|
0 * */3 * * root . /usr/local/openbmp/pg_profile && psql -c "SELECT drop_chunks(interval '4 weeks');"
|
||||||
|
|
||||||
|
SETVAR
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
# -----------------------------------------------
|
||||||
|
# run_consumer
|
||||||
|
# -----------------------------------------------
|
||||||
|
run_consumer() {
|
||||||
|
echo "===> Starting consumer"
|
||||||
|
|
||||||
|
if [[ ! -f /config/obmp-psql.yml ]]; then
|
||||||
|
cd /config
|
||||||
|
unzip /usr/local/openbmp/obmp-psql-consumer.jar obmp-psql.yml
|
||||||
|
|
||||||
|
|
||||||
|
if [[ ! -f /config/obmp-psql.yml ]]; then
|
||||||
|
echo "ERROR: Cannot create /config/obmp-psql.yml"
|
||||||
|
echo " Update permissions on /config volume to 7777 OR add configuration file to /config volume"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Update configuration
|
||||||
|
sed -i -e "s/\([ ]*bootstrap.servers:\)\(.*\)/\1 \"${KAFKA_FQDN}\"/" /config/obmp-psql.yml
|
||||||
|
sed -i -e "s/\([ ]*host[ ]*:\)\(.*\)/\1 \"${POSTGRES_HOST}\"/" /config/obmp-psql.yml
|
||||||
|
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ $MEM -gt 20 ]]; then
|
||||||
|
heap_mem="3g"
|
||||||
|
else
|
||||||
|
heap_mem="1536m"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Run
|
||||||
|
cd /var/log
|
||||||
|
java -Xmx${heap_mem} -Xms128m -XX:+UseG1GC -XX:+UnlockExperimentalVMOptions \
|
||||||
|
-XX:InitiatingHeapOccupancyPercent=30 -XX:G1MixedGCLiveThresholdPercent=30 \
|
||||||
|
-XX:MaxGCPauseMillis=200 -XX:ParallelGCThreads=20 -XX:ConcGCThreads=5 \
|
||||||
|
-Duser.timezone=UTC \
|
||||||
|
-jar /usr/local/openbmp/obmp-psql-consumer.jar \
|
||||||
|
-cf /config/obmp-psql.yml > /var/log/psql-console.log &
|
||||||
|
|
||||||
|
cd /tmp
|
||||||
|
}
|
||||||
|
|
||||||
|
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
# Run
|
||||||
|
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
SYS_NUM_CPU=$(grep processor /proc/cpuinfo | wc -l)
|
||||||
|
|
||||||
|
update_hosts
|
||||||
|
|
||||||
|
check_kafka
|
||||||
|
|
||||||
|
config_cron
|
||||||
|
|
||||||
|
config_postgres_profile
|
||||||
|
|
||||||
|
rm -f /etc/cron.d/openbmp-rpki
|
||||||
|
if [[ ${ENABLE_RPKI:-""} != "" ]]; then
|
||||||
|
enable_rpki
|
||||||
|
fi
|
||||||
|
|
||||||
|
rm -f /etc/cron.d/openbmp-irr
|
||||||
|
if [[ ${ENABLE_IRR:-""} != "" ]]; then
|
||||||
|
enable_irr
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
|
initdb_postgres
|
||||||
|
|
||||||
|
# Get rid of previous rsyslogd pid
|
||||||
|
rm -f /var/run/rsyslogd.pid
|
||||||
|
|
||||||
|
#service rsyslog start
|
||||||
|
#service cron start
|
||||||
|
|
||||||
|
|
||||||
|
run_consumer
|
||||||
|
|
||||||
|
echo "===> Now running!!!"
|
||||||
|
|
||||||
|
while [ 1 ]; do
|
||||||
|
sleep 1800
|
||||||
|
done
|
||||||
Loading…
x
Reference in New Issue
Block a user