diff --git a/README.md b/README.md index 9025669..65ff30c 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Each docker file contains a readme file, see below: * [Collector](collector/README.md) * [PostgreSQL](postgres/README.md) -* [PSQL Consumer](psql-consumer/README.md) +* [PSQL Consumer](psql-app/README.md) ## Using Docker Compose to run everything @@ -39,8 +39,33 @@ Each docker file contains a readme file, see below: You will need docker-compose. You can install that via [Docker Compose](https://docs.docker.com/compose/install/) instructions. Docker compose will run everything, including handling restarts of containers. +#### (1) Mount/Make persistent directories +Create expected directories. You can choose to mount these as well or update the compose file to change them. +> **NOTE** +> If you are using OSX/Mac, then you will need to update your docker preferences to allow ```/var/openbmp``` + +Make sure to create the **OBMP_DATA_ROOT** directory first. ``` -docker-compose -p obmp up +export OBMP_DATA_ROOT=/var/openbmp +sudo mkdir -p $OBMP_DATA_ROOT +``` + +Create sub directories +``` +mkdir -p ${OBMP_DATA_ROOT}/config +mkdir -p ${OBMP_DATA_ROOT``}/kafka-data +mkdir -p ${OBMP_DATA_ROOT}/postgres/data +mkdir -p ${OBMP_DATA_ROOT}/postgres/ts +mkdir -p ${OBMP_DATA_ROOT}/grafana + +sudo chmod -R 7777 $OBMP_DATA_ROOT +``` + + +Change ```OBMP_DATA_ROOT=``` to where you created the directories above. The default is ```/var/openbmp``` + +``` +OBMP_DATA_ROOT=/var/openbmp docker-compose -p obmp up -d ``` diff --git a/collector/scripts/run b/collector/scripts/run index 5775d23..5cd5b3c 100755 --- a/collector/scripts/run +++ b/collector/scripts/run @@ -54,7 +54,7 @@ fi # Update openbmpd config file OPENBMP_CFG_FILE=/usr/etc/openbmp/openbmpd.conf sed -r -i "s/admin_id:.*/admin_id: ${ADMIN_ID}/" /usr/etc/openbmp/openbmpd.conf -sed -r -i "s/localhost:9092/${KAFKA_FQDN}:9092/" /usr/etc/openbmp/openbmpd.conf +sed -r -i "s/localhost:9092/${KAFKA_FQDN}/" /usr/etc/openbmp/openbmpd.conf if [[ -f /config/openbmpd.conf ]]; then OPENBMP_CFG_FILE=/config/openbmpd.conf diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..ad18d48 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,111 @@ +--- +version: '3' +services: + + zookeeper: + restart: unless-stopped + container_name: zookeeper + image: confluentinc/cp-zookeeper:6.0.2 + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + + kafka: + restart: unless-stopped + container_name: kafka + image: confluentinc/cp-kafka:6.0.2 + + # Change the mount point to where you want to store Kafka data. + # Normally 80GB or more + volumes: + - ${OBMP_DATA_ROOT}/kafka-data:/var/lib/kafka/data + depends_on: + - zookeeper + ports: + - 9092:9092 + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + + # Change/add listeners based on your FQDN that the host and other containers can access. You can use + # an IP address as well. By default, only within the compose/containers can Kafka be accesssed + # using port 29092. Outside access can be enabled, but you should use an FQDN listener. + #KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://:9092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_NUM_PARTITIONS: 8 + KAFKA_LOG_RETENTION_MINUTES: 90 + KAFKA_LOG_ROLL_MS: 3600000 + KAFKA_LOG_SEGMENT_BYTES: 1073741824 + KAFKA_MESSAGE_MAX_BYTES: 100000000 + KAFKA_LOG_CLEANER_THREADS: 2 + + grafana: + restart: unless-stopped + container_name: grafana + image: grafana/grafana:latest + ports: + - 3000:3000 + volumes: + - ${OBMP_DATA_ROOT}/grafana:/var/lib/grafana + - ${OBMP_DATA_ROOT}/grafana-provisioning/:/etc/grafana/provisioning/ + environment: + - GF_SECURITY_ADMIN_PASSWORD=openbmp + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_SERVER_ROOT_URL=/ # Change this if you have grafana behind URL/paths, such as /grafana/ + - GF_INSTALL_PLUGINS=agenty-flowcharting-panel,briangann-datatable-panel,digrich-bubblechart-panel,grafana-piechart-panel,grafana-worldmap-panel,vonage-status-panel + + psql: + restart: unless-stopped + container_name: psql + image: openbmp/postgres:build-50 + ports: + - 5432:5432 + volumes: + - ${OBMP_DATA_ROOT}/postgres/data:/var/lib/postgresql/data # change this to 80GB SSD slice/partition + - ${OBMP_DATA_ROOT}/postgres/ts:/var/lib/postgresql/ts # Chnage this to 500GB SSD slice/partition + environment: + - MEM=2 # Set MEM to at least 2GB but ideally >16GB + - POSTGRES_PASSWORD=openbmp + - POSTGRES_USER=openbmp + - POSTGRES_DB=openbmp + + collector: + restart: unless-stopped + container_name: collector + image: openbmp/collector:build-50 + sysctls: + - net.ipv4.tcp_keepalive_intvl=30 + - net.ipv4.tcp_keepalive_probes=5 + - net.ipv4.tcp_keepalive_time=180 + ports: + - 5000:5000 + volumes: + - ${OBMP_DATA_ROOT}/config:/config + environment: + - KAFKA_FQDN=kafka:29092 + + psql-app: + restart: unless-stopped + container_name: psql-app + image: openbmp/psql-app:build-50 + sysctls: + - net.ipv4.tcp_keepalive_intvl=30 + - net.ipv4.tcp_keepalive_probes=5 + - net.ipv4.tcp_keepalive_time=180 + ports: + - 9005:9005 + - 8080:8080 + volumes: + - ${OBMP_DATA_ROOT}/config:/config + environment: + - MEM=2 # Set memory to at least 2GB but ideally 4GB + - KAFKA_FQDN=kafka:29092 + - ENABLE_RPKI=0 # 1 enables, 0 disables RPKI sync + - ENABLE_IRR=0 # 1 enables, 0 disables IRR sync + - POSTGRES_PASSWORD=openbmp + - POSTGRES_USER=openbmp + - POSTGRES_DB=openbmp + - POSTGRES_HOST=psql:5432 diff --git a/postgres/Dockerfile b/postgres/Dockerfile index 7855de9..1a29f9f 100644 --- a/postgres/Dockerfile +++ b/postgres/Dockerfile @@ -18,6 +18,11 @@ FROM timescale/timescaledb:2.1.0-pg13 VOLUME ["/ws"] WORKDIR /ws +# Expected data locations for base tables and timeseries +# +VOLUME ["/var/lib/postgresql/data"] +VOLUME ["/var/lib/postgresql/ts"] + RUN apk update \ && apk add openssl \ @@ -31,10 +36,12 @@ RUN apk update \ && sed -i -e "s/^\#*listen_addresses.*=.*/listen_addresses = '*'/" /usr/local/share/postgresql/postgresql.conf.sample \ && sed -i -e "s/^\#*ssl[ ]*=.*/ssl = on/" /usr/local/share/postgresql/postgresql.conf.sample \ && sed -i -e "s/^\#*ssl_cert_file.*=.*/ssl_cert_file = \'\/psql_server.crt\'/" /usr/local/share/postgresql/postgresql.conf.sample \ - && sed -i -e "s/^\#*ssl_key_file.*=.*/ssl_key_file = \'\/psql_server.key\'/" /usr/local/share/postgresql/postgresql.conf.sample + && sed -i -e "s/^\#*ssl_key_file.*=.*/ssl_key_file = \'\/psql_server.key\'/" /usr/local/share/postgresql/postgresql.conf.sample \ + && sed -i -e "s/^\#*shared_buffers.*=.*/shared_buffers = ${MEM:-'1'}GB/" /usr/local/share/postgresql/postgresql.conf.sample \ + && sed -i -e "s/^\#*work_mem.*=.*/work_mem = $(( (${MEM:-1} * 1024) * 5 / 100))MB/" /usr/local/share/postgresql/postgresql.conf.sample + +# Init timesries location +RUN echo 'psql -U $POSTGRES_USER -c "CREATE TABLESPACE timeseries LOCATION '\''/var/lib/postgresql/ts'\'';" $POSTGRES_DB' > /docker-entrypoint-initdb.d/0_obmp_init.sh + -#RUN echo 'psql -U $POSTGRES_USER -c "CREATE TABLESPACE timeseries LOCATION '\''/var/lib/postgresql/ts'\'';" $POSTGRES_DB' > /docker-entrypoint-initdb.d/0_cnis_schema.sh \ -# && echo 'psql -U $POSTGRES_USER $POSTGRES_DB < /tmp/cnis-mq.sql' >> /docker-entrypoint-initdb.d/0_cnis_schema.sh \ -# && echo 'psql -U $POSTGRES_USER $POSTGRES_DB < /tmp/cnis-mq-functions.sql' >> /docker-entrypoint-initdb.d/0_cnis_schema.sh \ -# && echo 'psql -U $POSTGRES_USER $POSTGRES_DB < /tmp/dev-psql.sql' >> /docker-entrypoint-initdb.d/0_cnis_schema.sh diff --git a/postgres/README.md b/postgres/README.md index 11d1d6d..dbae4fc 100644 --- a/postgres/README.md +++ b/postgres/README.md @@ -20,3 +20,70 @@ See both [Postgres](https://hub.docker.com/_/postgres) and [TimescaleDB](https://hub.docker.com/r/timescale/timescaledb) documentation for more information on how to configure/run the docker container. +### PostgreSQL Related + +#### Postgres can be killed by the Linux OOM-Killer +This is very bad as it causes Postgres to restart. This will happen because postgres uses a large shared buffer, +which causes the OOM to believe it's using a lot of VM. + +It is suggested to run the postgres server with the following Linux settings: + + # Update runtime + sysctl -w vm.vfs_cache_pressure=500 + sysctl -w vm.swappiness=10 + sysctl -w vm.min_free_kbytes=1000000 + sysctl -w vm.overcommit_memory=2 + sysctl -w vm.overcommit_ratio=95 + + # Update startup + echo "vm.vfs_cache_pressure=500" >> /etc/sysctl.conf + echo "vm.min_free_kbytes=1000000" >> /etc/sysctl.conf + echo "vm.swappiness=10" >> /etc/sysctl.conf + echo "vm.overcommit_memory=2" >> /etc/sysctl.conf + echo "vm.overcommit_ratio=95" >> /etc/sysctl.conf + + +See Postgres [hugepages](https://www.postgresql.org/docs/current/static/kernel-resources.html#LINUX-HUGE-PAGES) for +details on how to enable and use hugepages. Some Linux distributions enable **transparent hugepages** which +will prevent the ability to configure ```vm.nr_hugepages```. If you find that you cannot set ```vm.nr_hugepages```, +then try the below: + + echo never > /sys/kernel/mm/transparent_hugepage/enabled + echo never > /sys/kernel/mm/transparent_hugepage/defrag + sync && echo 3 > /proc/sys/vm/drop_caches + + +#### Postgres Vacuum (reclaim disk space) +Postgres reclaims deleted/updated records using the vacuum process. You can run this manually/cron via the +```VACUUM``` command. **autovacuum** is used to do this periodically. Careful tuning of this +is required. Checkout [autovacuum-tuning-basics](https://blog.2ndquadrant.com/autovacuum-tuning-basics/), +[Routine Vacuuming](https://www.postgresql.org/docs/current/static/routine-vacuuming.html), and +[VACUUM](https://www.postgresql.org/docs/current/static/sql-vacuum.html) for more details. + +#### Create persistent postgres locations + +*You should use fast SSD and/or ZFS.* Size of these locations/mount points are directly related to the +number of NLRI's maintained and number of changes/updates per second. + +> TODO: Will post numbers of how to determine the disk size needed. For now, if you have less +> than 50,000,00 prefixes, then you can use 1TB. If you have more than that, you should consider +> multiple disks. ZFS can make your life easier as you can easily add disks and it supports compression. + +- **postgres/main** - This location will be used for the main postgres data + files and tables. + +> This really should be a mount point to a dedicated filesystem + +``` + mkdir -p /var/openbmp/postgres/main + chmod 7777 /var/openbmp/postgres/main +``` + +- **postgres/ts** - This location will be used for the time series postgres tables + +> This really should be a mount point to a dedicated filesystem + +``` + mkdir -p /var/openbmp/postgres/ts + chmod 7777 /var/openbmp/postgres/ts +``` diff --git a/psql-app/Dockerfile b/psql-app/Dockerfile new file mode 100644 index 0000000..6764402 --- /dev/null +++ b/psql-app/Dockerfile @@ -0,0 +1,115 @@ +# Postgres Backend: openbmp/psql-consumer +# +# Copyright (c) 2021 Cisco Systems, Inc. and Tim Evens. All rights reserved. +# +# Author: Tim Evens +# +# Docker context does not support multiple paths or mounting volumes for builds. +# In effort to build the container from local git clones, we dynamically build a context +# +# Clone the obmp-psql, obmp-java-api-message, and obmp-docker repos into the same directory. +# Change directories to obmp-docker/psql-app and run the below from that +# directory. +# +# Example docker build: +# tar -cL -C ../../ ./obmp-psql ./obmp-docker/psql-app ./obmp-java-api-message \ +# | docker build --build-arg BUILD_NUMBER=50 \ +# -f obmp-docker/psql-app/Dockerfile -t openbmp/psql-app:build-50 - + +# ----------------------------------------------- +# stage: Build +# ----------------------------------------------- +FROM openbmp/dev-image:latest AS build + +ARG BUILD_NUMBER=0 + +# Proxy servers +#ENV http_proxy http://proxy:80 +#ENV https_proxy http://proxy:80 +#ENV no_proxy "domain.com" + +COPY obmp-psql/ /ws +COPY obmp-java-api-message/ /tmp/obmp-java-api-message +WORKDIR /ws + +RUN cd /tmp/obmp-java-api-message \ + && mvn clean install \ + && cd /ws \ + && mvn clean package + +# ----------------------------------------------- +# stage: Final container +# ----------------------------------------------- +FROM openjdk:17-slim + +# Copy files from previous stages +COPY --from=build /ws/target/obmp-psql-consumer-0.1.0-SNAPSHOT.jar /usr/local/openbmp/obmp-psql-consumer.jar +COPY --from=build /ws/database/ /usr/local/openbmp/database +COPY --from=build /ws/cron_scripts/gen-whois/*.py /usr/local/openbmp/ +COPY --from=build /ws/cron_scripts/rpki/*.py /usr/local/openbmp/ +COPY --from=build /ws/scripts/dbip-to-psql.py /usr/local/openbmp/ + +# Add files +ADD obmp-docker/psql-app/scripts/run /usr/sbin/ + +#---------------------------------- +# Define persistent data volumes +VOLUME ["/config"] + +#---------------------------------- +# Expose ports. + +# Consumer JMX console +EXPOSE 9005 + +# RPKI Validator port +EXPOSE 8080 + +#---------------------------------- +# Define working directory. +WORKDIR /tmp + +# Base setup tasks +RUN touch /usr/local/build-${BUILD_NUMBER} \ + && chmod 755 /usr/local/openbmp/*.py + +#---------------------------------- +# Install depends +# Depends are installed in layers so that this doesn't have to be done each time +# the image is built. +RUN apt-get update \ + && apt-get install --allow-unauthenticated -y unzip curl wget whois vim rsyslog cron rsync kafkacat \ + procps python3-minimal python3-distutils python3-psycopg2 python3-dnspython postgresql-client \ + && ln -s /usr/bin/python3 /usr/bin/python + +RUN cd /tmp && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \ + && python3 get-pip.py + +RUN pip install ipaddr + +# Install latest postgres client +#RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' \ +# && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - \ +# && apt-get update +# && apt-get install postgresql-13-client + +# Cleanup +RUN apt-get autoremove && apt-get clean + +#---------------------------------- +# Install RPKI validator (https://github.com/RIPE-NCC/rpki-validator-3/wiki) +RUN mkdir /usr/local/rpki && cd /tmp \ + && wget https://ftp.ripe.net/tools/rpki/validator3/prod/generic/rpki-validator-3-latest-dist.tar.gz \ + && tar xzf rpki-validator-3-latest-dist.tar.gz \ + && cd rpki-validator-*/ \ + && mv * /usr/local/rpki/ \ + && rm -rf /tmp/rpki-* \ + && cd /usr/local/rpki \ + && sed -i -r 's/.*server.address=.*/server.address=0.0.0.0/' /usr/local/rpki/conf/application.properties \ + && sed -i -r 's/jvm.mem.maximum=.*/jvm.mem.maximum=2g/' /usr/local/rpki/conf/application.properties + + +#---------------------------------- +# Define default command. +CMD ["/usr/sbin/run"] + diff --git a/psql-app/README.md b/psql-app/README.md new file mode 100644 index 0000000..3687b96 --- /dev/null +++ b/psql-app/README.md @@ -0,0 +1,129 @@ +# OpenBMP Postgres Application Container +This container is the main application container for OpenBMP and PostgreSQL. + +It provides: + +* PostgreSQL consumer +* RPKI validator improt/sync +* IRR and peering DB import/sync +* Schedules and runs the metric DB functions +* Schedules and runs the DB timescale DB chunk drops + +## Building +See the [Dockerfile](Dockerfile) notes for build instructions. + +## Running + +### Kafka Validation Testing +The Kafka setup can be tricky due to docker networking between containers and remote systems. Kafka clustering +makes use of a bootstrap server which will advertise each broker ```hostname:port``` that the consumer/producer +will use. Each consumer/producer will connect to the brokers using these **advertised** hostnames and ports. The +setting in Kafka to configure the broker hostname is ```advertised.listeners```. + +The postgres container (**this container**) uses the **KAFKA_FQDN** as the bootstrap server, +syntax is ``````. This will work with an +IP or hostname. When using a hostname, the hostname *MUST* resolve within the container. While this may work for +bootstrap server conection, the advertised hostnames need to also resolve in the container. + +**Kafka Validation is a 3 step process** + +1. Successfully connect to the bootstrap server and retrieve metadata (e.g. broker hostname:port) +2. Successfully produce a test message to ```openbmp.parsed.test``` topic +3. Successfully consume a test message from ```openbmp.parsed.test``` topic + +> **IMPORTANT** +> If using your own Kafka install, make sure you allow producing/consuming to/from **openbmp.parsed.test** +> for the consumer validation. + +### Hostnames in Container +You can map the Kafka hostname and each broker if they are different using two methods: + +1. add ```--add-host HOSTNAME:IP``` to **docker run** command. Make sure to add one for the bootstrap and each broker. +2. Create a **/var/openbmp/config/hosts** file and add the Kafka bootstrap and broker hostname to IP mappings. + +### VM Specifications + +#### Storage + +You will need to dedicate space for the postgres instance. Normally two partitions are used. A good +starting size for postgres main is 500GB and postgres ts (timescaleDB) is 1TB. Both disks +should be fast SSD. ZFS can be used on either of them to add compression. The size you need will depend +on the number of NLRI's and updates per second. + +#### Memory & CPU + +The size of memory will depend on the type of queries and number of NLRI's. A good starting point for +memory is a server with more than 48GB RAM. You can run on as little as 4GB RAM but that will only +scale to about 10,000,000 NLRI's. 64BG of RAM should scale to 150,000,000 NLRI's. + +The number of vCPU's also varies by the number of concurrent connections and how many threads you use for +the postgres consumer. A good starting point is at least 8 vCPU's. + + +### 1) Install docker +Follow the [Docker Instructions](https://docs.docker.com/install) to install docker CE. + +### 2) Add persistent volumes + +Persistent volumes make it possible for upgrades without loosing any data. + +#### (a) Create persistent config location + + mkdir -p /var/openbmp/config + chmod 777 /var/openbmp/config + +##### config/hosts +You can add custom host entries so that the collector will reverse lookup IP addresses +using a persistent hosts file. + +Run docker with ```-v /var/openbmp/config:/config``` to make use of the persistent config files. + +##### config/obmp-psql.yml +If the [obmp-psql.yml](https://github.com/OpenBMP/obmp-postgres/blob/master/src/main/resources/obmp-psql.yml) file +does not exist, a default one will be created. You should update this based on your settings. This file +is inline documented. + + +### 3) Run docker container + +> Running the docker container for the first time will download the container image. + +#### Environment Variables +Below table lists the environment variables that can be used with ``docker run -e `` + +NAME | Value | Details +:---- | ----- |:------- +KAFKA\_FQDN | hostanme or IP | Kafka broker hostname. Hostname can be an IP address. +ENABLE_RPKI | 1 | Set to 1 to eanble RPKI. RPKI is disabled by default +ENABLE_IRR | 1 | Set to 1 to enable IRR. IRR is disabled by default +MEM | number | Number value in GB to allocate to Postgres. This will be the shared_buffers value. +PGUSER | username | Postgres username, default is **openbmp** +PGPASSWORD | password | Postgres password, default is **openbmp** +PGDATABASE | database | Name of postgres database, default is **openbmp** + +#### Docker Run obmp-psql-app +> **NOTE:** +> If the container fails to start, it's likely due to the configuration. Check using +> ```docker logs obmp-psql-app``` + +``` +docker run --rm -d --name obmp-psql-app \ + -h obmp-psql-app \ + -e ENABLE_RPKI=1 \ + -e ENABLE_IRR=1 \ + -e KAFKA_FQDN=kafka \ + -e MEM=16 \ + -v /var/openbmp/config:/config \ + -p 9005:9005 -p 8080:8080 \ + openbmp/psql-app:build-50 +``` + +### Monitoring/Troubleshooting + +Useful commands: + +- docker logs obmp-psql-app +- docker exec obmp-psql-app tail -f /var/log/obmp-psql.log +- docker exec obmp-psql-app tail -f /var/log/postgresql/postgresql-10-main.log +- docker exec -it obmp-psql-app bash + diff --git a/psql-app/scripts/run b/psql-app/scripts/run new file mode 100755 index 0000000..07c49d6 --- /dev/null +++ b/psql-app/scripts/run @@ -0,0 +1,243 @@ +#!/bin/bash +# Postgres Backend: Run script +# +# Copyright (c) 2021 Cisco Systems, Inc. and Tim Evens. All rights reserved. +# +# Author: Tim Evens +# + +# Postgres details - Can be set using docker -e +export POSTGRES_USER=${POSTGRES_USER:="openbmp"} +export POSTGRES_PASSWORD=${POSTGRES_PASSWORD:="openbmp"} +export POSTGRES_HOST=${POSTGRES_HOST:="127.0.0.1"} +export POSTGRES_DB=${POSTGRES_DB:="openbmp"} +export MEM=${MEM:="1"} # mem in gigabytes +export PGCONNECT_TIMEOUT=15 + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +# Functions +# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +# ----------------------------------------------- +# Check Kafka to make sure it's valid +# ----------------------------------------------- +check_kafka() { + echo "===> Performing Kafka check" + + if [[ ${KAFKA_FQDN:-""} == "" ]]; then + echo "ERROR: Missing ENV KAFKA_FQDN. Cannot proceed until you add that in docker run -e KAFKA_FQDN=<...>" + exit 1 + + fi + + echo "===> Checking Kafka bootstrap server connection" + kafkacat -u -b $KAFKA_FQDN -L | grep broker + + if [[ $? -ne 0 ]]; then + echo "ERROR: Failed to connect to Kafka at $KAFKA_FQDN, check the docker run -e KAFKA_FQDN= value" + exit 1 + fi + + echo "testing" | timeout 5 kafkacat -b $KAFKA_FQDN -P -t openbmp.parsed.test + echo "===> Checking if we can successfully consume messages" + timeout 5 kafkacat -u -b $KAFKA_FQDN -C -c 1 -o beginning -t openbmp.parsed.test > /dev/null + + if [[ $? -ne 0 ]]; then + echo "ERROR: Failed to connect to Kafka broker, check the Kafka 'advertised.listeners' configuration." + echo " Advertised hostname must be reachable within the container. You can run this container" + echo " with --add-host : to map the ip address within the container." + echo " You can also add/update the persistent /config/hosts file with the broker hostname/ip." + exit 1 + fi +} + +# ----------------------------------------------- +# Configure Postgres shell profile +# ----------------------------------------------- +config_postgres_profile() { + echo "===> Configuring PostgreSQL Shell Profile" + + echo "export PGUSER=$POSTGRES_USER" > /usr/local/openbmp/pg_profile + echo "export PGPASSWORD=$POSTGRES_PASSWORD" >> /usr/local/openbmp/pg_profile + echo "export PGHOST=$POSTGRES_HOST" >> /usr/local/openbmp/pg_profile + echo "export PGDATABASE=$POSTGRES_DB" >> /usr/local/openbmp/pg_profile +} + +# ----------------------------------------------- +# Initdb Postgres +# ----------------------------------------------- +initdb_postgres() { + echo " ===> Initializing the DB" + + # Load the schema files + echo " ===> Loading Schemas" + + echo "------" > /var/log/db_schema_load.log + for file in $(ls -v /usr/local/openbmp/db_schema/*.sql); do + echo " ===[ $file ] ========================================" >> /var/log/db_schema_load.log + su - -c "psql -U $POSTGRES_USER $POSTGRES_DB < $file" >> /var/log/db_schema_load.log 2>&1 + done +} + +# ----------------------------------------------- +# Update hosts file +# ----------------------------------------------- +update_hosts() { + echo "===> Updating /etc/hosts" + + # Update the etc hosts file + if [[ -f /config/hosts ]]; then + cat /config/hosts >> /etc/hosts + fi +} + +# ----------------------------------------------- +# Enable RPKI +# ----------------------------------------------- +enable_rpki() { + echo "===> Enabling RPKI" + + cat > /etc/cron.d/openbmp-rpki < Downloading and Installing ARIN TAL" + wget -q https://www.arin.net/resources/manage/rpki/arin-rfc7730.tal -O /usr/local/rpki/preconfigured-tals/arin.tal + + # start RPKI validator + /usr/local/rpki/rpki-validator-3.sh > /var/log/rpki-validator.log & +} + +# ----------------------------------------------- +# Enable IRR +# ----------------------------------------------- +enable_irr() { + echo "===> Enabling IRR" + + cat > /etc/cron.d/openbmp-irr < /var/log/irr_load.log + +SETVAR + + # Load IRR data + echo "Loading IRR data" + /usr/local/openbmp/gen_whois_route.py -u $PGUSER -p $PGPASSWORD $PGHOST > /var/log/irr_load.log & +} + +# ----------------------------------------------- +# config_cron +# ----------------------------------------------- +config_cron() { + cat > /etc/cron.d/openbmp <> /var/log/asn_load.log + +# Update aggregation table stats +*/3 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_chg_stats('8 minute')" + +# Update peer rib counts +*/15 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_peer_rib_counts()" + +# Update peer update counts +*/30 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_peer_update_counts(1800)" + +# Update origin stats +21 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_global_ip_rib();" + +# Purge time series data that is older than desired retention +0 * */3 * * root . /usr/local/openbmp/pg_profile && psql -c "SELECT drop_chunks(interval '4 weeks');" + +SETVAR + +} + +# ----------------------------------------------- +# run_consumer +# ----------------------------------------------- +run_consumer() { + echo "===> Starting consumer" + + if [[ ! -f /config/obmp-psql.yml ]]; then + cd /config + unzip /usr/local/openbmp/obmp-psql-consumer.jar obmp-psql.yml + + + if [[ ! -f /config/obmp-psql.yml ]]; then + echo "ERROR: Cannot create /config/obmp-psql.yml" + echo " Update permissions on /config volume to 7777 OR add configuration file to /config volume" + exit 1 + fi + + # Update configuration + sed -i -e "s/\([ ]*bootstrap.servers:\)\(.*\)/\1 \"${KAFKA_FQDN}\"/" /config/obmp-psql.yml + sed -i -e "s/\([ ]*host[ ]*:\)\(.*\)/\1 \"${POSTGRES_HOST}\"/" /config/obmp-psql.yml + + fi + + if [[ $MEM -gt 20 ]]; then + heap_mem="3g" + else + heap_mem="1536m" + fi + + # Run + cd /var/log + java -Xmx${heap_mem} -Xms128m -XX:+UseG1GC -XX:+UnlockExperimentalVMOptions \ + -XX:InitiatingHeapOccupancyPercent=30 -XX:G1MixedGCLiveThresholdPercent=30 \ + -XX:MaxGCPauseMillis=200 -XX:ParallelGCThreads=20 -XX:ConcGCThreads=5 \ + -Duser.timezone=UTC \ + -jar /usr/local/openbmp/obmp-psql-consumer.jar \ + -cf /config/obmp-psql.yml > /var/log/psql-console.log & + + cd /tmp +} + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +# Run +# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +SYS_NUM_CPU=$(grep processor /proc/cpuinfo | wc -l) + +update_hosts + +check_kafka + +config_cron + +config_postgres_profile + +rm -f /etc/cron.d/openbmp-rpki +if [[ ${ENABLE_RPKI:-""} != "" ]]; then + enable_rpki +fi + +rm -f /etc/cron.d/openbmp-irr +if [[ ${ENABLE_IRR:-""} != "" ]]; then + enable_irr +fi + + +initdb_postgres + +# Get rid of previous rsyslogd pid +rm -f /var/run/rsyslogd.pid + +#service rsyslog start +#service cron start + + +run_consumer + +echo "===> Now running!!!" + +while [ 1 ]; do + sleep 1800 +done