Merge pull request #9 from OpenBMP/2.1.0

2.1.0

collector/Dockerfile

@@ -1,6 +1,7 @@
+# syntax=docker/dockerfile:1.3
 # Collector: openbmp/collector
 #
-#  Copyright (c) 2021-2022 Cisco Systems, Inc. and Tim Evens.  All rights reserved.
+#  Copyright (c) 2021-2022 Cisco Systems, Inc. and others.  All rights reserved.
 #
 # Author: Tim Evens <tim@openbmp.org>
 #
@@ -13,10 +14,13 @@
 #   Change directories to obmp-docker/collector and run the below from that
 #   directory.
 #
+# Use buildkit - either via export DOCKER_BUILDKIT=1 or by using buildx cli
+#
 #   Example docker build:
-#     tar -cL -C ../../ ./obmp-collector ./obmp-docker/collector \
+#     tar -c -C ../../ ./obmp-collector ./obmp-docker/collector \
-#           | docker build --build-arg VERSION=2.0.3 \
+#           | docker buildx build --progress=plain \
-#                 -f obmp-docker/collector/Dockerfile -t openbmp/collector:2.0.3 -
+#                 --build-arg VERSION=2.1.0 \
+#                 -f obmp-docker/collector/Dockerfile -t openbmp/collector:2.1.0 -
 #
 
 # -----------------------------------------------
@@ -27,7 +31,7 @@ FROM openbmp/dev-image:latest AS build
 COPY obmp-collector/ /ws
 WORKDIR /ws
 
-RUN mkdir -p build && cd build \
+RUN rm -rf build && mkdir -p build && cd build \
     && cmake -DCMAKE_INSTALL_PREFIX:PATH=/usr ../ \
     && make \
     && make install
@@ -40,13 +44,13 @@ RUN mkdir -p build && cd build \
 FROM debian:bullseye-slim
 
 # Add files.
-ADD obmp-docker/collector/scripts/install /tmp/
+ADD --chmod=755 obmp-docker/collector/scripts/install /tmp/
-ADD obmp-docker/collector/scripts/run /usr/sbin/
+ADD --chmod=755 obmp-docker/collector/scripts/run /usr/sbin/
 
 ARG VERSION=0
 
 # Copy files from previous stages
-COPY --from=build /usr/bin/openbmpd /usr/bin/
+COPY --chmod=755 --from=build /usr/bin/openbmpd /usr/bin/
 COPY --from=build /usr/etc/openbmp/openbmpd.conf /usr/etc/openbmp/openbmpd.conf
 COPY --from=build /etc/init/openbmpd.conf /etc/init/openbmpd.conf
 COPY --from=build /etc/default/openbmpd.new /etc/default/openbmpd

dev-image/Dockerfile

@@ -1,14 +1,12 @@
 # Development build container: openbmp/dev-image
 #
 #  Copyright (c) 2021-2022 Cisco Systems, Inc. and others.  All rights reserved.
-#  Copyright (c) 2021-2022 Tim Evens.  All rights reserved.
 #
 #  Container used to build the OpenBMP components
 #
 # Author: Tim Evens <tim@openbmp.org>
 #
-# BUILD: docker build -t openbmp/dev-image .
+# BUILD: docker buildx build --progress=plain -t openbmp/dev-image:latest .
-#        docker tag openbmp/dev-image openbmp/dev-image:latest
 
 FROM debian:bullseye-slim AS build
 
@@ -22,13 +20,12 @@ RUN apt-get install -y openjdk-17-jdk-headless maven
 RUN mkdir -p /usr/share/man/man1/ \
     && apt-get -y install git gcc g++ libboost-dev cmake zlib1g-dev libssl-dev libsasl2-dev \
          liblz4-dev libzstd-dev librdkafka-dev
-
-
 # Build/install yaml-cpp
 RUN cd /tmp && git clone https://github.com/jbeder/yaml-cpp.git \
     && cd yaml-cpp \
+    && git checkout yaml-cpp-0.7.0 \
     && mkdir build && cd build \
-    && cmake -DBUILD_SHARED_LIBS=OFF .. \
+    && cmake -DYAML_BUILD_SHARED_LIB=OFF .. \
     && make && make install \
     && cd /tmp
 

docker-compose.yml

@@ -25,7 +25,7 @@ services:
     depends_on:
       - zookeeper
     ports:
-      - 9092:9092
+      - "9092:9092"
     environment:
       KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: obmp-zookeeper:2181
@@ -50,7 +50,7 @@ services:
     container_name: obmp-grafana
     image: grafana/grafana:8.3.4
     ports:
-      - 3000:3000
+      - "3000:3000"
     volumes:
       - ${OBMP_DATA_ROOT}/grafana:/var/lib/grafana
       - ${OBMP_DATA_ROOT}/grafana/provisioning:/etc/grafana/provisioning/
@@ -58,14 +58,15 @@ services:
       - GF_SECURITY_ADMIN_PASSWORD=openbmp
       - GF_AUTH_ANONYMOUS_ENABLED=true
       - GF_SERVER_ROOT_URL=/   # Change this if you have grafana behind URL/paths, such as /grafana/
-      - GF_INSTALL_PLUGINS=agenty-flowcharting-panel,briangann-datatable-panel,digrich-bubblechart-panel,grafana-piechart-panel,grafana-worldmap-panel,vonage-status-panel
+      - GF_INSTALL_PLUGINS=agenty-flowcharting-panel,grafana-piechart-panel,grafana-worldmap-panel,grafana-simple-json-datasource,vonage-status-panel
 
   psql:
     restart: unless-stopped
     container_name: obmp-psql
-    image: openbmp/postgres:2.0.3
+    image: openbmp/postgres:2.1.0
+    shm_size: 1024m
     ports:
-      - 5432:5432
+      - "5432:5432"
     volumes:
       - ${OBMP_DATA_ROOT}/postgres/data:/var/lib/postgresql/data  # change this to 80GB SSD slice/partition
       - ${OBMP_DATA_ROOT}/postgres/ts:/var/lib/postgresql/ts      # Chnage this to 500GB SSD slice/partition
@@ -78,13 +79,13 @@ services:
   collector:
     restart: unless-stopped
     container_name: obmp-collector
-    image: openbmp/collector:2.0.3
+    image: openbmp/collector:2.1.0
     sysctls:
       - net.ipv4.tcp_keepalive_intvl=30
       - net.ipv4.tcp_keepalive_probes=5
       - net.ipv4.tcp_keepalive_time=180
     ports:
-      - 5000:5000
+      - "5000:5000"
     volumes:
       - ${OBMP_DATA_ROOT}/config:/config
     environment:
@@ -93,13 +94,13 @@ services:
   psql-app:
     restart: unless-stopped
     container_name: obmp-psql-app
-    image: openbmp/psql-app:2.0.3
+    image: openbmp/psql-app:2.1.0
     sysctls:
       - net.ipv4.tcp_keepalive_intvl=30
       - net.ipv4.tcp_keepalive_probes=5
       - net.ipv4.tcp_keepalive_time=180
     ports:
-      - 9005:9005
+      - "9005:9005"
 
     volumes:
       - ${OBMP_DATA_ROOT}/config:/config
@@ -132,3 +133,22 @@ services:
       - POSTGRES_DROP_stats_ip_origins='4 weeks'
       - POSTGRES_DROP_stats_peer_rib='4 weeks'
       - POSTGRES_DROP_stats_peer_update_counts='4 weeks'
+
+  whois:
+    restart: unless-stopped
+    container_name: obmp-whois
+    image: openbmp/whois:2.1.0
+    sysctls:
+      - net.ipv4.tcp_keepalive_intvl=30
+      - net.ipv4.tcp_keepalive_probes=5
+      - net.ipv4.tcp_keepalive_time=180
+    ports:
+      - "4300:43"
+#    volumes:
+#      - ${OBMP_DATA_ROOT}/config:/config
+    environment:
+      - POSTGRES_PASSWORD=openbmp
+      - POSTGRES_USER=openbmp
+      - POSTGRES_DB=openbmp
+      - POSTGRES_HOST=obmp-psql
+      - POSTGRES_PORT=5432

postgres/Dockerfile

@@ -1,16 +1,16 @@
 # Postgres Container
 #
-#  Copyright (c) 2021-2022 Cisco Systems, Inc. and Tim Evens.  All rights reserved.
+#  Copyright (c) 2021-2022 Cisco Systems, Inc. and others.  All rights reserved.
 #
 #  Build:
-#       docker build -t openbmp/postgres:2.0.3 .
+#       docker build --platform linux/amd64 -t openbmp/postgres:2.1.0 .
 #
 #  Run:
 #       docker run --rm -it -p 5432:5432 \
 #              -e POSTGRES_PASSWORD=openbmp \
 #              -e POSTGRES_USER=openbmp \
 #              -e POSTGRES_DB=openbmp \
-#              openbmp/postgres:2.0.2
+#              openbmp/postgres:2.1.0
 
 FROM timescale/timescaledb:2.5.1-pg14
 
@@ -36,9 +36,7 @@ RUN apk update \
     && sed -i -e "s/^\#*listen_addresses.*=.*/listen_addresses = '*'/" /usr/local/share/postgresql/postgresql.conf.sample \
     && sed -i -e "s/^\#*ssl[ ]*=.*/ssl = on/" /usr/local/share/postgresql/postgresql.conf.sample \
     && sed -i -e "s/^\#*ssl_cert_file.*=.*/ssl_cert_file = \'\/psql_server.crt\'/" /usr/local/share/postgresql/postgresql.conf.sample \
-    && sed -i -e "s/^\#*ssl_key_file.*=.*/ssl_key_file = \'\/psql_server.key\'/" /usr/local/share/postgresql/postgresql.conf.sample \
+    && sed -i -e "s/^\#*ssl_key_file.*=.*/ssl_key_file = \'\/psql_server.key\'/" /usr/local/share/postgresql/postgresql.conf.sample
-    && sed -i -e "s/^\#*shared_buffers.*=.*/shared_buffers = ${MEM:-1}GB/" /usr/local/share/postgresql/postgresql.conf.sample \
-    && sed -i -e "s/^\#*work_mem.*=.*/work_mem = $(( (${MEM:-1} * 1024) * 5 / 100))MB/" /usr/local/share/postgresql/postgresql.conf.sample
 
 # Init timesries location
 RUN echo 'mkdir -p /var/lib/postgresql/ts/data' > /docker-entrypoint-initdb.d/0_obmp_init.sh \

psql-app/Dockerfile

@@ -1,6 +1,7 @@
-# Postgres Backend: openbmp/psql-consumer
+# syntax=docker/dockerfile:1.3
+# Postgres Backend: openbmp/psql-app
 #
-# Copyright (c) 2021-2022 Cisco Systems, Inc. and Tim Evens.  All rights reserved.
+# Copyright (c) 2021-2022 Cisco Systems, Inc. and others.  All rights reserved.
 #
 # Author: Tim Evens <tim@openbmp.org>
 #
@@ -11,10 +12,13 @@
 #   Change directories to obmp-docker/psql-app and run the below from that
 #   directory.
 #
+# Use buildkit - either via export DOCKER_BUILDKIT=1 or by using buildx cli
+#
 #   Example docker build:
-#     tar -cL -C ../../ ./obmp-psql ./obmp-docker/psql-app ./obmp-java-api-message \
+#     tar -c -C ../../ ./obmp-psql ./obmp-docker/psql-app ./obmp-java-api-message \
-#           | docker build --build-arg VERSION=2.0.3 \
+#           | docker buildx build --platform linux/amd64 --progress=plain \
-#                 -f obmp-docker/psql-app/Dockerfile -t openbmp/psql-app:2.0.3 -
+#                 --build-arg VERSION=2.1.0 \
+#                 -f obmp-docker/psql-app/Dockerfile -t openbmp/psql-app:2.1.0 -
 
 # -----------------------------------------------
 # stage: Build
@@ -45,12 +49,13 @@ FROM openjdk:17-slim
 # Copy files from previous stages
 COPY --from=build /ws/target/obmp-psql-consumer-0.1.0-SNAPSHOT.jar /usr/local/openbmp/obmp-psql-consumer.jar
 COPY --from=build /ws/database/  /usr/local/openbmp/database
-COPY --from=build /ws/cron_scripts/gen-whois/*.py /usr/local/openbmp/
+COPY  --chmod=755 --from=build /ws/cron_scripts/gen-whois/*.py /usr/local/openbmp/
-COPY --from=build /ws/cron_scripts/rpki/*.py /usr/local/openbmp/
+COPY  --chmod=755 --from=build /ws/cron_scripts/rpki/*.py /usr/local/openbmp/
-COPY --from=build /ws/scripts/dbip-to-psql.py /usr/local/openbmp/
+COPY  --chmod=755 --from=build /ws/scripts/dbip-to-psql.py /usr/local/openbmp/
 
 # Add files
-ADD obmp-docker/psql-app/scripts/run /usr/sbin/
+ADD  --chmod=755 obmp-docker/psql-app/scripts/run /usr/sbin/
+ADD  --chmod=755 obmp-docker/psql-app/scripts/upgrade_l3vpn.sh /usr/local/openbmp/upgrade_l3vpn.sh
 
 #----------------------------------
 # Define persistent data volumes

psql-app/scripts/run

@@ -146,7 +146,9 @@ MAILTO=""
 6 */2 * * *	root  . /usr/local/openbmp/pg_profile && /usr/local/openbmp/gen_whois_asn.py -u $PGUSER -p $PGPASSWORD $PGHOST >> /var/log/asn_load.log
 
 # Update aggregation table stats
-*/3 * * * *  root   . /usr/local/openbmp/pg_profile && psql -c "select update_chg_stats($POSTGRES_REPORT_WINDOW)"
+*/5 * * * *  root   . /usr/local/openbmp/pg_profile && psql -c "select update_chg_stats('5 minute')"
+*/5 * * * *  root   . /usr/local/openbmp/pg_profile && psql -c "select update_l3vpn_chg_stats('5 minute')"
+
 
 # Update peer rib counts
 */15 * * * *	root   . /usr/local/openbmp/pg_profile && psql -c "select update_peer_rib_counts()"
@@ -165,6 +167,19 @@ SETVAR
 
 }
 
+# -----------------------------------------------
+# Upgrade SQL
+# -----------------------------------------------
+upgrade() {
+    if [[ ! -f /config/psql-app-upgraded.2.1.0 ]]; then
+      echo "===> Upgrading to 2.1.0"
+      /usr/local/openbmp/upgrade_l3vpn.sh
+      touch /config/psql-app-upgraded.2.1.0
+      echo "===> Done with upgrade"
+    fi
+}
+
+
 # -----------------------------------------------
 # run_consumer
 # -----------------------------------------------
@@ -244,6 +259,8 @@ rm -f /var/run/rsyslogd.pid
 service cron start
 service rsyslog start
 
+upgrade
+
 run_consumer
 
 echo "===> Now running!!!"
@@ -251,3 +268,5 @@ echo "===> Now running!!!"
 while [ 1 ]; do
     sleep 1800
 done
+
+}

psql-app/scripts/upgrade_l3vpn.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Upgrade script for L3VPN
+#
+#  Copyright (c) 2022 Cisco Systems, Inc. and Tim Evens.  All rights reserved.
+#
+# Author: Tim Evens <tim@evensweb.com>
+#
+
+. /usr/local/openbmp/pg_profile && psql -c "select * from l3vpn_rib limit 1" > /dev/null 2>&1
+
+if [[ $? -ne 0 ]]; then
+  echo "==> Upgrading L3VPN SQL ======================================= "
+  psql < /usr/local/openbmp/database/10_l3vpn.sql
+  echo "==> Done upgrading L3VPN SQL ================================== "
+fi

whois/Dockerfile

@@ -0,0 +1,58 @@
+# syntax=docker/dockerfile:1.3
+# Postgres Backend: openbmp/whois
+#
+# Copyright (c) 2022 Cisco Systems, Inc. and others.  All rights reserved.
+#
+# BUILD:
+#     tar -c -C ../../ ./obmp-whois ./obmp-docker/whois  \
+#           | docker buildx build --platform linux/amd64 --progress=plain \
+#                 -f obmp-docker/whois/Dockerfile -t openbmp/whois:2.1.0 -
+
+# -----------------------------------------------
+# stage: Build
+# -----------------------------------------------
+FROM golang:alpine AS build
+
+WORKDIR /ws
+
+COPY obmp-whois/ /ws
+
+ENV GO111MODULE=on
+ENV GCGO_ENABLED=0
+ENV GOARCH=amd64
+ENV GOOS=linux
+
+RUN apk add --no-cache bash git
+RUN cd /ws && go mod tidy
+
+# Build the app
+RUN cd /ws && go build -o obmp-whoisd .
+
+
+# -----------------------------------------------
+# stage: Final container
+# -----------------------------------------------
+FROM alpine
+
+COPY --from=build  /ws/obmp-whoisd /usr/local/bin/
+
+WORKDIR /tmp
+
+# Environment varialbes that should be set when running container
+#   Defaults are given below
+ENV DB_MAX_THREADS=10
+ENV WHOIS_PORT=43
+
+ENV PGHOST=obmp-psql
+ENV PGDATABASE=openbmp
+ENV PGUSER=openbmp
+ENV PGPASSWORD=openbmp
+ENV PGPORT=5432
+
+EXPOSE 43
+
+# -- general depends
+RUN apk add --no-cache bash
+
+# -- set entry point to bash and act as login (load profile)
+ENTRYPOINT /usr/local/bin/obmp-whoisd --logfile /dev/stdout -p $WHOIS_PORT -t $DB_MAX_THREADS