version 2.2.0 updates

collector/Dockerfile

@@ -19,8 +19,8 @@
 #   Example docker build:
 #     tar -c -C ../../ ./obmp-collector ./obmp-docker/collector \
 #           | docker buildx build --progress=plain \
-#                 --build-arg VERSION=2.1.1 \
+#                 --build-arg VERSION=2.2.0 \
-#                 -f obmp-docker/collector/Dockerfile -t openbmp/collector:2.1.1 -
+#                 -f obmp-docker/collector/Dockerfile -t openbmp/collector:2.2.0 -
 #
 
 # -----------------------------------------------

docker-compose.yml

@@ -5,7 +5,7 @@ services:
   zookeeper:
     restart: unless-stopped
     container_name: obmp-zookeeper
-    image: confluentinc/cp-zookeeper:7.0.1
+    image: confluentinc/cp-zookeeper:7.1.1
     volumes:
       - ${OBMP_DATA_ROOT}/zk-data:/var/lib/zookeeper/data
       - ${OBMP_DATA_ROOT}/zk-log:/var/lib/zookeeper/log
@@ -16,7 +16,7 @@ services:
   kafka:
     restart: unless-stopped
     container_name: obmp-kafka
-    image: confluentinc/cp-kafka:7.0.1
+    image: confluentinc/cp-kafka:7.1.1
 
     # Change the mount point to where you want to store Kafka data.
     #   Normally 80GB or more
@@ -73,7 +73,7 @@ services:
   grafana:
     restart: unless-stopped
     container_name: obmp-grafana
-    image: grafana/grafana:8.5.2
+    image: grafana/grafana:8.5.4
     ports:
       - "3000:3000"
     volumes:
@@ -107,7 +107,7 @@ services:
   psql:
     restart: unless-stopped
     container_name: obmp-psql
-    image: openbmp/postgres:2.1.1
+    image: openbmp/postgres:2.2.0
     shm_size: 1536m
     sysctls:
       - net.ipv4.tcp_keepalive_intvl=30
@@ -129,7 +129,7 @@ services:
   collector:
     restart: unless-stopped
     container_name: obmp-collector
-    image: openbmp/collector:2.1.1
+    image: openbmp/collector:2.2.0
     sysctls:
       - net.ipv4.tcp_keepalive_intvl=30
       - net.ipv4.tcp_keepalive_probes=5
@@ -144,7 +144,7 @@ services:
   psql-app:
     restart: unless-stopped
     container_name: obmp-psql-app
-    image: openbmp/psql-app:2.1.1
+    image: openbmp/psql-app:2.2.0
     sysctls:
       - net.ipv4.tcp_keepalive_intvl=30
       - net.ipv4.tcp_keepalive_probes=5
@@ -187,7 +187,7 @@ services:
   whois:
     restart: unless-stopped
     container_name: obmp-whois
-    image: openbmp/whois:2.1.0
+    image: openbmp/whois:2.2.0
     sysctls:
       - net.ipv4.tcp_keepalive_intvl=30
       - net.ipv4.tcp_keepalive_probes=5

postgres/Dockerfile

@@ -3,75 +3,47 @@
 #  Copyright (c) 2021-2022 Cisco Systems, Inc. and others.  All rights reserved.
 #
 #  Build:
-#        DOCKER_BUILDKIT=1 docker build --platform linux/amd64 -t openbmp/postgres:2.1.1 .
+#        DOCKER_BUILDKIT=1 docker build --platform linux/amd64 \
+#            --build-arg VERSION=2.2.0 \
+#            --squash \
+#            -t openbmp/postgres:2.2.0 .
 #
 #  Run:
 #       docker run --rm -it -p 5432:5432 \
 #              -e POSTGRES_PASSWORD=openbmp \
 #              -e POSTGRES_USER=openbmp \
 #              -e POSTGRES_DB=openbmp \
-#              openbmp/postgres:2.1.0
+#              openbmp/postgres:2.2.0
 
 # -----------------------------------------------
 # stage: Build
 # -----------------------------------------------
-FROM timescale/timescaledb:2.6.0-pg14 AS build
 
-ENV PG_CRON_VERSION 1.3.0
-
-WORKDIR /ws
-
-RUN apk update \
-    && apk add --no-cache --virtual .build-deps build-base ca-certificates clang-dev llvm12 openssl
-
-## Install pg_cron
-RUN wget -O pg_cron.tgz https://github.com/citusdata/pg_cron/archive/v$PG_CRON_VERSION.tar.gz \
-    && tar xvzf pg_cron.tgz && cd pg_cron-$PG_CRON_VERSION/ \ 
-    && sed -i.bak -e 's/-Werror//g' Makefile \
-    && sed -i.bak -e 's/-Wno-implicit-fallthrough//g' Makefile \
-    && make
 
 # -----------------------------------------------
 # stage: Final container
 # -----------------------------------------------
+#FROM timescale/timescaledb-ha:pg14.3-ts2.7-latest
+FROM timescale/timescaledb-ha:pg14-ts2.7-oss-latest
+ARG VERSION=0.0.0
 
-FROM timescale/timescaledb:2.6.0-pg14
+ENV PGDATA=/var/lib/postgresql/data
-
+ENV PGDATA_TS=/var/lib/postgresql/ts/data
-ENV PG_CRON_VERSION 1.3.0
-
-# Current/working dir
-VOLUME ["/ws"]
-WORKDIR /ws
 
 # Expected data locations for base tables and timeseries
 #
 VOLUME ["/var/lib/postgresql/data"]
 VOLUME ["/var/lib/postgresql/ts"]
 
+ADD --chmod=755 scripts/004_obmp_psql_cfg.sh /docker-entrypoint-initdb.d/004_obmp_psql_cfg.sh
+ADD --chmod=755 scripts/005_obmp_init.sh /docker-entrypoint-initdb.d/005_obmp_init.sh
 
-RUN apk update \
+USER root
-    && apk add openssl\
-    && openssl req -x509 -newkey rsa:4096 -nodes -subj "/C=US/ST=CA/L=Seattle/O=OpenBMP/CN=localhost"  \
-        -keyout /psql_server.key -out /psql_server.crt -days 2048 \
-    && chown postgres /psql_server.* \
-    && mkdir -p /var/lib/postgresql/ts \
-    && chown postgres /var/lib/postgresql/ts \
-    && egrep -q -e '^hostssl( |\t)+all' /usr/local/share/postgresql/pg_hba.conf.sample || \
-            echo 'hostssl    all        all        0.0.0.0/0        md5' >> /usr/local/share/postgresql/pg_hba.conf.sample \
-    && sed -i -e "s/^\#*listen_addresses.*=.*/listen_addresses = '*'/" /usr/local/share/postgresql/postgresql.conf.sample \
-    && sed -i -e "s/^\#*max_wal_size.*=.*/max_wal_size = 10GB/" /usr/local/share/postgresql/postgresql.conf.sample \
-    && sed -i -e "s/^\#*ssl[ ]*=.*/ssl = on/" /usr/local/share/postgresql/postgresql.conf.sample \
-    && sed -i -e "s/^\#*ssl_cert_file.*=.*/ssl_cert_file = \'\/psql_server.crt\'/" /usr/local/share/postgresql/postgresql.conf.sample \
-    && sed -i -e "s/^\#*ssl_key_file.*=.*/ssl_key_file = \'\/psql_server.key\'/" /usr/local/share/postgresql/postgresql.conf.sample \
-    && sed -i -e "s/^shared_preload_libraries.*/shared_preload_libraries = 'timescaledb,pg_cron'/g" /usr/local/share/postgresql/postgresql.conf.sample
 
-COPY --chmod=755 --from=build /ws/pg_cron-$PG_CRON_VERSION/pg_cron.so /usr/local/lib/postgresql/pg_cron.so
+RUN  rm -rf /usr/lib/postgresql/12 /usr/lib/postgresql/13  \
-COPY --chmod=644 --from=build /ws/pg_cron-$PG_CRON_VERSION/pg_cron.control /usr/local/share/postgresql/extension/pg_cron.control
+     && mkdir -p /var/lib/postgresql/data /var/lib/postgresql/ts/data \
-COPY --chmod=644 --from=build /ws/pg_cron-$PG_CRON_VERSION/pg_cron--1.0--1.1.sql /usr/local/share/postgresql/extension/pg_cron--1.0--1.1.sql
+     && chown -R postgres /var/lib/postgresql/data /var/lib/postgresql/ts/data
-COPY --chmod=644 --from=build /ws/pg_cron-$PG_CRON_VERSION/pg_cron--1.1--1.2.sql /usr/local/share/postgresql/extension/pg_cron--1.1--1.2.sql
-COPY --chmod=644 --from=build /ws/pg_cron-$PG_CRON_VERSION/pg_cron--1.2--1.3.sql /usr/local/share/postgresql/extension/pg_cron--1.2--1.3.sql
-COPY --chmod=644 --from=build /ws/pg_cron-$PG_CRON_VERSION/pg_cron--1.0.sql /usr/local/share/postgresql/extension/pg_cron--1.0.sql
 
-ADD --chmod=755 scripts/0_obmp_init.sh /docker-entrypoint-initdb.d/0_obmp_init.sh
+USER postgres
-## providing db name for cron metadata tables
+
-RUN echo "cron.database_name = 'openbmp'" >> /usr/local/share/postgresql/postgresql.conf.sample
+RUN touch /var/lib/postgresql/data/version-${VERSION}

postgres/scripts/004_obmp_psql_cfg.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# OpenBMP Postgres configuration
+#
+#  Copyright (c) 2022 Cisco Systems, Inc. and Tim Evens.  All rights reserved.
+#
+
+
+# Create SSL cert
+openssl req -x509 -newkey rsa:4096 -nodes -subj "/C=US/ST=CA/L=Seattle/O=OpenBMP/CN=localhost"  \
+        -keyout $PGDATA/psql_server.key -out $PGDATA/psql_server.crt -days 2048 \
+
+# Init timeseries location
+mkdir -p $PGDATA_TS
+chmod 0700 $PGDATA_TS
+psql -U $POSTGRES_USER -c "CREATE TABLESPACE timeseries LOCATION '$PGDATA_TS';" $POSTGRES_DB
+
+# Update postgres conf
+sed -i -e "s/^\#*listen_addresses.*=.*/listen_addresses = '*'/" $PGDATA/postgresql.conf
+sed -i -e "s/^\#*ssl[ ]*=.*/ssl = on/" $PGDATA/postgresql.conf
+sed -i -e "s/^\#*ssl_cert_file.*=.*/ssl_cert_file =  '${PGDATA//\//\\\/}\/psql_server.crt'/" $PGDATA/postgresql.conf
+sed -i -e "s/^\#*ssl_key_file.*=.*/ssl_key_file =  '${PGDATA//\//\\\/}\/psql_server.key'/" $PGDATA/postgresql.conf
+
+sed -i -e "s/^shared_preload_libraries.*/shared_preload_libraries = 'timescaledb,pg_cron'/g" $PGDATA/postgresql.conf
+
+echo "cron.database_name = 'openbmp'" >> $PGDATA/postgresql.conf
+
+egrep -q -e '^hostssl( |\t)+all' $PGDATA/pg_hba.conf
+if [[ $? ]]; then
+    echo 'hostssl    all        all        0.0.0.0/0        md5' >> $PGDATA/pg_hba.conf
+fi
+
+
+pg_ctl -D "$PGDATA" -m fast -w restart

postgres/scripts/005_obmp_init.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# postgres: Init script
+#
+#  Copyright (c) 2021-2022 Cisco Systems, Inc. and Tim Evens.  All rights reserved.
+#
+
+# >> NOTE, before adding extensions, required preload/config should be done first in 004_obmp_psql_cfg.sh
+
+# Add extensions
+psql -U $POSTGRES_USER -c "CREATE EXTENSION IF NOT EXISTS postgis CASCADE;" $POSTGRES_DB
+psql -U $POSTGRES_USER -c "CREATE EXTENSION IF NOT EXISTS pgrouting CASCADE;" $POSTGRES_DB
+
+# Add cron extension and config
+psql -U $POSTGRES_USER -c "CREATE EXTENSION IF NOT EXISTS pg_cron;" $POSTGRES_DB
+psql -U $POSTGRES_USER -c "GRANT USAGE ON SCHEMA cron TO $POSTGRES_USER;" $POSTGRES_DB
+

postgres/scripts/0_obmp_init.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-# postgres: Init script
-#
-#  Copyright (c) 2021 Cisco Systems, Inc. and Tim Evens.  All rights reserved.
-#
-
-
-# Init timesries location
-mkdir -p /var/lib/postgresql/ts/data
-chmod 0700 /var/lib/postgresql/ts/data
-psql -U $POSTGRES_USER -c "CREATE TABLESPACE timeseries LOCATION '/var/lib/postgresql/ts/data';" $POSTGRES_DB
-
-# Config pg cron to database schema
-psql -U $POSTGRES_USER -c "CREATE EXTENSION pg_cron;" $POSTGRES_DB
-psql -U $POSTGRES_USER -c "GRANT USAGE ON SCHEMA cron TO $POSTGRES_USER;" $POSTGRES_DB

psql-app/Dockerfile

@@ -17,8 +17,8 @@
 #   Example docker build:
 #     tar -c -C ../../ ./obmp-psql ./obmp-docker/psql-app ./obmp-java-api-message \
 #           | docker buildx build --platform linux/amd64 --progress=plain \
-#                 --build-arg VERSION=2.1.1 \
+#                 --build-arg VERSION=2.2.0 \
-#                 -f obmp-docker/psql-app/Dockerfile -t openbmp/psql-app:2.1.1 -
+#                 -f obmp-docker/psql-app/Dockerfile -t openbmp/psql-app:2.2.0 -
 
 # -----------------------------------------------
 # stage: Build
@@ -52,7 +52,7 @@ COPY --from=build /ws/database/  /usr/local/openbmp/database
 COPY  --chmod=755 --from=build /ws/cron_scripts/gen-whois/*.py /usr/local/openbmp/
 COPY  --chmod=755 --from=build /ws/cron_scripts/peeringdb/*.py /usr/local/openbmp/
 COPY  --chmod=755 --from=build /ws/cron_scripts/rpki/*.py /usr/local/openbmp/
-COPY  --chmod=755 --from=build /ws/scripts/dbip-to-psql.py /usr/local/openbmp/
+COPY  --chmod=755 --from=build /ws/scripts/geo-csv-to-psql.py /usr/local/openbmp/
 
 # Add files
 ADD  --chmod=755 obmp-docker/psql-app/scripts/run /usr/sbin/

psql-app/scripts/run

@@ -173,12 +173,21 @@ SETVAR
 # Upgrade SQL
 # -----------------------------------------------
 upgrade() {
+
     if [[ ! -f /config/psql-app-upgraded.2.1.0 ]]; then
       echo "===> Upgrading to 2.1.0"
       /tmp/upgrade/upgrade_2.1.0.sh
       touch /config/psql-app-upgraded.2.1.0
       echo "===> Done with upgrade"
     fi
+
+    if [[ ! -f /config/psql-app-upgraded.2.2.0 ]]; then
+      echo "===> Upgrading to 2.2.0"
+      /tmp/upgrade/upgrade_2.2.0.sh
+      touch /config/psql-app-upgraded.2.2.0
+      echo "===> Done with upgrade"
+    fi
+
 }
 
 

whois/Dockerfile

@@ -6,7 +6,7 @@
 # BUILD:
 #     tar -c -C ../../ ./obmp-whois ./obmp-docker/whois  \
 #           | docker buildx build --platform linux/amd64 --progress=plain \
-#                 -f obmp-docker/whois/Dockerfile -t openbmp/whois:2.1.0 -
+#                 -f obmp-docker/whois/Dockerfile -t openbmp/whois:2.2.0 -
 
 # -----------------------------------------------
 # stage: Build