diff --git a/collector/Dockerfile b/collector/Dockerfile index 47a607d..dcfd402 100644 --- a/collector/Dockerfile +++ b/collector/Dockerfile @@ -19,8 +19,8 @@ # Example docker build: # tar -c -C ../../ ./obmp-collector ./obmp-docker/collector \ # | docker buildx build --progress=plain \ -# --build-arg VERSION=2.1.1 \ -# -f obmp-docker/collector/Dockerfile -t openbmp/collector:2.1.1 - +# --build-arg VERSION=2.2.0 \ +# -f obmp-docker/collector/Dockerfile -t openbmp/collector:2.2.0 - # # ----------------------------------------------- diff --git a/docker-compose.yml b/docker-compose.yml index 69c522a..2662be0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: zookeeper: restart: unless-stopped container_name: obmp-zookeeper - image: confluentinc/cp-zookeeper:7.0.1 + image: confluentinc/cp-zookeeper:7.1.1 volumes: - ${OBMP_DATA_ROOT}/zk-data:/var/lib/zookeeper/data - ${OBMP_DATA_ROOT}/zk-log:/var/lib/zookeeper/log @@ -16,7 +16,7 @@ services: kafka: restart: unless-stopped container_name: obmp-kafka - image: confluentinc/cp-kafka:7.0.1 + image: confluentinc/cp-kafka:7.1.1 # Change the mount point to where you want to store Kafka data. # Normally 80GB or more @@ -73,7 +73,7 @@ services: grafana: restart: unless-stopped container_name: obmp-grafana - image: grafana/grafana:8.5.2 + image: grafana/grafana:8.5.4 ports: - "3000:3000" volumes: @@ -107,7 +107,7 @@ services: psql: restart: unless-stopped container_name: obmp-psql - image: openbmp/postgres:2.1.1 + image: openbmp/postgres:2.2.0 shm_size: 1536m sysctls: - net.ipv4.tcp_keepalive_intvl=30 @@ -129,7 +129,7 @@ services: collector: restart: unless-stopped container_name: obmp-collector - image: openbmp/collector:2.1.1 + image: openbmp/collector:2.2.0 sysctls: - net.ipv4.tcp_keepalive_intvl=30 - net.ipv4.tcp_keepalive_probes=5 @@ -144,7 +144,7 @@ services: psql-app: restart: unless-stopped container_name: obmp-psql-app - image: openbmp/psql-app:2.1.1 + image: openbmp/psql-app:2.2.0 sysctls: - net.ipv4.tcp_keepalive_intvl=30 - net.ipv4.tcp_keepalive_probes=5 @@ -187,7 +187,7 @@ services: whois: restart: unless-stopped container_name: obmp-whois - image: openbmp/whois:2.1.0 + image: openbmp/whois:2.2.0 sysctls: - net.ipv4.tcp_keepalive_intvl=30 - net.ipv4.tcp_keepalive_probes=5 diff --git a/postgres/Dockerfile b/postgres/Dockerfile index 2c38e9c..14c91e8 100644 --- a/postgres/Dockerfile +++ b/postgres/Dockerfile @@ -3,75 +3,47 @@ # Copyright (c) 2021-2022 Cisco Systems, Inc. and others. All rights reserved. # # Build: -# DOCKER_BUILDKIT=1 docker build --platform linux/amd64 -t openbmp/postgres:2.1.1 . +# DOCKER_BUILDKIT=1 docker build --platform linux/amd64 \ +# --build-arg VERSION=2.2.0 \ +# --squash \ +# -t openbmp/postgres:2.2.0 . # # Run: # docker run --rm -it -p 5432:5432 \ # -e POSTGRES_PASSWORD=openbmp \ # -e POSTGRES_USER=openbmp \ # -e POSTGRES_DB=openbmp \ -# openbmp/postgres:2.1.0 +# openbmp/postgres:2.2.0 # ----------------------------------------------- # stage: Build # ----------------------------------------------- -FROM timescale/timescaledb:2.6.0-pg14 AS build -ENV PG_CRON_VERSION 1.3.0 - -WORKDIR /ws - -RUN apk update \ - && apk add --no-cache --virtual .build-deps build-base ca-certificates clang-dev llvm12 openssl - -## Install pg_cron -RUN wget -O pg_cron.tgz https://github.com/citusdata/pg_cron/archive/v$PG_CRON_VERSION.tar.gz \ - && tar xvzf pg_cron.tgz && cd pg_cron-$PG_CRON_VERSION/ \ - && sed -i.bak -e 's/-Werror//g' Makefile \ - && sed -i.bak -e 's/-Wno-implicit-fallthrough//g' Makefile \ - && make # ----------------------------------------------- # stage: Final container # ----------------------------------------------- +#FROM timescale/timescaledb-ha:pg14.3-ts2.7-latest +FROM timescale/timescaledb-ha:pg14-ts2.7-oss-latest +ARG VERSION=0.0.0 -FROM timescale/timescaledb:2.6.0-pg14 - -ENV PG_CRON_VERSION 1.3.0 - -# Current/working dir -VOLUME ["/ws"] -WORKDIR /ws +ENV PGDATA=/var/lib/postgresql/data +ENV PGDATA_TS=/var/lib/postgresql/ts/data # Expected data locations for base tables and timeseries # VOLUME ["/var/lib/postgresql/data"] VOLUME ["/var/lib/postgresql/ts"] +ADD --chmod=755 scripts/004_obmp_psql_cfg.sh /docker-entrypoint-initdb.d/004_obmp_psql_cfg.sh +ADD --chmod=755 scripts/005_obmp_init.sh /docker-entrypoint-initdb.d/005_obmp_init.sh -RUN apk update \ - && apk add openssl\ - && openssl req -x509 -newkey rsa:4096 -nodes -subj "/C=US/ST=CA/L=Seattle/O=OpenBMP/CN=localhost" \ - -keyout /psql_server.key -out /psql_server.crt -days 2048 \ - && chown postgres /psql_server.* \ - && mkdir -p /var/lib/postgresql/ts \ - && chown postgres /var/lib/postgresql/ts \ - && egrep -q -e '^hostssl( |\t)+all' /usr/local/share/postgresql/pg_hba.conf.sample || \ - echo 'hostssl all all 0.0.0.0/0 md5' >> /usr/local/share/postgresql/pg_hba.conf.sample \ - && sed -i -e "s/^\#*listen_addresses.*=.*/listen_addresses = '*'/" /usr/local/share/postgresql/postgresql.conf.sample \ - && sed -i -e "s/^\#*max_wal_size.*=.*/max_wal_size = 10GB/" /usr/local/share/postgresql/postgresql.conf.sample \ - && sed -i -e "s/^\#*ssl[ ]*=.*/ssl = on/" /usr/local/share/postgresql/postgresql.conf.sample \ - && sed -i -e "s/^\#*ssl_cert_file.*=.*/ssl_cert_file = \'\/psql_server.crt\'/" /usr/local/share/postgresql/postgresql.conf.sample \ - && sed -i -e "s/^\#*ssl_key_file.*=.*/ssl_key_file = \'\/psql_server.key\'/" /usr/local/share/postgresql/postgresql.conf.sample \ - && sed -i -e "s/^shared_preload_libraries.*/shared_preload_libraries = 'timescaledb,pg_cron'/g" /usr/local/share/postgresql/postgresql.conf.sample +USER root -COPY --chmod=755 --from=build /ws/pg_cron-$PG_CRON_VERSION/pg_cron.so /usr/local/lib/postgresql/pg_cron.so -COPY --chmod=644 --from=build /ws/pg_cron-$PG_CRON_VERSION/pg_cron.control /usr/local/share/postgresql/extension/pg_cron.control -COPY --chmod=644 --from=build /ws/pg_cron-$PG_CRON_VERSION/pg_cron--1.0--1.1.sql /usr/local/share/postgresql/extension/pg_cron--1.0--1.1.sql -COPY --chmod=644 --from=build /ws/pg_cron-$PG_CRON_VERSION/pg_cron--1.1--1.2.sql /usr/local/share/postgresql/extension/pg_cron--1.1--1.2.sql -COPY --chmod=644 --from=build /ws/pg_cron-$PG_CRON_VERSION/pg_cron--1.2--1.3.sql /usr/local/share/postgresql/extension/pg_cron--1.2--1.3.sql -COPY --chmod=644 --from=build /ws/pg_cron-$PG_CRON_VERSION/pg_cron--1.0.sql /usr/local/share/postgresql/extension/pg_cron--1.0.sql +RUN rm -rf /usr/lib/postgresql/12 /usr/lib/postgresql/13 \ + && mkdir -p /var/lib/postgresql/data /var/lib/postgresql/ts/data \ + && chown -R postgres /var/lib/postgresql/data /var/lib/postgresql/ts/data -ADD --chmod=755 scripts/0_obmp_init.sh /docker-entrypoint-initdb.d/0_obmp_init.sh -## providing db name for cron metadata tables -RUN echo "cron.database_name = 'openbmp'" >> /usr/local/share/postgresql/postgresql.conf.sample +USER postgres + +RUN touch /var/lib/postgresql/data/version-${VERSION} diff --git a/postgres/scripts/004_obmp_psql_cfg.sh b/postgres/scripts/004_obmp_psql_cfg.sh new file mode 100755 index 0000000..eee5172 --- /dev/null +++ b/postgres/scripts/004_obmp_psql_cfg.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# OpenBMP Postgres configuration +# +# Copyright (c) 2022 Cisco Systems, Inc. and Tim Evens. All rights reserved. +# + + +# Create SSL cert +openssl req -x509 -newkey rsa:4096 -nodes -subj "/C=US/ST=CA/L=Seattle/O=OpenBMP/CN=localhost" \ + -keyout $PGDATA/psql_server.key -out $PGDATA/psql_server.crt -days 2048 \ + +# Init timeseries location +mkdir -p $PGDATA_TS +chmod 0700 $PGDATA_TS +psql -U $POSTGRES_USER -c "CREATE TABLESPACE timeseries LOCATION '$PGDATA_TS';" $POSTGRES_DB + +# Update postgres conf +sed -i -e "s/^\#*listen_addresses.*=.*/listen_addresses = '*'/" $PGDATA/postgresql.conf +sed -i -e "s/^\#*ssl[ ]*=.*/ssl = on/" $PGDATA/postgresql.conf +sed -i -e "s/^\#*ssl_cert_file.*=.*/ssl_cert_file = '${PGDATA//\//\\\/}\/psql_server.crt'/" $PGDATA/postgresql.conf +sed -i -e "s/^\#*ssl_key_file.*=.*/ssl_key_file = '${PGDATA//\//\\\/}\/psql_server.key'/" $PGDATA/postgresql.conf + +sed -i -e "s/^shared_preload_libraries.*/shared_preload_libraries = 'timescaledb,pg_cron'/g" $PGDATA/postgresql.conf + +echo "cron.database_name = 'openbmp'" >> $PGDATA/postgresql.conf + +egrep -q -e '^hostssl( |\t)+all' $PGDATA/pg_hba.conf +if [[ $? ]]; then + echo 'hostssl all all 0.0.0.0/0 md5' >> $PGDATA/pg_hba.conf +fi + + +pg_ctl -D "$PGDATA" -m fast -w restart \ No newline at end of file diff --git a/postgres/scripts/005_obmp_init.sh b/postgres/scripts/005_obmp_init.sh new file mode 100755 index 0000000..30e7d29 --- /dev/null +++ b/postgres/scripts/005_obmp_init.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# postgres: Init script +# +# Copyright (c) 2021-2022 Cisco Systems, Inc. and Tim Evens. All rights reserved. +# + +# >> NOTE, before adding extensions, required preload/config should be done first in 004_obmp_psql_cfg.sh + +# Add extensions +psql -U $POSTGRES_USER -c "CREATE EXTENSION IF NOT EXISTS postgis CASCADE;" $POSTGRES_DB +psql -U $POSTGRES_USER -c "CREATE EXTENSION IF NOT EXISTS pgrouting CASCADE;" $POSTGRES_DB + +# Add cron extension and config +psql -U $POSTGRES_USER -c "CREATE EXTENSION IF NOT EXISTS pg_cron;" $POSTGRES_DB +psql -U $POSTGRES_USER -c "GRANT USAGE ON SCHEMA cron TO $POSTGRES_USER;" $POSTGRES_DB + diff --git a/postgres/scripts/0_obmp_init.sh b/postgres/scripts/0_obmp_init.sh deleted file mode 100755 index e9669c3..0000000 --- a/postgres/scripts/0_obmp_init.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# postgres: Init script -# -# Copyright (c) 2021 Cisco Systems, Inc. and Tim Evens. All rights reserved. -# - - -# Init timesries location -mkdir -p /var/lib/postgresql/ts/data -chmod 0700 /var/lib/postgresql/ts/data -psql -U $POSTGRES_USER -c "CREATE TABLESPACE timeseries LOCATION '/var/lib/postgresql/ts/data';" $POSTGRES_DB - -# Config pg cron to database schema -psql -U $POSTGRES_USER -c "CREATE EXTENSION pg_cron;" $POSTGRES_DB -psql -U $POSTGRES_USER -c "GRANT USAGE ON SCHEMA cron TO $POSTGRES_USER;" $POSTGRES_DB diff --git a/psql-app/Dockerfile b/psql-app/Dockerfile index 140e739..e49d2b2 100644 --- a/psql-app/Dockerfile +++ b/psql-app/Dockerfile @@ -17,8 +17,8 @@ # Example docker build: # tar -c -C ../../ ./obmp-psql ./obmp-docker/psql-app ./obmp-java-api-message \ # | docker buildx build --platform linux/amd64 --progress=plain \ -# --build-arg VERSION=2.1.1 \ -# -f obmp-docker/psql-app/Dockerfile -t openbmp/psql-app:2.1.1 - +# --build-arg VERSION=2.2.0 \ +# -f obmp-docker/psql-app/Dockerfile -t openbmp/psql-app:2.2.0 - # ----------------------------------------------- # stage: Build @@ -52,7 +52,7 @@ COPY --from=build /ws/database/ /usr/local/openbmp/database COPY --chmod=755 --from=build /ws/cron_scripts/gen-whois/*.py /usr/local/openbmp/ COPY --chmod=755 --from=build /ws/cron_scripts/peeringdb/*.py /usr/local/openbmp/ COPY --chmod=755 --from=build /ws/cron_scripts/rpki/*.py /usr/local/openbmp/ -COPY --chmod=755 --from=build /ws/scripts/dbip-to-psql.py /usr/local/openbmp/ +COPY --chmod=755 --from=build /ws/scripts/geo-csv-to-psql.py /usr/local/openbmp/ # Add files ADD --chmod=755 obmp-docker/psql-app/scripts/run /usr/sbin/ diff --git a/psql-app/scripts/run b/psql-app/scripts/run index c33223c..88868a6 100755 --- a/psql-app/scripts/run +++ b/psql-app/scripts/run @@ -173,12 +173,21 @@ SETVAR # Upgrade SQL # ----------------------------------------------- upgrade() { + if [[ ! -f /config/psql-app-upgraded.2.1.0 ]]; then echo "===> Upgrading to 2.1.0" /tmp/upgrade/upgrade_2.1.0.sh touch /config/psql-app-upgraded.2.1.0 echo "===> Done with upgrade" fi + + if [[ ! -f /config/psql-app-upgraded.2.2.0 ]]; then + echo "===> Upgrading to 2.2.0" + /tmp/upgrade/upgrade_2.2.0.sh + touch /config/psql-app-upgraded.2.2.0 + echo "===> Done with upgrade" + fi + } diff --git a/whois/Dockerfile b/whois/Dockerfile index 74db2a1..04c785d 100644 --- a/whois/Dockerfile +++ b/whois/Dockerfile @@ -6,7 +6,7 @@ # BUILD: # tar -c -C ../../ ./obmp-whois ./obmp-docker/whois \ # | docker buildx build --platform linux/amd64 --progress=plain \ -# -f obmp-docker/whois/Dockerfile -t openbmp/whois:2.1.0 - +# -f obmp-docker/whois/Dockerfile -t openbmp/whois:2.2.0 - # ----------------------------------------------- # stage: Build