version 2.2.0 updates
This commit is contained in:
Tim Evens
parent
e7fad858d9
commit
84bec5293b
@ -19,8 +19,8 @@
|
||||
# Example docker build:
|
||||
# tar -c -C ../../ ./obmp-collector ./obmp-docker/collector \
|
||||
# | docker buildx build --progress=plain \
|
||||
# --build-arg VERSION=2.1.1 \
|
||||
# -f obmp-docker/collector/Dockerfile -t openbmp/collector:2.1.1 -
|
||||
# --build-arg VERSION=2.2.0 \
|
||||
# -f obmp-docker/collector/Dockerfile -t openbmp/collector:2.2.0 -
|
||||
#
|
||||
|
||||
# -----------------------------------------------
|
||||
|
||||
@ -5,7 +5,7 @@ services:
|
||||
zookeeper:
|
||||
restart: unless-stopped
|
||||
container_name: obmp-zookeeper
|
||||
image: confluentinc/cp-zookeeper:7.0.1
|
||||
image: confluentinc/cp-zookeeper:7.1.1
|
||||
volumes:
|
||||
- ${OBMP_DATA_ROOT}/zk-data:/var/lib/zookeeper/data
|
||||
- ${OBMP_DATA_ROOT}/zk-log:/var/lib/zookeeper/log
|
||||
@ -16,7 +16,7 @@ services:
|
||||
kafka:
|
||||
restart: unless-stopped
|
||||
container_name: obmp-kafka
|
||||
image: confluentinc/cp-kafka:7.0.1
|
||||
image: confluentinc/cp-kafka:7.1.1
|
||||
|
||||
# Change the mount point to where you want to store Kafka data.
|
||||
# Normally 80GB or more
|
||||
@ -73,7 +73,7 @@ services:
|
||||
grafana:
|
||||
restart: unless-stopped
|
||||
container_name: obmp-grafana
|
||||
image: grafana/grafana:8.5.2
|
||||
image: grafana/grafana:8.5.4
|
||||
ports:
|
||||
- "3000:3000"
|
||||
volumes:
|
||||
@ -107,7 +107,7 @@ services:
|
||||
psql:
|
||||
restart: unless-stopped
|
||||
container_name: obmp-psql
|
||||
image: openbmp/postgres:2.1.1
|
||||
image: openbmp/postgres:2.2.0
|
||||
shm_size: 1536m
|
||||
sysctls:
|
||||
- net.ipv4.tcp_keepalive_intvl=30
|
||||
@ -129,7 +129,7 @@ services:
|
||||
collector:
|
||||
restart: unless-stopped
|
||||
container_name: obmp-collector
|
||||
image: openbmp/collector:2.1.1
|
||||
image: openbmp/collector:2.2.0
|
||||
sysctls:
|
||||
- net.ipv4.tcp_keepalive_intvl=30
|
||||
- net.ipv4.tcp_keepalive_probes=5
|
||||
@ -144,7 +144,7 @@ services:
|
||||
psql-app:
|
||||
restart: unless-stopped
|
||||
container_name: obmp-psql-app
|
||||
image: openbmp/psql-app:2.1.1
|
||||
image: openbmp/psql-app:2.2.0
|
||||
sysctls:
|
||||
- net.ipv4.tcp_keepalive_intvl=30
|
||||
- net.ipv4.tcp_keepalive_probes=5
|
||||
@ -187,7 +187,7 @@ services:
|
||||
whois:
|
||||
restart: unless-stopped
|
||||
container_name: obmp-whois
|
||||
image: openbmp/whois:2.1.0
|
||||
image: openbmp/whois:2.2.0
|
||||
sysctls:
|
||||
- net.ipv4.tcp_keepalive_intvl=30
|
||||
- net.ipv4.tcp_keepalive_probes=5
|
||||
|
||||
@ -3,75 +3,47 @@
|
||||
# Copyright (c) 2021-2022 Cisco Systems, Inc. and others. All rights reserved.
|
||||
#
|
||||
# Build:
|
||||
# DOCKER_BUILDKIT=1 docker build --platform linux/amd64 -t openbmp/postgres:2.1.1 .
|
||||
# DOCKER_BUILDKIT=1 docker build --platform linux/amd64 \
|
||||
# --build-arg VERSION=2.2.0 \
|
||||
# --squash \
|
||||
# -t openbmp/postgres:2.2.0 .
|
||||
#
|
||||
# Run:
|
||||
# docker run --rm -it -p 5432:5432 \
|
||||
# -e POSTGRES_PASSWORD=openbmp \
|
||||
# -e POSTGRES_USER=openbmp \
|
||||
# -e POSTGRES_DB=openbmp \
|
||||
# openbmp/postgres:2.1.0
|
||||
# openbmp/postgres:2.2.0
|
||||
|
||||
# -----------------------------------------------
|
||||
# stage: Build
|
||||
# -----------------------------------------------
|
||||
FROM timescale/timescaledb:2.6.0-pg14 AS build
|
||||
|
||||
ENV PG_CRON_VERSION 1.3.0
|
||||
|
||||
WORKDIR /ws
|
||||
|
||||
RUN apk update \
|
||||
&& apk add --no-cache --virtual .build-deps build-base ca-certificates clang-dev llvm12 openssl
|
||||
|
||||
## Install pg_cron
|
||||
RUN wget -O pg_cron.tgz https://github.com/citusdata/pg_cron/archive/v$PG_CRON_VERSION.tar.gz \
|
||||
&& tar xvzf pg_cron.tgz && cd pg_cron-$PG_CRON_VERSION/ \
|
||||
&& sed -i.bak -e 's/-Werror//g' Makefile \
|
||||
&& sed -i.bak -e 's/-Wno-implicit-fallthrough//g' Makefile \
|
||||
&& make
|
||||
|
||||
# -----------------------------------------------
|
||||
# stage: Final container
|
||||
# -----------------------------------------------
|
||||
#FROM timescale/timescaledb-ha:pg14.3-ts2.7-latest
|
||||
FROM timescale/timescaledb-ha:pg14-ts2.7-oss-latest
|
||||
ARG VERSION=0.0.0
|
||||
|
||||
FROM timescale/timescaledb:2.6.0-pg14
|
||||
|
||||
ENV PG_CRON_VERSION 1.3.0
|
||||
|
||||
# Current/working dir
|
||||
VOLUME ["/ws"]
|
||||
WORKDIR /ws
|
||||
ENV PGDATA=/var/lib/postgresql/data
|
||||
ENV PGDATA_TS=/var/lib/postgresql/ts/data
|
||||
|
||||
# Expected data locations for base tables and timeseries
|
||||
#
|
||||
VOLUME ["/var/lib/postgresql/data"]
|
||||
VOLUME ["/var/lib/postgresql/ts"]
|
||||
|
||||
ADD --chmod=755 scripts/004_obmp_psql_cfg.sh /docker-entrypoint-initdb.d/004_obmp_psql_cfg.sh
|
||||
ADD --chmod=755 scripts/005_obmp_init.sh /docker-entrypoint-initdb.d/005_obmp_init.sh
|
||||
|
||||
RUN apk update \
|
||||
&& apk add openssl\
|
||||
&& openssl req -x509 -newkey rsa:4096 -nodes -subj "/C=US/ST=CA/L=Seattle/O=OpenBMP/CN=localhost" \
|
||||
-keyout /psql_server.key -out /psql_server.crt -days 2048 \
|
||||
&& chown postgres /psql_server.* \
|
||||
&& mkdir -p /var/lib/postgresql/ts \
|
||||
&& chown postgres /var/lib/postgresql/ts \
|
||||
&& egrep -q -e '^hostssl( |\t)+all' /usr/local/share/postgresql/pg_hba.conf.sample || \
|
||||
echo 'hostssl all all 0.0.0.0/0 md5' >> /usr/local/share/postgresql/pg_hba.conf.sample \
|
||||
&& sed -i -e "s/^\#*listen_addresses.*=.*/listen_addresses = '*'/" /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& sed -i -e "s/^\#*max_wal_size.*=.*/max_wal_size = 10GB/" /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& sed -i -e "s/^\#*ssl[ ]*=.*/ssl = on/" /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& sed -i -e "s/^\#*ssl_cert_file.*=.*/ssl_cert_file = \'\/psql_server.crt\'/" /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& sed -i -e "s/^\#*ssl_key_file.*=.*/ssl_key_file = \'\/psql_server.key\'/" /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& sed -i -e "s/^shared_preload_libraries.*/shared_preload_libraries = 'timescaledb,pg_cron'/g" /usr/local/share/postgresql/postgresql.conf.sample
|
||||
USER root
|
||||
|
||||
COPY --chmod=755 --from=build /ws/pg_cron-$PG_CRON_VERSION/pg_cron.so /usr/local/lib/postgresql/pg_cron.so
|
||||
COPY --chmod=644 --from=build /ws/pg_cron-$PG_CRON_VERSION/pg_cron.control /usr/local/share/postgresql/extension/pg_cron.control
|
||||
COPY --chmod=644 --from=build /ws/pg_cron-$PG_CRON_VERSION/pg_cron--1.0--1.1.sql /usr/local/share/postgresql/extension/pg_cron--1.0--1.1.sql
|
||||
COPY --chmod=644 --from=build /ws/pg_cron-$PG_CRON_VERSION/pg_cron--1.1--1.2.sql /usr/local/share/postgresql/extension/pg_cron--1.1--1.2.sql
|
||||
COPY --chmod=644 --from=build /ws/pg_cron-$PG_CRON_VERSION/pg_cron--1.2--1.3.sql /usr/local/share/postgresql/extension/pg_cron--1.2--1.3.sql
|
||||
COPY --chmod=644 --from=build /ws/pg_cron-$PG_CRON_VERSION/pg_cron--1.0.sql /usr/local/share/postgresql/extension/pg_cron--1.0.sql
|
||||
RUN rm -rf /usr/lib/postgresql/12 /usr/lib/postgresql/13 \
|
||||
&& mkdir -p /var/lib/postgresql/data /var/lib/postgresql/ts/data \
|
||||
&& chown -R postgres /var/lib/postgresql/data /var/lib/postgresql/ts/data
|
||||
|
||||
ADD --chmod=755 scripts/0_obmp_init.sh /docker-entrypoint-initdb.d/0_obmp_init.sh
|
||||
## providing db name for cron metadata tables
|
||||
RUN echo "cron.database_name = 'openbmp'" >> /usr/local/share/postgresql/postgresql.conf.sample
|
||||
USER postgres
|
||||
|
||||
RUN touch /var/lib/postgresql/data/version-${VERSION}
|
||||
|
||||
34
postgres/scripts/004_obmp_psql_cfg.sh
Executable file
34
postgres/scripts/004_obmp_psql_cfg.sh
Executable file
@ -0,0 +1,34 @@
|
||||
#!/bin/bash
|
||||
|
||||
# OpenBMP Postgres configuration
|
||||
#
|
||||
# Copyright (c) 2022 Cisco Systems, Inc. and Tim Evens. All rights reserved.
|
||||
#
|
||||
|
||||
|
||||
# Create SSL cert
|
||||
openssl req -x509 -newkey rsa:4096 -nodes -subj "/C=US/ST=CA/L=Seattle/O=OpenBMP/CN=localhost" \
|
||||
-keyout $PGDATA/psql_server.key -out $PGDATA/psql_server.crt -days 2048 \
|
||||
|
||||
# Init timeseries location
|
||||
mkdir -p $PGDATA_TS
|
||||
chmod 0700 $PGDATA_TS
|
||||
psql -U $POSTGRES_USER -c "CREATE TABLESPACE timeseries LOCATION '$PGDATA_TS';" $POSTGRES_DB
|
||||
|
||||
# Update postgres conf
|
||||
sed -i -e "s/^\#*listen_addresses.*=.*/listen_addresses = '*'/" $PGDATA/postgresql.conf
|
||||
sed -i -e "s/^\#*ssl[ ]*=.*/ssl = on/" $PGDATA/postgresql.conf
|
||||
sed -i -e "s/^\#*ssl_cert_file.*=.*/ssl_cert_file = '${PGDATA//\//\\\/}\/psql_server.crt'/" $PGDATA/postgresql.conf
|
||||
sed -i -e "s/^\#*ssl_key_file.*=.*/ssl_key_file = '${PGDATA//\//\\\/}\/psql_server.key'/" $PGDATA/postgresql.conf
|
||||
|
||||
sed -i -e "s/^shared_preload_libraries.*/shared_preload_libraries = 'timescaledb,pg_cron'/g" $PGDATA/postgresql.conf
|
||||
|
||||
echo "cron.database_name = 'openbmp'" >> $PGDATA/postgresql.conf
|
||||
|
||||
egrep -q -e '^hostssl( |\t)+all' $PGDATA/pg_hba.conf
|
||||
if [[ $? ]]; then
|
||||
echo 'hostssl all all 0.0.0.0/0 md5' >> $PGDATA/pg_hba.conf
|
||||
fi
|
||||
|
||||
|
||||
pg_ctl -D "$PGDATA" -m fast -w restart
|
||||
17
postgres/scripts/005_obmp_init.sh
Executable file
17
postgres/scripts/005_obmp_init.sh
Executable file
@ -0,0 +1,17 @@
|
||||
#!/bin/bash
|
||||
|
||||
# postgres: Init script
|
||||
#
|
||||
# Copyright (c) 2021-2022 Cisco Systems, Inc. and Tim Evens. All rights reserved.
|
||||
#
|
||||
|
||||
# >> NOTE, before adding extensions, required preload/config should be done first in 004_obmp_psql_cfg.sh
|
||||
|
||||
# Add extensions
|
||||
psql -U $POSTGRES_USER -c "CREATE EXTENSION IF NOT EXISTS postgis CASCADE;" $POSTGRES_DB
|
||||
psql -U $POSTGRES_USER -c "CREATE EXTENSION IF NOT EXISTS pgrouting CASCADE;" $POSTGRES_DB
|
||||
|
||||
# Add cron extension and config
|
||||
psql -U $POSTGRES_USER -c "CREATE EXTENSION IF NOT EXISTS pg_cron;" $POSTGRES_DB
|
||||
psql -U $POSTGRES_USER -c "GRANT USAGE ON SCHEMA cron TO $POSTGRES_USER;" $POSTGRES_DB
|
||||
|
||||
@ -1,16 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# postgres: Init script
|
||||
#
|
||||
# Copyright (c) 2021 Cisco Systems, Inc. and Tim Evens. All rights reserved.
|
||||
#
|
||||
|
||||
|
||||
# Init timesries location
|
||||
mkdir -p /var/lib/postgresql/ts/data
|
||||
chmod 0700 /var/lib/postgresql/ts/data
|
||||
psql -U $POSTGRES_USER -c "CREATE TABLESPACE timeseries LOCATION '/var/lib/postgresql/ts/data';" $POSTGRES_DB
|
||||
|
||||
# Config pg cron to database schema
|
||||
psql -U $POSTGRES_USER -c "CREATE EXTENSION pg_cron;" $POSTGRES_DB
|
||||
psql -U $POSTGRES_USER -c "GRANT USAGE ON SCHEMA cron TO $POSTGRES_USER;" $POSTGRES_DB
|
||||
@ -17,8 +17,8 @@
|
||||
# Example docker build:
|
||||
# tar -c -C ../../ ./obmp-psql ./obmp-docker/psql-app ./obmp-java-api-message \
|
||||
# | docker buildx build --platform linux/amd64 --progress=plain \
|
||||
# --build-arg VERSION=2.1.1 \
|
||||
# -f obmp-docker/psql-app/Dockerfile -t openbmp/psql-app:2.1.1 -
|
||||
# --build-arg VERSION=2.2.0 \
|
||||
# -f obmp-docker/psql-app/Dockerfile -t openbmp/psql-app:2.2.0 -
|
||||
|
||||
# -----------------------------------------------
|
||||
# stage: Build
|
||||
@ -52,7 +52,7 @@ COPY --from=build /ws/database/ /usr/local/openbmp/database
|
||||
COPY --chmod=755 --from=build /ws/cron_scripts/gen-whois/*.py /usr/local/openbmp/
|
||||
COPY --chmod=755 --from=build /ws/cron_scripts/peeringdb/*.py /usr/local/openbmp/
|
||||
COPY --chmod=755 --from=build /ws/cron_scripts/rpki/*.py /usr/local/openbmp/
|
||||
COPY --chmod=755 --from=build /ws/scripts/dbip-to-psql.py /usr/local/openbmp/
|
||||
COPY --chmod=755 --from=build /ws/scripts/geo-csv-to-psql.py /usr/local/openbmp/
|
||||
|
||||
# Add files
|
||||
ADD --chmod=755 obmp-docker/psql-app/scripts/run /usr/sbin/
|
||||
|
||||
@ -173,12 +173,21 @@ SETVAR
|
||||
# Upgrade SQL
|
||||
# -----------------------------------------------
|
||||
upgrade() {
|
||||
|
||||
if [[ ! -f /config/psql-app-upgraded.2.1.0 ]]; then
|
||||
echo "===> Upgrading to 2.1.0"
|
||||
/tmp/upgrade/upgrade_2.1.0.sh
|
||||
touch /config/psql-app-upgraded.2.1.0
|
||||
echo "===> Done with upgrade"
|
||||
fi
|
||||
|
||||
if [[ ! -f /config/psql-app-upgraded.2.2.0 ]]; then
|
||||
echo "===> Upgrading to 2.2.0"
|
||||
/tmp/upgrade/upgrade_2.2.0.sh
|
||||
touch /config/psql-app-upgraded.2.2.0
|
||||
echo "===> Done with upgrade"
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
# BUILD:
|
||||
# tar -c -C ../../ ./obmp-whois ./obmp-docker/whois \
|
||||
# | docker buildx build --platform linux/amd64 --progress=plain \
|
||||
# -f obmp-docker/whois/Dockerfile -t openbmp/whois:2.1.0 -
|
||||
# -f obmp-docker/whois/Dockerfile -t openbmp/whois:2.2.0 -
|
||||
|
||||
# -----------------------------------------------
|
||||
# stage: Build
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user