diff --git a/collector/Dockerfile b/collector/Dockerfile index 1b3c4e9..a0e8f14 100644 --- a/collector/Dockerfile +++ b/collector/Dockerfile @@ -1,6 +1,7 @@ +# syntax=docker/dockerfile:1.3 # Collector: openbmp/collector # -# Copyright (c) 2021-2022 Cisco Systems, Inc. and Tim Evens. All rights reserved. +# Copyright (c) 2021-2022 Cisco Systems, Inc. and others. All rights reserved. # # Author: Tim Evens # @@ -13,10 +14,13 @@ # Change directories to obmp-docker/collector and run the below from that # directory. # +# Use buildkit - either via export DOCKER_BUILDKIT=1 or by using buildx cli +# # Example docker build: -# tar -cL -C ../../ ./obmp-collector ./obmp-docker/collector \ -# | docker build --build-arg VERSION=2.0.3 \ -# -f obmp-docker/collector/Dockerfile -t openbmp/collector:2.0.3 - +# tar -c -C ../../ ./obmp-collector ./obmp-docker/collector \ +# | docker buildx build --progress=plain \ +# --build-arg VERSION=2.1.0 \ +# -f obmp-docker/collector/Dockerfile -t openbmp/collector:2.1.0 - # # ----------------------------------------------- @@ -27,7 +31,7 @@ FROM openbmp/dev-image:latest AS build COPY obmp-collector/ /ws WORKDIR /ws -RUN mkdir -p build && cd build \ +RUN rm -rf build && mkdir -p build && cd build \ && cmake -DCMAKE_INSTALL_PREFIX:PATH=/usr ../ \ && make \ && make install @@ -40,13 +44,13 @@ RUN mkdir -p build && cd build \ FROM debian:bullseye-slim # Add files. -ADD obmp-docker/collector/scripts/install /tmp/ -ADD obmp-docker/collector/scripts/run /usr/sbin/ +ADD --chmod=755 obmp-docker/collector/scripts/install /tmp/ +ADD --chmod=755 obmp-docker/collector/scripts/run /usr/sbin/ ARG VERSION=0 # Copy files from previous stages -COPY --from=build /usr/bin/openbmpd /usr/bin/ +COPY --chmod=755 --from=build /usr/bin/openbmpd /usr/bin/ COPY --from=build /usr/etc/openbmp/openbmpd.conf /usr/etc/openbmp/openbmpd.conf COPY --from=build /etc/init/openbmpd.conf /etc/init/openbmpd.conf COPY --from=build /etc/default/openbmpd.new /etc/default/openbmpd diff --git a/dev-image/Dockerfile b/dev-image/Dockerfile index 8076d04..0bcb94c 100644 --- a/dev-image/Dockerfile +++ b/dev-image/Dockerfile @@ -1,14 +1,12 @@ # Development build container: openbmp/dev-image # # Copyright (c) 2021-2022 Cisco Systems, Inc. and others. All rights reserved. -# Copyright (c) 2021-2022 Tim Evens. All rights reserved. # # Container used to build the OpenBMP components # # Author: Tim Evens # -# BUILD: docker build -t openbmp/dev-image . -# docker tag openbmp/dev-image openbmp/dev-image:latest +# BUILD: docker buildx build --progress=plain -t openbmp/dev-image:latest . FROM debian:bullseye-slim AS build @@ -22,13 +20,12 @@ RUN apt-get install -y openjdk-17-jdk-headless maven RUN mkdir -p /usr/share/man/man1/ \ && apt-get -y install git gcc g++ libboost-dev cmake zlib1g-dev libssl-dev libsasl2-dev \ liblz4-dev libzstd-dev librdkafka-dev - - # Build/install yaml-cpp RUN cd /tmp && git clone https://github.com/jbeder/yaml-cpp.git \ && cd yaml-cpp \ + && git checkout yaml-cpp-0.7.0 \ && mkdir build && cd build \ - && cmake -DBUILD_SHARED_LIBS=OFF .. \ + && cmake -DYAML_BUILD_SHARED_LIB=OFF .. \ && make && make install \ && cd /tmp diff --git a/docker-compose.yml b/docker-compose.yml index ed298b2..eef3742 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,7 +25,7 @@ services: depends_on: - zookeeper ports: - - 9092:9092 + - "9092:9092" environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: obmp-zookeeper:2181 @@ -50,7 +50,7 @@ services: container_name: obmp-grafana image: grafana/grafana:8.3.4 ports: - - 3000:3000 + - "3000:3000" volumes: - ${OBMP_DATA_ROOT}/grafana:/var/lib/grafana - ${OBMP_DATA_ROOT}/grafana/provisioning:/etc/grafana/provisioning/ @@ -58,14 +58,15 @@ services: - GF_SECURITY_ADMIN_PASSWORD=openbmp - GF_AUTH_ANONYMOUS_ENABLED=true - GF_SERVER_ROOT_URL=/ # Change this if you have grafana behind URL/paths, such as /grafana/ - - GF_INSTALL_PLUGINS=agenty-flowcharting-panel,briangann-datatable-panel,digrich-bubblechart-panel,grafana-piechart-panel,grafana-worldmap-panel,vonage-status-panel + - GF_INSTALL_PLUGINS=agenty-flowcharting-panel,grafana-piechart-panel,grafana-worldmap-panel,grafana-simple-json-datasource,vonage-status-panel psql: restart: unless-stopped container_name: obmp-psql - image: openbmp/postgres:2.0.3 + image: openbmp/postgres:2.1.0 + shm_size: 1024m ports: - - 5432:5432 + - "5432:5432" volumes: - ${OBMP_DATA_ROOT}/postgres/data:/var/lib/postgresql/data # change this to 80GB SSD slice/partition - ${OBMP_DATA_ROOT}/postgres/ts:/var/lib/postgresql/ts # Chnage this to 500GB SSD slice/partition @@ -78,13 +79,13 @@ services: collector: restart: unless-stopped container_name: obmp-collector - image: openbmp/collector:2.0.3 + image: openbmp/collector:2.1.0 sysctls: - net.ipv4.tcp_keepalive_intvl=30 - net.ipv4.tcp_keepalive_probes=5 - net.ipv4.tcp_keepalive_time=180 ports: - - 5000:5000 + - "5000:5000" volumes: - ${OBMP_DATA_ROOT}/config:/config environment: @@ -93,13 +94,13 @@ services: psql-app: restart: unless-stopped container_name: obmp-psql-app - image: openbmp/psql-app:2.0.3 + image: openbmp/psql-app:2.1.0 sysctls: - net.ipv4.tcp_keepalive_intvl=30 - net.ipv4.tcp_keepalive_probes=5 - net.ipv4.tcp_keepalive_time=180 ports: - - 9005:9005 + - "9005:9005" volumes: - ${OBMP_DATA_ROOT}/config:/config @@ -132,3 +133,22 @@ services: - POSTGRES_DROP_stats_ip_origins='4 weeks' - POSTGRES_DROP_stats_peer_rib='4 weeks' - POSTGRES_DROP_stats_peer_update_counts='4 weeks' + + whois: + restart: unless-stopped + container_name: obmp-whois + image: openbmp/whois:2.1.0 + sysctls: + - net.ipv4.tcp_keepalive_intvl=30 + - net.ipv4.tcp_keepalive_probes=5 + - net.ipv4.tcp_keepalive_time=180 + ports: + - "4300:43" +# volumes: +# - ${OBMP_DATA_ROOT}/config:/config + environment: + - POSTGRES_PASSWORD=openbmp + - POSTGRES_USER=openbmp + - POSTGRES_DB=openbmp + - POSTGRES_HOST=obmp-psql + - POSTGRES_PORT=5432 \ No newline at end of file diff --git a/postgres/Dockerfile b/postgres/Dockerfile index e243b22..23fbae0 100644 --- a/postgres/Dockerfile +++ b/postgres/Dockerfile @@ -1,16 +1,16 @@ # Postgres Container # -# Copyright (c) 2021-2022 Cisco Systems, Inc. and Tim Evens. All rights reserved. +# Copyright (c) 2021-2022 Cisco Systems, Inc. and others. All rights reserved. # # Build: -# docker build -t openbmp/postgres:2.0.3 . +# docker build --platform linux/amd64 -t openbmp/postgres:2.1.0 . # # Run: # docker run --rm -it -p 5432:5432 \ # -e POSTGRES_PASSWORD=openbmp \ # -e POSTGRES_USER=openbmp \ # -e POSTGRES_DB=openbmp \ -# openbmp/postgres:2.0.2 +# openbmp/postgres:2.1.0 FROM timescale/timescaledb:2.5.1-pg14 @@ -36,9 +36,7 @@ RUN apk update \ && sed -i -e "s/^\#*listen_addresses.*=.*/listen_addresses = '*'/" /usr/local/share/postgresql/postgresql.conf.sample \ && sed -i -e "s/^\#*ssl[ ]*=.*/ssl = on/" /usr/local/share/postgresql/postgresql.conf.sample \ && sed -i -e "s/^\#*ssl_cert_file.*=.*/ssl_cert_file = \'\/psql_server.crt\'/" /usr/local/share/postgresql/postgresql.conf.sample \ - && sed -i -e "s/^\#*ssl_key_file.*=.*/ssl_key_file = \'\/psql_server.key\'/" /usr/local/share/postgresql/postgresql.conf.sample \ - && sed -i -e "s/^\#*shared_buffers.*=.*/shared_buffers = ${MEM:-1}GB/" /usr/local/share/postgresql/postgresql.conf.sample \ - && sed -i -e "s/^\#*work_mem.*=.*/work_mem = $(( (${MEM:-1} * 1024) * 5 / 100))MB/" /usr/local/share/postgresql/postgresql.conf.sample + && sed -i -e "s/^\#*ssl_key_file.*=.*/ssl_key_file = \'\/psql_server.key\'/" /usr/local/share/postgresql/postgresql.conf.sample # Init timesries location RUN echo 'mkdir -p /var/lib/postgresql/ts/data' > /docker-entrypoint-initdb.d/0_obmp_init.sh \ diff --git a/psql-app/Dockerfile b/psql-app/Dockerfile index d4b36a6..89e9521 100644 --- a/psql-app/Dockerfile +++ b/psql-app/Dockerfile @@ -1,6 +1,7 @@ -# Postgres Backend: openbmp/psql-consumer +# syntax=docker/dockerfile:1.3 +# Postgres Backend: openbmp/psql-app # -# Copyright (c) 2021-2022 Cisco Systems, Inc. and Tim Evens. All rights reserved. +# Copyright (c) 2021-2022 Cisco Systems, Inc. and others. All rights reserved. # # Author: Tim Evens # @@ -11,10 +12,13 @@ # Change directories to obmp-docker/psql-app and run the below from that # directory. # +# Use buildkit - either via export DOCKER_BUILDKIT=1 or by using buildx cli +# # Example docker build: -# tar -cL -C ../../ ./obmp-psql ./obmp-docker/psql-app ./obmp-java-api-message \ -# | docker build --build-arg VERSION=2.0.3 \ -# -f obmp-docker/psql-app/Dockerfile -t openbmp/psql-app:2.0.3 - +# tar -c -C ../../ ./obmp-psql ./obmp-docker/psql-app ./obmp-java-api-message \ +# | docker buildx build --platform linux/amd64 --progress=plain \ +# --build-arg VERSION=2.1.0 \ +# -f obmp-docker/psql-app/Dockerfile -t openbmp/psql-app:2.1.0 - # ----------------------------------------------- # stage: Build @@ -45,12 +49,13 @@ FROM openjdk:17-slim # Copy files from previous stages COPY --from=build /ws/target/obmp-psql-consumer-0.1.0-SNAPSHOT.jar /usr/local/openbmp/obmp-psql-consumer.jar COPY --from=build /ws/database/ /usr/local/openbmp/database -COPY --from=build /ws/cron_scripts/gen-whois/*.py /usr/local/openbmp/ -COPY --from=build /ws/cron_scripts/rpki/*.py /usr/local/openbmp/ -COPY --from=build /ws/scripts/dbip-to-psql.py /usr/local/openbmp/ +COPY --chmod=755 --from=build /ws/cron_scripts/gen-whois/*.py /usr/local/openbmp/ +COPY --chmod=755 --from=build /ws/cron_scripts/rpki/*.py /usr/local/openbmp/ +COPY --chmod=755 --from=build /ws/scripts/dbip-to-psql.py /usr/local/openbmp/ # Add files -ADD obmp-docker/psql-app/scripts/run /usr/sbin/ +ADD --chmod=755 obmp-docker/psql-app/scripts/run /usr/sbin/ +ADD --chmod=755 obmp-docker/psql-app/scripts/upgrade_l3vpn.sh /usr/local/openbmp/upgrade_l3vpn.sh #---------------------------------- # Define persistent data volumes diff --git a/psql-app/scripts/run b/psql-app/scripts/run index c86f6e1..48cd5d0 100755 --- a/psql-app/scripts/run +++ b/psql-app/scripts/run @@ -146,7 +146,9 @@ MAILTO="" 6 */2 * * * root . /usr/local/openbmp/pg_profile && /usr/local/openbmp/gen_whois_asn.py -u $PGUSER -p $PGPASSWORD $PGHOST >> /var/log/asn_load.log # Update aggregation table stats -*/3 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_chg_stats($POSTGRES_REPORT_WINDOW)" +*/5 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_chg_stats('5 minute')" +*/5 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_l3vpn_chg_stats('5 minute')" + # Update peer rib counts */15 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_peer_rib_counts()" @@ -165,6 +167,19 @@ SETVAR } +# ----------------------------------------------- +# Upgrade SQL +# ----------------------------------------------- +upgrade() { + if [[ ! -f /config/psql-app-upgraded.2.1.0 ]]; then + echo "===> Upgrading to 2.1.0" + /usr/local/openbmp/upgrade_l3vpn.sh + touch /config/psql-app-upgraded.2.1.0 + echo "===> Done with upgrade" + fi +} + + # ----------------------------------------------- # run_consumer # ----------------------------------------------- @@ -244,6 +259,8 @@ rm -f /var/run/rsyslogd.pid service cron start service rsyslog start +upgrade + run_consumer echo "===> Now running!!!" @@ -251,3 +268,5 @@ echo "===> Now running!!!" while [ 1 ]; do sleep 1800 done + +} \ No newline at end of file diff --git a/psql-app/scripts/upgrade_l3vpn.sh b/psql-app/scripts/upgrade_l3vpn.sh new file mode 100755 index 0000000..e65b45a --- /dev/null +++ b/psql-app/scripts/upgrade_l3vpn.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Upgrade script for L3VPN +# +# Copyright (c) 2022 Cisco Systems, Inc. and Tim Evens. All rights reserved. +# +# Author: Tim Evens +# + +. /usr/local/openbmp/pg_profile && psql -c "select * from l3vpn_rib limit 1" > /dev/null 2>&1 + +if [[ $? -ne 0 ]]; then + echo "==> Upgrading L3VPN SQL ======================================= " + psql < /usr/local/openbmp/database/10_l3vpn.sql + echo "==> Done upgrading L3VPN SQL ================================== " +fi \ No newline at end of file diff --git a/whois/Dockerfile b/whois/Dockerfile new file mode 100644 index 0000000..74db2a1 --- /dev/null +++ b/whois/Dockerfile @@ -0,0 +1,58 @@ +# syntax=docker/dockerfile:1.3 +# Postgres Backend: openbmp/whois +# +# Copyright (c) 2022 Cisco Systems, Inc. and others. All rights reserved. +# +# BUILD: +# tar -c -C ../../ ./obmp-whois ./obmp-docker/whois \ +# | docker buildx build --platform linux/amd64 --progress=plain \ +# -f obmp-docker/whois/Dockerfile -t openbmp/whois:2.1.0 - + +# ----------------------------------------------- +# stage: Build +# ----------------------------------------------- +FROM golang:alpine AS build + +WORKDIR /ws + +COPY obmp-whois/ /ws + +ENV GO111MODULE=on +ENV GCGO_ENABLED=0 +ENV GOARCH=amd64 +ENV GOOS=linux + +RUN apk add --no-cache bash git +RUN cd /ws && go mod tidy + +# Build the app +RUN cd /ws && go build -o obmp-whoisd . + + +# ----------------------------------------------- +# stage: Final container +# ----------------------------------------------- +FROM alpine + +COPY --from=build /ws/obmp-whoisd /usr/local/bin/ + +WORKDIR /tmp + +# Environment varialbes that should be set when running container +# Defaults are given below +ENV DB_MAX_THREADS=10 +ENV WHOIS_PORT=43 + +ENV PGHOST=obmp-psql +ENV PGDATABASE=openbmp +ENV PGUSER=openbmp +ENV PGPASSWORD=openbmp +ENV PGPORT=5432 + +EXPOSE 43 + +# -- general depends +RUN apk add --no-cache bash + +# -- set entry point to bash and act as login (load profile) +ENTRYPOINT /usr/local/bin/obmp-whoisd --logfile /dev/stdout -p $WHOIS_PORT -t $DB_MAX_THREADS \ No newline at end of file