From 05737d2682477061b119272d7bd2ebaaf52c8afa Mon Sep 17 00:00:00 2001 From: Tim Evens Date: Fri, 4 Mar 2022 07:27:23 -0800 Subject: [PATCH] v2.1.0 updates * Add peeringdb script and cron job * Fix running more than one cronjob at a time * Update upgrade script for psql-app --- docker-compose.yml | 1 - postgres/Dockerfile | 3 ++- psql-app/Dockerfile | 5 ++-- psql-app/scripts/run | 27 ++++++++++++------- .../upgrade_2.1.0.sh} | 12 +++++++-- 5 files changed, 33 insertions(+), 15 deletions(-) rename psql-app/{scripts/upgrade_l3vpn.sh => upgrade/upgrade_2.1.0.sh} (56%) diff --git a/docker-compose.yml b/docker-compose.yml index eef3742..ce94b6c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,7 +71,6 @@ services: - ${OBMP_DATA_ROOT}/postgres/data:/var/lib/postgresql/data # change this to 80GB SSD slice/partition - ${OBMP_DATA_ROOT}/postgres/ts:/var/lib/postgresql/ts # Chnage this to 500GB SSD slice/partition environment: - - MEM=10 # Set MEM to at least 2GB but ideally >16GB - POSTGRES_PASSWORD=openbmp - POSTGRES_USER=openbmp - POSTGRES_DB=openbmp diff --git a/postgres/Dockerfile b/postgres/Dockerfile index 23fbae0..b03bdd5 100644 --- a/postgres/Dockerfile +++ b/postgres/Dockerfile @@ -27,13 +27,14 @@ VOLUME ["/var/lib/postgresql/ts"] RUN apk update \ && apk add openssl \ && openssl req -x509 -newkey rsa:4096 -nodes -subj "/C=US/ST=CA/L=Seattle/O=OpenBMP/CN=localhost" \ - -keyout /psql_server.key -out /psql_server.crt -days 365 \ + -keyout /psql_server.key -out /psql_server.crt -days 2048 \ && chown postgres /psql_server.* \ && mkdir -p /var/lib/postgresql/ts \ && chown postgres /var/lib/postgresql/ts \ && egrep -q -e '^hostssl( |\t)+all' /usr/local/share/postgresql/pg_hba.conf.sample || \ echo 'hostssl all all 0.0.0.0/0 md5' >> /usr/local/share/postgresql/pg_hba.conf.sample \ && sed -i -e "s/^\#*listen_addresses.*=.*/listen_addresses = '*'/" /usr/local/share/postgresql/postgresql.conf.sample \ + && sed -i -e "s/^\#*max_wal_size.*=.*/max_wal_size = 10GB/" /usr/local/share/postgresql/postgresql.conf.sample \ && sed -i -e "s/^\#*ssl[ ]*=.*/ssl = on/" /usr/local/share/postgresql/postgresql.conf.sample \ && sed -i -e "s/^\#*ssl_cert_file.*=.*/ssl_cert_file = \'\/psql_server.crt\'/" /usr/local/share/postgresql/postgresql.conf.sample \ && sed -i -e "s/^\#*ssl_key_file.*=.*/ssl_key_file = \'\/psql_server.key\'/" /usr/local/share/postgresql/postgresql.conf.sample diff --git a/psql-app/Dockerfile b/psql-app/Dockerfile index 89e9521..f91fa06 100644 --- a/psql-app/Dockerfile +++ b/psql-app/Dockerfile @@ -50,12 +50,13 @@ FROM openjdk:17-slim COPY --from=build /ws/target/obmp-psql-consumer-0.1.0-SNAPSHOT.jar /usr/local/openbmp/obmp-psql-consumer.jar COPY --from=build /ws/database/ /usr/local/openbmp/database COPY --chmod=755 --from=build /ws/cron_scripts/gen-whois/*.py /usr/local/openbmp/ +COPY --chmod=755 --from=build /ws/cron_scripts/peeringdb/*.py /usr/local/openbmp/ COPY --chmod=755 --from=build /ws/cron_scripts/rpki/*.py /usr/local/openbmp/ COPY --chmod=755 --from=build /ws/scripts/dbip-to-psql.py /usr/local/openbmp/ # Add files ADD --chmod=755 obmp-docker/psql-app/scripts/run /usr/sbin/ -ADD --chmod=755 obmp-docker/psql-app/scripts/upgrade_l3vpn.sh /usr/local/openbmp/upgrade_l3vpn.sh +COPY --chmod=755 obmp-docker/psql-app/upgrade /tmp/upgrade #---------------------------------- # Define persistent data volumes @@ -85,7 +86,7 @@ RUN apt-get update \ RUN cd /tmp && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \ && python3 get-pip.py -RUN pip install ipaddr pykafka +RUN pip install ipaddr pykafka click RUN pip3 install urllib3 requests diff --git a/psql-app/scripts/run b/psql-app/scripts/run index 48cd5d0..b511191 100755 --- a/psql-app/scripts/run +++ b/psql-app/scripts/run @@ -1,7 +1,7 @@ #!/bin/bash # Postgres Backend: Run script # -# Copyright (c) 2021-2022 Cisco Systems, Inc. and Tim Evens. All rights reserved. +# Copyright (c) 2021-2022 Cisco Systems, Inc. and others. All rights reserved. # # Author: Tim Evens # @@ -143,24 +143,26 @@ config_cron() { MAILTO="" # Update ASN info -6 */2 * * * root . /usr/local/openbmp/pg_profile && /usr/local/openbmp/gen_whois_asn.py -u $PGUSER -p $PGPASSWORD $PGHOST >> /var/log/asn_load.log +6 */2 * * * root . /usr/local/openbmp/pg_profile && flock -n /tmp/locks/gen_whois.lock /usr/local/openbmp/gen_whois_asn.py -u $PGUSER -p $PGPASSWORD $PGHOST > /var/log/asn_load.log 2>&1 +5 */8 * * * root . /usr/local/openbmp/pg_profile && flock -n /tmp/locks/peeringdb.lock /usr/local/openbmp/peeringdb.py > /var/log/cron-peeringdb.log 2>&1 # Update aggregation table stats -*/5 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_chg_stats('5 minute')" -*/5 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_l3vpn_chg_stats('5 minute')" +*/5 * * * * root . /usr/local/openbmp/pg_profile && flock -n /tmp/locks/update_chg_stats.lock psql -c "select update_chg_stats('5 minute')" > /var/log/cron-update_chg_stats.log 2>&1 +*/5 * * * * root . /usr/local/openbmp/pg_profile && flock -n /tmp/locks/update_l3vpn_chg_stats.lock psql -c "select update_l3vpn_chg_stats('5 minute')" > /var/log/cron-update_l3vpn_chg_stats.log 2>&1 # Update peer rib counts -*/15 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_peer_rib_counts()" +*/15 * * * * root . /usr/local/openbmp/pg_profile && flock -n /tmp/locks/update_peer_rib_counts.lock psql -c "select update_peer_rib_counts()" > /var/log/cron-update_peer_rib_counts.log 2>&1 # Update peer update counts -*/30 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_peer_update_counts(1800)" +*/30 * * * * root . /usr/local/openbmp/pg_profile && flock -n /tmp/locks/update_peer_counts.lock psql -c "select update_peer_update_counts(1800)" > /var/log/cron-update_peer_counts.log 2>&1 # Update global rib -*/5 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_global_ip_rib('5 minute');" +*/5 * * * * root . /usr/local/openbmp/pg_profile && flock -n /tmp/locks/global_ip_rib.lock psql -c "select update_global_ip_rib('5 minute');" > /var/log/cron-update_global_ip_rib.log 2>&1 +5 */4 * * * root . /usr/local/openbmp/pg_profile && flock -n /tmp/locks/global_ip_rib.lock psql -c "select purge_global_ip_rib('6 hour');" > /var/log/cron-purge_global_ip_rib.log 2>&1 # Update origin stats -21 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_origin_stats('1 hour');" +21 * * * * root . /usr/local/openbmp/pg_profile && flock -n /tmp/locks/update_origin_stats.lock psql -c "select update_origin_stats('1 hour');" > /var/log/cron-update_origin_stats.log 2>&1 SETVAR @@ -173,7 +175,7 @@ SETVAR upgrade() { if [[ ! -f /config/psql-app-upgraded.2.1.0 ]]; then echo "===> Upgrading to 2.1.0" - /usr/local/openbmp/upgrade_l3vpn.sh + /tmp/upgrade/upgrade_2.1.0.sh touch /config/psql-app-upgraded.2.1.0 echo "===> Done with upgrade" fi @@ -230,6 +232,13 @@ run_consumer() { # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ SYS_NUM_CPU=$(grep processor /proc/cpuinfo | wc -l) +# Clear locks +if [[ ! -d /tmp/locks ]]; then + mkdir /tmp/locks +else + rm -rf /tmp/locks/* +fi + update_hosts check_kafka diff --git a/psql-app/scripts/upgrade_l3vpn.sh b/psql-app/upgrade/upgrade_2.1.0.sh similarity index 56% rename from psql-app/scripts/upgrade_l3vpn.sh rename to psql-app/upgrade/upgrade_2.1.0.sh index e65b45a..5451d38 100755 --- a/psql-app/scripts/upgrade_l3vpn.sh +++ b/psql-app/upgrade/upgrade_2.1.0.sh @@ -6,10 +6,18 @@ # Author: Tim Evens # -. /usr/local/openbmp/pg_profile && psql -c "select * from l3vpn_rib limit 1" > /dev/null 2>&1 +source /usr/local/openbmp/pg_profile + +psql -c "select * from l3vpn_rib limit 1" > /dev/null 2>&1 if [[ $? -ne 0 ]]; then echo "==> Upgrading L3VPN SQL ======================================= " psql < /usr/local/openbmp/database/10_l3vpn.sql echo "==> Done upgrading L3VPN SQL ================================== " -fi \ No newline at end of file + + echo "==> Upgrading to 2.1.0 SQL ==================================== " + psql < /tmp/upgrade/upgrade_2.1.0.sql + echo "==> Done upgrading to 2.1.0 SQL ================================== " +fi + +