v2.1.0 updates
* Add peeringdb script and cron job * Fix running more than one cronjob at a time * Update upgrade script for psql-app
This commit is contained in:
Tim Evens
parent
b0511daf00
commit
05737d2682
@ -71,7 +71,6 @@ services:
|
||||
- ${OBMP_DATA_ROOT}/postgres/data:/var/lib/postgresql/data # change this to 80GB SSD slice/partition
|
||||
- ${OBMP_DATA_ROOT}/postgres/ts:/var/lib/postgresql/ts # Chnage this to 500GB SSD slice/partition
|
||||
environment:
|
||||
- MEM=10 # Set MEM to at least 2GB but ideally >16GB
|
||||
- POSTGRES_PASSWORD=openbmp
|
||||
- POSTGRES_USER=openbmp
|
||||
- POSTGRES_DB=openbmp
|
||||
|
||||
@ -27,13 +27,14 @@ VOLUME ["/var/lib/postgresql/ts"]
|
||||
RUN apk update \
|
||||
&& apk add openssl \
|
||||
&& openssl req -x509 -newkey rsa:4096 -nodes -subj "/C=US/ST=CA/L=Seattle/O=OpenBMP/CN=localhost" \
|
||||
-keyout /psql_server.key -out /psql_server.crt -days 365 \
|
||||
-keyout /psql_server.key -out /psql_server.crt -days 2048 \
|
||||
&& chown postgres /psql_server.* \
|
||||
&& mkdir -p /var/lib/postgresql/ts \
|
||||
&& chown postgres /var/lib/postgresql/ts \
|
||||
&& egrep -q -e '^hostssl( |\t)+all' /usr/local/share/postgresql/pg_hba.conf.sample || \
|
||||
echo 'hostssl all all 0.0.0.0/0 md5' >> /usr/local/share/postgresql/pg_hba.conf.sample \
|
||||
&& sed -i -e "s/^\#*listen_addresses.*=.*/listen_addresses = '*'/" /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& sed -i -e "s/^\#*max_wal_size.*=.*/max_wal_size = 10GB/" /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& sed -i -e "s/^\#*ssl[ ]*=.*/ssl = on/" /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& sed -i -e "s/^\#*ssl_cert_file.*=.*/ssl_cert_file = \'\/psql_server.crt\'/" /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& sed -i -e "s/^\#*ssl_key_file.*=.*/ssl_key_file = \'\/psql_server.key\'/" /usr/local/share/postgresql/postgresql.conf.sample
|
||||
|
||||
@ -50,12 +50,13 @@ FROM openjdk:17-slim
|
||||
COPY --from=build /ws/target/obmp-psql-consumer-0.1.0-SNAPSHOT.jar /usr/local/openbmp/obmp-psql-consumer.jar
|
||||
COPY --from=build /ws/database/ /usr/local/openbmp/database
|
||||
COPY --chmod=755 --from=build /ws/cron_scripts/gen-whois/*.py /usr/local/openbmp/
|
||||
COPY --chmod=755 --from=build /ws/cron_scripts/peeringdb/*.py /usr/local/openbmp/
|
||||
COPY --chmod=755 --from=build /ws/cron_scripts/rpki/*.py /usr/local/openbmp/
|
||||
COPY --chmod=755 --from=build /ws/scripts/dbip-to-psql.py /usr/local/openbmp/
|
||||
|
||||
# Add files
|
||||
ADD --chmod=755 obmp-docker/psql-app/scripts/run /usr/sbin/
|
||||
ADD --chmod=755 obmp-docker/psql-app/scripts/upgrade_l3vpn.sh /usr/local/openbmp/upgrade_l3vpn.sh
|
||||
COPY --chmod=755 obmp-docker/psql-app/upgrade /tmp/upgrade
|
||||
|
||||
#----------------------------------
|
||||
# Define persistent data volumes
|
||||
@ -85,7 +86,7 @@ RUN apt-get update \
|
||||
RUN cd /tmp && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \
|
||||
&& python3 get-pip.py
|
||||
|
||||
RUN pip install ipaddr pykafka
|
||||
RUN pip install ipaddr pykafka click
|
||||
|
||||
RUN pip3 install urllib3 requests
|
||||
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
#!/bin/bash
|
||||
# Postgres Backend: Run script
|
||||
#
|
||||
# Copyright (c) 2021-2022 Cisco Systems, Inc. and Tim Evens. All rights reserved.
|
||||
# Copyright (c) 2021-2022 Cisco Systems, Inc. and others. All rights reserved.
|
||||
#
|
||||
# Author: Tim Evens <tim@evensweb.com>
|
||||
#
|
||||
@ -143,24 +143,26 @@ config_cron() {
|
||||
MAILTO=""
|
||||
|
||||
# Update ASN info
|
||||
6 */2 * * * root . /usr/local/openbmp/pg_profile && /usr/local/openbmp/gen_whois_asn.py -u $PGUSER -p $PGPASSWORD $PGHOST >> /var/log/asn_load.log
|
||||
6 */2 * * * root . /usr/local/openbmp/pg_profile && flock -n /tmp/locks/gen_whois.lock /usr/local/openbmp/gen_whois_asn.py -u $PGUSER -p $PGPASSWORD $PGHOST > /var/log/asn_load.log 2>&1
|
||||
5 */8 * * * root . /usr/local/openbmp/pg_profile && flock -n /tmp/locks/peeringdb.lock /usr/local/openbmp/peeringdb.py > /var/log/cron-peeringdb.log 2>&1
|
||||
|
||||
# Update aggregation table stats
|
||||
*/5 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_chg_stats('5 minute')"
|
||||
*/5 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_l3vpn_chg_stats('5 minute')"
|
||||
*/5 * * * * root . /usr/local/openbmp/pg_profile && flock -n /tmp/locks/update_chg_stats.lock psql -c "select update_chg_stats('5 minute')" > /var/log/cron-update_chg_stats.log 2>&1
|
||||
*/5 * * * * root . /usr/local/openbmp/pg_profile && flock -n /tmp/locks/update_l3vpn_chg_stats.lock psql -c "select update_l3vpn_chg_stats('5 minute')" > /var/log/cron-update_l3vpn_chg_stats.log 2>&1
|
||||
|
||||
|
||||
# Update peer rib counts
|
||||
*/15 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_peer_rib_counts()"
|
||||
*/15 * * * * root . /usr/local/openbmp/pg_profile && flock -n /tmp/locks/update_peer_rib_counts.lock psql -c "select update_peer_rib_counts()" > /var/log/cron-update_peer_rib_counts.log 2>&1
|
||||
|
||||
# Update peer update counts
|
||||
*/30 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_peer_update_counts(1800)"
|
||||
*/30 * * * * root . /usr/local/openbmp/pg_profile && flock -n /tmp/locks/update_peer_counts.lock psql -c "select update_peer_update_counts(1800)" > /var/log/cron-update_peer_counts.log 2>&1
|
||||
|
||||
# Update global rib
|
||||
*/5 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_global_ip_rib('5 minute');"
|
||||
*/5 * * * * root . /usr/local/openbmp/pg_profile && flock -n /tmp/locks/global_ip_rib.lock psql -c "select update_global_ip_rib('5 minute');" > /var/log/cron-update_global_ip_rib.log 2>&1
|
||||
5 */4 * * * root . /usr/local/openbmp/pg_profile && flock -n /tmp/locks/global_ip_rib.lock psql -c "select purge_global_ip_rib('6 hour');" > /var/log/cron-purge_global_ip_rib.log 2>&1
|
||||
|
||||
# Update origin stats
|
||||
21 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_origin_stats('1 hour');"
|
||||
21 * * * * root . /usr/local/openbmp/pg_profile && flock -n /tmp/locks/update_origin_stats.lock psql -c "select update_origin_stats('1 hour');" > /var/log/cron-update_origin_stats.log 2>&1
|
||||
|
||||
|
||||
SETVAR
|
||||
@ -173,7 +175,7 @@ SETVAR
|
||||
upgrade() {
|
||||
if [[ ! -f /config/psql-app-upgraded.2.1.0 ]]; then
|
||||
echo "===> Upgrading to 2.1.0"
|
||||
/usr/local/openbmp/upgrade_l3vpn.sh
|
||||
/tmp/upgrade/upgrade_2.1.0.sh
|
||||
touch /config/psql-app-upgraded.2.1.0
|
||||
echo "===> Done with upgrade"
|
||||
fi
|
||||
@ -230,6 +232,13 @@ run_consumer() {
|
||||
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||
SYS_NUM_CPU=$(grep processor /proc/cpuinfo | wc -l)
|
||||
|
||||
# Clear locks
|
||||
if [[ ! -d /tmp/locks ]]; then
|
||||
mkdir /tmp/locks
|
||||
else
|
||||
rm -rf /tmp/locks/*
|
||||
fi
|
||||
|
||||
update_hosts
|
||||
|
||||
check_kafka
|
||||
|
||||
@ -6,10 +6,18 @@
|
||||
# Author: Tim Evens <tim@evensweb.com>
|
||||
#
|
||||
|
||||
. /usr/local/openbmp/pg_profile && psql -c "select * from l3vpn_rib limit 1" > /dev/null 2>&1
|
||||
source /usr/local/openbmp/pg_profile
|
||||
|
||||
psql -c "select * from l3vpn_rib limit 1" > /dev/null 2>&1
|
||||
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo "==> Upgrading L3VPN SQL ======================================= "
|
||||
psql < /usr/local/openbmp/database/10_l3vpn.sql
|
||||
echo "==> Done upgrading L3VPN SQL ================================== "
|
||||
fi
|
||||
|
||||
echo "==> Upgrading to 2.1.0 SQL ==================================== "
|
||||
psql < /tmp/upgrade/upgrade_2.1.0.sql
|
||||
echo "==> Done upgrading to 2.1.0 SQL ================================== "
|
||||
fi
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user