259 lines
9.2 KiB
Bash
Executable File
259 lines
9.2 KiB
Bash
Executable File
#!/bin/bash
|
|
# Postgres Backend: Run script
|
|
#
|
|
# Copyright (c) 2021 Cisco Systems, Inc. and Tim Evens. All rights reserved.
|
|
#
|
|
# Author: Tim Evens <tim@evensweb.com>
|
|
#
|
|
|
|
# Postgres details - Can be set using docker -e
|
|
export POSTGRES_USER=${POSTGRES_USER:="openbmp"}
|
|
export POSTGRES_PASSWORD=${POSTGRES_PASSWORD:="openbmp"}
|
|
export POSTGRES_HOST=${POSTGRES_HOST:="127.0.0.1"}
|
|
export POSTGRES_PORT=${POSTGRES_PORT:="5432"}
|
|
export POSTGRES_DB=${POSTGRES_DB:="openbmp"}
|
|
export MEM=${MEM:="1"} # mem in gigabytes
|
|
export PGCONNECT_TIMEOUT=15
|
|
|
|
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
|
# Functions
|
|
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
|
|
|
# -----------------------------------------------
|
|
# Check Kafka to make sure it's valid
|
|
# -----------------------------------------------
|
|
check_kafka() {
|
|
echo "===> Performing Kafka check"
|
|
|
|
if [[ ${KAFKA_FQDN:-""} == "" ]]; then
|
|
echo "ERROR: Missing ENV KAFKA_FQDN. Cannot proceed until you add that in docker run -e KAFKA_FQDN=<...>"
|
|
exit 1
|
|
|
|
fi
|
|
|
|
echo "===> Checking Kafka bootstrap server connection"
|
|
kafkacat -u -b $KAFKA_FQDN -L | grep broker
|
|
|
|
if [[ $? -ne 0 ]]; then
|
|
echo "ERROR: Failed to connect to Kafka at $KAFKA_FQDN, check the docker run -e KAFKA_FQDN= value"
|
|
exit 1
|
|
fi
|
|
|
|
echo "testing" | timeout 5 kafkacat -b $KAFKA_FQDN -P -t openbmp.parsed.test
|
|
echo "===> Checking if we can successfully consume messages"
|
|
timeout 5 kafkacat -u -b $KAFKA_FQDN -C -c 1 -o beginning -t openbmp.parsed.test > /dev/null
|
|
|
|
if [[ $? -ne 0 ]]; then
|
|
echo "ERROR: Failed to connect to Kafka broker, check the Kafka 'advertised.listeners' configuration."
|
|
echo " Advertised hostname must be reachable within the container. You can run this container"
|
|
echo " with --add-host <hostname>:<ip> to map the ip address within the container."
|
|
echo " You can also add/update the persistent /config/hosts file with the broker hostname/ip."
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# -----------------------------------------------
|
|
# Configure Postgres shell profile
|
|
# -----------------------------------------------
|
|
config_postgres_profile() {
|
|
echo "===> Configuring PostgreSQL Shell Profile"
|
|
|
|
echo "export PGUSER=$POSTGRES_USER" > /usr/local/openbmp/pg_profile
|
|
echo "export PGPASSWORD=$POSTGRES_PASSWORD" >> /usr/local/openbmp/pg_profile
|
|
echo "export PGHOST=$POSTGRES_HOST" >> /usr/local/openbmp/pg_profile
|
|
echo "export PGPORT=$POSTGRES_PORT" >> /usr/local/openbmp/pg_profile
|
|
echo "export PGDATABASE=$POSTGRES_DB" >> /usr/local/openbmp/pg_profile
|
|
}
|
|
|
|
# -----------------------------------------------
|
|
# Initdb Postgres
|
|
# -----------------------------------------------
|
|
initdb_postgres() {
|
|
if [[ ! -f /config/do_not_init_db ]]; then
|
|
echo " ===> Initializing the DB"
|
|
|
|
# Load the schema files
|
|
echo " ===> Loading Schemas"
|
|
|
|
|
|
echo "------" > /var/log/db_schema_load.log
|
|
for file in $(ls -v /usr/local/openbmp/database/*.sql); do
|
|
echo " ===[ $file ] ========================================" >> /var/log/db_schema_load.log
|
|
psql < $file >> /var/log/db_schema_load.log 2>&1
|
|
done
|
|
|
|
touch /config/do_not_init_db
|
|
fi
|
|
}
|
|
|
|
# -----------------------------------------------
|
|
# Update hosts file
|
|
# -----------------------------------------------
|
|
update_hosts() {
|
|
echo "===> Updating /etc/hosts"
|
|
|
|
# Update the etc hosts file
|
|
if [[ -f /config/hosts ]]; then
|
|
cat /config/hosts >> /etc/hosts
|
|
fi
|
|
}
|
|
|
|
# -----------------------------------------------
|
|
# Enable RPKI
|
|
# -----------------------------------------------
|
|
enable_rpki() {
|
|
echo "===> Enabling RPKI"
|
|
|
|
cat > /etc/cron.d/openbmp-rpki <<SETVAR
|
|
MAILTO=""
|
|
|
|
# Update RPKI
|
|
31 */2 * * * root . /usr/local/openbmp/pg_profile && /usr/local/openbmp/rpki_validator.py -u $PGUSER -p $PGPASSWORD -s $RPKI_URL --rpkipassword $RPKI_PASS --rpkiuser $RPKI_USER $PGHOST
|
|
|
|
SETVAR
|
|
|
|
}
|
|
|
|
# -----------------------------------------------
|
|
# Enable IRR
|
|
# -----------------------------------------------
|
|
enable_irr() {
|
|
echo "===> Enabling IRR"
|
|
|
|
cat > /etc/cron.d/openbmp-irr <<SETVAR
|
|
MAILTO=""
|
|
|
|
# Update IRR
|
|
1 1 * * * root . /usr/local/openbmp/pg_profile && /usr/local/openbmp/gen_whois_route.py -u $PGUSER -p $PGPASSWORD $PGHOST > /var/log/irr_load.log
|
|
|
|
SETVAR
|
|
|
|
# Load IRR data
|
|
echo "Loading IRR data"
|
|
/usr/local/openbmp/gen_whois_route.py -u $PGUSER -p $PGPASSWORD $PGHOST > /var/log/irr_load.log &
|
|
}
|
|
|
|
# -----------------------------------------------
|
|
# config_cron
|
|
# -----------------------------------------------
|
|
config_cron() {
|
|
cat > /etc/cron.d/openbmp <<SETVAR
|
|
MAILTO=""
|
|
|
|
# Update ASN info
|
|
6 */2 * * * root . /usr/local/openbmp/pg_profile && /usr/local/openbmp/gen_whois_asn.py -u $PGUSER -p $PGPASSWORD $PGHOST >> /var/log/asn_load.log
|
|
|
|
# Update aggregation table stats
|
|
*/3 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_chg_stats($POSTGRES_REPORT_WINDOW)"
|
|
|
|
# Update peer rib counts
|
|
*/15 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_peer_rib_counts()"
|
|
|
|
# Update peer update counts
|
|
*/30 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_peer_update_counts(1800)"
|
|
|
|
# Update origin stats
|
|
21 * * * * root . /usr/local/openbmp/pg_profile && psql -c "select update_global_ip_rib();"
|
|
|
|
# Purge time series data that is older than desired retention
|
|
0 * */3 * * root . /usr/local/openbmp/pg_profile && psql -c "SELECT drop_chunks('peer_event_log', interval $POSTGRES_DROP_peer_event_log);"
|
|
1 * */3 * * root . /usr/local/openbmp/pg_profile && psql -c "SELECT drop_chunks('stat_reports', interval $POSTGRES_DROP_stat_reports);"
|
|
2 * */3 * * root . /usr/local/openbmp/pg_profile && psql -c "SELECT drop_chunks('ip_rib_log', interval $POSTGRES_DROP_ip_rib_log);"
|
|
3 * */3 * * root . /usr/local/openbmp/pg_profile && psql -c "SELECT drop_chunks('alerts', interval $POSTGRES_DROP_alerts);"
|
|
4 * */3 * * root . /usr/local/openbmp/pg_profile && psql -c "SELECT drop_chunks('ls_nodes_log', interval $POSTGRES_DROP_ls_nodes_log);"
|
|
5 * */3 * * root . /usr/local/openbmp/pg_profile && psql -c "SELECT drop_chunks('ls_links_log', interval $POSTGRES_DROP_ls_links_log);"
|
|
6 * */3 * * root . /usr/local/openbmp/pg_profile && psql -c "SELECT drop_chunks('ls_prefixes_log', interval $POSTGRES_DROP_ls_prefixes_log);"
|
|
7 * */3 * * root . /usr/local/openbmp/pg_profile && psql -c "SELECT drop_chunks('stats_chg_byprefix', interval $POSTGRES_DROP_stats_chg_byprefix);"
|
|
8 * */3 * * root . /usr/local/openbmp/pg_profile && psql -c "SELECT drop_chunks('stats_chg_byasn', interval $POSTGRES_DROP_stats_chg_byasn);"
|
|
9 * */3 * * root . /usr/local/openbmp/pg_profile && psql -c "SELECT drop_chunks('stats_chg_bypeer', interval $POSTGRES_DROP_stats_chg_bypeer);"
|
|
10 * */3 * * root . /usr/local/openbmp/pg_profile && psql -c "SELECT drop_chunks('stats_ip_origins', interval $POSTGRES_DROP_stats_ip_origins);"
|
|
11 * */3 * * root . /usr/local/openbmp/pg_profile && psql -c "SELECT drop_chunks('stats_peer_rib', interval $POSTGRES_DROP_stats_peer_rib);"
|
|
12 * */3 * * root . /usr/local/openbmp/pg_profile && psql -c "SELECT drop_chunks('stats_peer_update_counts', interval $POSTGRES_DROP_stats_peer_update_counts);"
|
|
|
|
SETVAR
|
|
|
|
}
|
|
|
|
# -----------------------------------------------
|
|
# run_consumer
|
|
# -----------------------------------------------
|
|
run_consumer() {
|
|
echo "===> Starting consumer"
|
|
|
|
if [[ ! -f /config/obmp-psql.yml ]]; then
|
|
cd /config
|
|
unzip /usr/local/openbmp/obmp-psql-consumer.jar obmp-psql.yml
|
|
|
|
|
|
if [[ ! -f /config/obmp-psql.yml ]]; then
|
|
echo "ERROR: Cannot create /config/obmp-psql.yml"
|
|
echo " Update permissions on /config volume to 7777 OR add configuration file to /config volume"
|
|
exit 1
|
|
fi
|
|
|
|
# Update configuration
|
|
sed -i -e "s/\([ ]*bootstrap.servers:\)\(.*\)/\1 \"${KAFKA_FQDN}\"/" /config/obmp-psql.yml
|
|
sed -i -e "s/\([ ]*host[ ]*:\)\(.*\)/\1 \"${POSTGRES_HOST}\"/" /config/obmp-psql.yml
|
|
|
|
fi
|
|
|
|
if [[ $MEM -gt 20 ]]; then
|
|
heap_mem="3g"
|
|
else
|
|
heap_mem="1536m"
|
|
fi
|
|
|
|
# Run
|
|
cd /var/log
|
|
java -Xmx${heap_mem} -Xms128m -XX:+UseG1GC -XX:+UnlockExperimentalVMOptions \
|
|
-XX:InitiatingHeapOccupancyPercent=30 -XX:G1MixedGCLiveThresholdPercent=30 \
|
|
-XX:MaxGCPauseMillis=200 -XX:ParallelGCThreads=20 -XX:ConcGCThreads=5 \
|
|
-Duser.timezone=UTC \
|
|
-jar /usr/local/openbmp/obmp-psql-consumer.jar \
|
|
-cf /config/obmp-psql.yml > /var/log/psql-console.log &
|
|
|
|
cd /tmp
|
|
}
|
|
|
|
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
|
# Run
|
|
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
|
SYS_NUM_CPU=$(grep processor /proc/cpuinfo | wc -l)
|
|
|
|
update_hosts
|
|
|
|
check_kafka
|
|
|
|
config_postgres_profile
|
|
|
|
source /usr/local/openbmp/pg_profile
|
|
|
|
config_cron
|
|
|
|
rm -f /etc/cron.d/openbmp-rpki
|
|
if [[ ${ENABLE_RPKI:-""} != "" ]]; then
|
|
enable_rpki
|
|
fi
|
|
|
|
rm -f /etc/cron.d/openbmp-irr
|
|
if [[ ${ENABLE_IRR:-""} != "" ]]; then
|
|
enable_irr
|
|
fi
|
|
|
|
|
|
initdb_postgres
|
|
|
|
# Get rid of previous rsyslogd pid
|
|
rm -f /var/run/rsyslogd.pid
|
|
|
|
service cron start
|
|
service rsyslog start
|
|
|
|
run_consumer
|
|
|
|
echo "===> Now running!!!"
|
|
|
|
while [ 1 ]; do
|
|
sleep 1800
|
|
done
|