From 3f38af5312ae4b99cc3b3dcc6f54f6909439579d Mon Sep 17 00:00:00 2001 From: Tim Evens Date: Thu, 20 Oct 2022 07:12:08 -0700 Subject: [PATCH] Version 2.2.3 updates * collector v2.2.3 * collector using debian-stable-slim * dev-image updated to use debian-stable-slim * Upgraded librdkafka to v1.9.2 * Fixed permission problems with postgres * Grafana upgraded to 9.1.7 * psql-app v2.2.2 * postgres updated to use timescaledb-ha:pg14-ts2.8 --- collector/Dockerfile | 13 ++- dev-image/Dockerfile | 27 +++++- docker-compose.yml | 24 ++++-- postgres/Dockerfile | 8 +- psql-app/Dockerfile | 4 +- psql-app/scripts/run | 14 +++ psql-app/upgrade/upgrade_2.2.2.sh | 15 ++++ psql-app/upgrade/upgrade_2.2.2.sql | 131 +++++++++++++++++++++++++++++ 8 files changed, 212 insertions(+), 24 deletions(-) create mode 100755 psql-app/upgrade/upgrade_2.2.2.sh create mode 100644 psql-app/upgrade/upgrade_2.2.2.sql diff --git a/collector/Dockerfile b/collector/Dockerfile index dcfd402..06cf6d7 100644 --- a/collector/Dockerfile +++ b/collector/Dockerfile @@ -18,9 +18,9 @@ # # Example docker build: # tar -c -C ../../ ./obmp-collector ./obmp-docker/collector \ -# | docker buildx build --progress=plain \ -# --build-arg VERSION=2.2.0 \ -# -f obmp-docker/collector/Dockerfile -t openbmp/collector:2.2.0 - +# | docker buildx build --platform linux/amd64 --progress=plain \ +# --build-arg VERSION=2.2.3 \ +# -f obmp-docker/collector/Dockerfile -t openbmp/collector:2.2.3 - # # ----------------------------------------------- @@ -32,7 +32,7 @@ COPY obmp-collector/ /ws WORKDIR /ws RUN rm -rf build && mkdir -p build && cd build \ - && cmake -DCMAKE_INSTALL_PREFIX:PATH=/usr ../ \ + && cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX:PATH=/usr ../ \ && make \ && make install @@ -41,7 +41,7 @@ RUN rm -rf build && mkdir -p build && cd build \ # stage: Final container # ----------------------------------------------- # Pull base image. -FROM debian:bullseye-slim +FROM debian:stable-slim # Add files. ADD --chmod=755 obmp-docker/collector/scripts/install /tmp/ @@ -52,9 +52,6 @@ ARG VERSION=0 # Copy files from previous stages COPY --chmod=755 --from=build /usr/bin/openbmpd /usr/bin/ COPY --from=build /usr/etc/openbmp/openbmpd.conf /usr/etc/openbmp/openbmpd.conf -COPY --from=build /etc/init/openbmpd.conf /etc/init/openbmpd.conf -COPY --from=build /etc/default/openbmpd.new /etc/default/openbmpd -COPY --from=build /etc/logrotate.d/openbmpd /etc/logrotate.d/openbmpd # Proxy servers #ENV http_proxy http://proxy:80 diff --git a/dev-image/Dockerfile b/dev-image/Dockerfile index 0bcb94c..d1190b6 100644 --- a/dev-image/Dockerfile +++ b/dev-image/Dockerfile @@ -6,9 +6,9 @@ # # Author: Tim Evens # -# BUILD: docker buildx build --progress=plain -t openbmp/dev-image:latest . +# BUILD: docker buildx build --platform linux/amd64 --progress=plain -t openbmp/dev-image:latest . -FROM debian:bullseye-slim AS build +FROM debian:stable-slim AS build ENV DEBIAN_FRONTEND=noninteractive @@ -18,8 +18,27 @@ WORKDIR /ws RUN apt-get update RUN apt-get install -y openjdk-17-jdk-headless maven RUN mkdir -p /usr/share/man/man1/ \ - && apt-get -y install git gcc g++ libboost-dev cmake zlib1g-dev libssl-dev libsasl2-dev \ - liblz4-dev libzstd-dev librdkafka-dev + && apt-get -y install git gcc g++ libboost-dev cmake libssl-dev libsasl2-dev \ + curl wget libgss-dev liblz4-dev libzstd-dev + # zlib1g-dev + +# Build/install zlib - zlib1g-dev does not work for static builds of librdkafka +RUN cd /tmp && git clone https://github.com/madler/zlib.git \ + && cd zlib \ + && git checkout v1.2.12 \ + && CFLAGS=-fPIC ./configure --static \ + && make install + +# Build/install librdkafka +RUN cd /tmp && git clone https://github.com/edenhill/librdkafka.git \ + && cd librdkafka \ + && git checkout v1.9.2 \ + && ./configure --enable-static --disable-curl \ + && make \ + && make install + +# Installed under /usr/local/lib + # Build/install yaml-cpp RUN cd /tmp && git clone https://github.com/jbeder/yaml-cpp.git \ && cd yaml-cpp \ diff --git a/docker-compose.yml b/docker-compose.yml index aa3bf73..69d44a4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,17 @@ --- version: '3' +volumes: + data-volume: + driver_opts: + type: none + device: ${OBMP_DATA_ROOT}/postgres/data + o: bind + ts-volume: + driver_opts: + type: none + device: ${OBMP_DATA_ROOT}/postgres/ts + o: bind + services: zookeeper: @@ -73,7 +85,7 @@ services: grafana: restart: unless-stopped container_name: obmp-grafana - image: grafana/grafana:8.5.4 + image: grafana/grafana:9.1.7 ports: - "3000:3000" volumes: @@ -107,7 +119,7 @@ services: psql: restart: unless-stopped container_name: obmp-psql - image: openbmp/postgres:2.2.0 + image: openbmp/postgres:2.2.1 privileged: true shm_size: 1536m sysctls: @@ -117,8 +129,8 @@ services: ports: - "5432:5432" volumes: - - ${OBMP_DATA_ROOT}/postgres/data:/var/lib/postgresql/data # change this to 80GB SSD slice/partition - - ${OBMP_DATA_ROOT}/postgres/ts:/var/lib/postgresql/ts # Chnage this to 500GB SSD slice/partition + - data-volume:/var/lib/postgresql/data + - ts-volume:/var/lib/postgresql/ts # alter_job max_runtime in _timescaledb_config.bgw_job ( https://docs.timescale.com/latest/api#alter_job ) command: > -c max_wal_size=10GB @@ -130,7 +142,7 @@ services: collector: restart: unless-stopped container_name: obmp-collector - image: openbmp/collector:2.2.0 + image: openbmp/collector:2.2.3 sysctls: - net.ipv4.tcp_keepalive_intvl=30 - net.ipv4.tcp_keepalive_probes=5 @@ -145,7 +157,7 @@ services: psql-app: restart: unless-stopped container_name: obmp-psql-app - image: openbmp/psql-app:2.2.1 + image: openbmp/psql-app:2.2.2 sysctls: - net.ipv4.tcp_keepalive_intvl=30 - net.ipv4.tcp_keepalive_probes=5 diff --git a/postgres/Dockerfile b/postgres/Dockerfile index 2069e1f..af520d3 100644 --- a/postgres/Dockerfile +++ b/postgres/Dockerfile @@ -4,16 +4,16 @@ # # Build: # DOCKER_BUILDKIT=1 docker build --platform linux/amd64 \ -# --build-arg VERSION=2.2.0 \ +# --build-arg VERSION=2.2.1 \ # --squash \ -# -t openbmp/postgres:2.2.0 . +# -t openbmp/postgres:2.2.1 . # # Run: # docker run --rm -it -p 5432:5432 \ # -e POSTGRES_PASSWORD=openbmp \ # -e POSTGRES_USER=openbmp \ # -e POSTGRES_DB=openbmp \ -# openbmp/postgres:2.2.0 +# openbmp/postgres:2.2.1 # ----------------------------------------------- # stage: Build @@ -23,7 +23,7 @@ # ----------------------------------------------- # stage: Final container # ----------------------------------------------- -FROM timescale/timescaledb-ha:pg14-ts2.7-latest +FROM timescale/timescaledb-ha:pg14-ts2.8-latest ARG VERSION=0.0.0 diff --git a/psql-app/Dockerfile b/psql-app/Dockerfile index 48e2675..f439f17 100644 --- a/psql-app/Dockerfile +++ b/psql-app/Dockerfile @@ -17,8 +17,8 @@ # Example docker build: # tar -c -C ../../ ./obmp-psql ./obmp-docker/psql-app ./obmp-java-api-message \ # | docker buildx build --platform linux/amd64 --progress=plain \ -# --build-arg VERSION=2.2.1 \ -# -f obmp-docker/psql-app/Dockerfile -t openbmp/psql-app:2.2.1 - +# --build-arg VERSION=2.2.2 \ +# -f obmp-docker/psql-app/Dockerfile -t openbmp/psql-app:2.2.2 - # ----------------------------------------------- # stage: Build diff --git a/psql-app/scripts/run b/psql-app/scripts/run index 89dfa25..650abc5 100755 --- a/psql-app/scripts/run +++ b/psql-app/scripts/run @@ -205,6 +205,8 @@ SETVAR # ----------------------------------------------- upgrade() { + if [[ -f /config/do_not_init_db ]]; then + if [[ ! -f /config/psql-app-upgraded.2.1.0 ]]; then echo "===> Upgrading to 2.1.0" /tmp/upgrade/upgrade_2.1.0.sh @@ -226,7 +228,19 @@ upgrade() { echo "===> Done with upgrade" fi + if [[ ! -f /config/psql-app-upgraded.2.2.2 ]]; then + echo "===> Upgrading to 2.2.2" + /tmp/upgrade/upgrade_2.2.2.sh + touch /config/psql-app-upgraded.2.2.2 + echo "===> Done with upgrade" + fi + else + touch /config/psql-app-upgraded.2.1.0 + touch /config/psql-app-upgraded.2.2.0 + touch /config/psql-app-upgraded.2.2.1 + touch /config/psql-app-upgraded.2.2.2 + fi } diff --git a/psql-app/upgrade/upgrade_2.2.2.sh b/psql-app/upgrade/upgrade_2.2.2.sh new file mode 100755 index 0000000..fe2931a --- /dev/null +++ b/psql-app/upgrade/upgrade_2.2.2.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Upgrade script for L3VPN +# +# Copyright (c) 2022 Cisco Systems, Inc. and Tim Evens. All rights reserved. +# +# Author: Tim Evens +# + +source /usr/local/openbmp/pg_profile + +echo "==> Upgrading to 2.2.2 SQL ==================================== " +psql < /tmp/upgrade/upgrade_2.2.2.sql +echo "==> Done upgrading to 2.2.2 SQL ================================== " + + diff --git a/psql-app/upgrade/upgrade_2.2.2.sql b/psql-app/upgrade/upgrade_2.2.2.sql new file mode 100644 index 0000000..6206799 --- /dev/null +++ b/psql-app/upgrade/upgrade_2.2.2.sql @@ -0,0 +1,131 @@ +-- ----------------------------------------------------------------------- +-- Copyright (c) 2022 Cisco Systems, Inc. and others. All rights reserved. +-- +-- Upgrade form 2.2.1 to 2.2.2 +-- ----------------------------------------------------------------------- +CREATE INDEX IF NOT EXISTS base_attrs_next_hop_idx ON base_attrs (next_hop); + + +CREATE OR REPLACE FUNCTION update_global_ip_rib(max_interval interval DEFAULT '2 hour') + RETURNS void AS $$ +DECLARE + execution_start timestamptz := clock_timestamp(); + insert_count int; + start_time timestamptz := now(); +BEGIN + + select time_bucket('5 minutes', timestamp - interval '5 minute') INTO start_time + FROM global_ip_rib order by timestamp desc limit 1; + + IF start_time is null THEN + start_time = time_bucket('5 minutes', now() - max_interval); + raise INFO '-> Last query time is null, setting last query time within %', max_interval; + ELSIF start_time < now() - max_interval THEN + start_time = time_bucket('5 minutes', now() - max_interval); + raise INFO '-> Last query time is greater than max % time, setting last query time', max_interval; + ELSIF start_time > now() THEN + start_time = time_bucket('5 minutes', now() - interval '15 minutes'); + raise INFO '-> Last query time is greater than current time, setting last query time to past 15 minutes'; + END IF; + + raise INFO 'Start time : %', execution_start; + raise INFO 'Last Query Time : %', start_time; + + raise INFO '-> Updating changed prefixes ...'; + + insert_count = 0; + + INSERT INTO global_ip_rib (prefix,prefix_len,recv_origin_as, + iswithdrawn,timestamp,first_added_timestamp,num_peers,advertising_peers,withdrawn_peers) + + SELECT r.prefix, + max(r.prefix_len), + r.origin_as, + bool_and(r.iswithdrawn) as isWithdrawn, + max(r.timestamp), + min(r.first_added_timestamp), + count(distinct r.peer_hash_id) as total_peers, + count(distinct r.peer_hash_id) FILTER (WHERE r.iswithdrawn = False) as advertising_peers, + count(distinct r.peer_hash_id) FILTER (WHERE r.iswithdrawn = True) as withdrawn_peers + FROM ip_rib r + WHERE + (timestamp >= start_time OR first_added_timestamp >= start_time) + AND origin_as != 23456 + GROUP BY r.prefix, r.origin_as + ON CONFLICT (prefix,recv_origin_as) + DO UPDATE SET timestamp=excluded.timestamp, + first_added_timestamp=excluded.first_added_timestamp, + iswithdrawn=excluded.iswithdrawn, + num_peers=excluded.num_peers, + advertising_peers=excluded.advertising_peers, + withdrawn_peers=excluded.withdrawn_peers; + + GET DIAGNOSTICS insert_count = row_count; + raise INFO 'Rows updated : %', insert_count; + raise INFO 'Duration : %', clock_timestamp() - execution_start; + raise INFO 'Completion time: %', clock_timestamp(); + + -- Update IRR + raise INFO '-> Updating IRR info'; + UPDATE global_ip_rib r SET + irr_origin_as=i.origin_as, + irr_source=i.source, + irr_descr=i.descr + FROM info_route i + WHERE r.timestamp >= start_time and i.prefix = r.prefix; + + GET DIAGNOSTICS insert_count = row_count; + raise INFO 'Rows updated : %', insert_count; + raise INFO 'Duration : %', clock_timestamp() - execution_start; + raise INFO 'Completion time: %', clock_timestamp(); + + + -- Update RPKI entries - Limit query to only update what has changed in interval time + -- NOTE: The global_ip_rib table should have current times when first run (new table). + -- This will result in this query taking a while. After first run, it shouldn't take + -- as long. + raise INFO '-> Updating RPKI info'; + UPDATE global_ip_rib r SET rpki_origin_as=p.origin_as + FROM rpki_validator p + WHERE r.timestamp >= start_time + AND p.prefix >>= r.prefix + AND r.prefix_len >= p.prefix_len + AND r.prefix_len <= p.prefix_len_max; + + GET DIAGNOSTICS insert_count = row_count; + raise INFO 'Rows updated : %', insert_count; + raise INFO 'Duration : %', clock_timestamp() - execution_start; + + + raise INFO 'Completion time: %', clock_timestamp(); + +END; +$$ LANGUAGE plpgsql; + +drop view IF EXISTS v_ls_links CASCADE; + +ALTER TABLE ls_links + ALTER COLUMN admin_group TYPE bigint, + ALTER COLUMN unreserved_bw TYPE varchar(128); +ALTER TABLE ls_links_log + ALTER COLUMN admin_group TYPE bigint, + ALTER COLUMN unreserved_bw TYPE varchar(128); + + +CREATE VIEW v_ls_links AS +SELECT localn.name as Local_Router_Name,remoten.name as Remote_Router_Name, + localn.igp_router_id as Local_IGP_RouterId,localn.router_id as Local_RouterId, + remoten.igp_router_id Remote_IGP_RouterId, remoten.router_id as Remote_RouterId, + localn.seq, localn.bgp_ls_id as bgpls_id, + CASE WHEN ln.protocol in ('OSPFv2', 'OSPFv3') THEN localn.ospf_area_id ELSE localn.isis_area_id END as AreaId, + ln.mt_id as MT_ID,interface_addr as InterfaceIP,neighbor_addr as NeighborIP, + ln.isIPv4,ln.protocol,igp_metric,local_link_id,remote_link_id,admin_group,max_link_bw,max_resv_bw, + unreserved_bw,te_def_metric,mpls_proto_mask,srlg,ln.name,ln.timestamp,local_node_hash_id,remote_node_hash_id, + localn.igp_router_id as localn_igp_router_id,remoten.igp_router_id as remoten_igp_router_id, + ln.base_attr_hash_id as base_attr_hash_id, ln.peer_hash_id as peer_hash_id, + CASE WHEN ln.iswithdrawn THEN 'WITHDRAWN' ELSE 'ACTIVE' END as state +FROM ls_links ln + JOIN ls_nodes localn ON (ln.local_node_hash_id = localn.hash_id + AND ln.peer_hash_id = localn.peer_hash_id) + JOIN ls_nodes remoten ON (ln.remote_node_hash_id = remoten.hash_id + AND ln.peer_hash_id = remoten.peer_hash_id);