Version 2.2.3 updates

* collector v2.2.3
* collector using debian-stable-slim
* dev-image updated to use debian-stable-slim
* Upgraded librdkafka to v1.9.2
* Fixed permission problems with postgres
* Grafana upgraded to 9.1.7
* psql-app v2.2.2
* postgres updated to use timescaledb-ha:pg14-ts2.8

collector/Dockerfile

@@ -18,9 +18,9 @@
 #
 #   Example docker build:
 #     tar -c -C ../../ ./obmp-collector ./obmp-docker/collector \
-#           | docker buildx build --progress=plain \
+#           | docker buildx build --platform linux/amd64 --progress=plain \
-#                 --build-arg VERSION=2.2.0 \
+#                 --build-arg VERSION=2.2.3 \
-#                 -f obmp-docker/collector/Dockerfile -t openbmp/collector:2.2.0 -
+#                 -f obmp-docker/collector/Dockerfile -t openbmp/collector:2.2.3 -
 #
 
 # -----------------------------------------------
@@ -32,7 +32,7 @@ COPY obmp-collector/ /ws
 WORKDIR /ws
 
 RUN rm -rf build && mkdir -p build && cd build \
-    && cmake -DCMAKE_INSTALL_PREFIX:PATH=/usr ../ \
+    && cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX:PATH=/usr ../ \
     && make \
     && make install
 
@@ -41,7 +41,7 @@ RUN rm -rf build && mkdir -p build && cd build \
 # stage: Final container
 # -----------------------------------------------
 # Pull base image.
-FROM debian:bullseye-slim
+FROM debian:stable-slim
 
 # Add files.
 ADD --chmod=755 obmp-docker/collector/scripts/install /tmp/
@@ -52,9 +52,6 @@ ARG VERSION=0
 # Copy files from previous stages
 COPY --chmod=755 --from=build /usr/bin/openbmpd /usr/bin/
 COPY --from=build /usr/etc/openbmp/openbmpd.conf /usr/etc/openbmp/openbmpd.conf
-COPY --from=build /etc/init/openbmpd.conf /etc/init/openbmpd.conf
-COPY --from=build /etc/default/openbmpd.new /etc/default/openbmpd
-COPY --from=build /etc/logrotate.d/openbmpd /etc/logrotate.d/openbmpd
 
 # Proxy servers
 #ENV http_proxy http://proxy:80

dev-image/Dockerfile

@@ -6,9 +6,9 @@
 #
 # Author: Tim Evens <tim@openbmp.org>
 #
-# BUILD: docker buildx build --progress=plain -t openbmp/dev-image:latest .
+# BUILD: docker buildx build  --platform linux/amd64 --progress=plain -t openbmp/dev-image:latest .
 
-FROM debian:bullseye-slim AS build
+FROM debian:stable-slim AS build
 
 ENV DEBIAN_FRONTEND=noninteractive
 
@@ -18,8 +18,27 @@ WORKDIR /ws
 RUN apt-get update
 RUN apt-get install -y openjdk-17-jdk-headless maven
 RUN mkdir -p /usr/share/man/man1/ \
-    && apt-get -y install git gcc g++ libboost-dev cmake zlib1g-dev libssl-dev libsasl2-dev \
+    && apt-get -y install git gcc g++ libboost-dev cmake libssl-dev libsasl2-dev \
-         liblz4-dev libzstd-dev librdkafka-dev
+         curl wget libgss-dev   liblz4-dev libzstd-dev
+    # zlib1g-dev
+
+# Build/install zlib - zlib1g-dev does not work for static builds of librdkafka
+RUN cd /tmp && git clone https://github.com/madler/zlib.git \
+    && cd zlib \
+    && git checkout v1.2.12 \
+    && CFLAGS=-fPIC  ./configure --static \
+    && make install
+
+# Build/install librdkafka
+RUN cd /tmp && git clone https://github.com/edenhill/librdkafka.git \
+    && cd librdkafka \
+    && git checkout v1.9.2 \
+    && ./configure --enable-static --disable-curl \
+    && make \
+    && make install
+
+# Installed under /usr/local/lib
+
 # Build/install yaml-cpp
 RUN cd /tmp && git clone https://github.com/jbeder/yaml-cpp.git \
     && cd yaml-cpp \

docker-compose.yml

@@ -1,5 +1,17 @@
 ---
 version: '3'
+volumes:
+  data-volume:
+    driver_opts:
+      type: none
+      device: ${OBMP_DATA_ROOT}/postgres/data
+      o: bind
+  ts-volume:
+    driver_opts:
+      type: none
+      device: ${OBMP_DATA_ROOT}/postgres/ts
+      o: bind
+
 services:
 
   zookeeper:
@@ -73,7 +85,7 @@ services:
   grafana:
     restart: unless-stopped
     container_name: obmp-grafana
-    image: grafana/grafana:8.5.4
+    image: grafana/grafana:9.1.7
     ports:
       - "3000:3000"
     volumes:
@@ -107,7 +119,7 @@ services:
   psql:
     restart: unless-stopped
     container_name: obmp-psql
-    image: openbmp/postgres:2.2.0
+    image: openbmp/postgres:2.2.1
     privileged: true
     shm_size: 1536m
     sysctls:
@@ -117,8 +129,8 @@ services:
     ports:
       - "5432:5432"
     volumes:
-      - ${OBMP_DATA_ROOT}/postgres/data:/var/lib/postgresql/data  # change this to 80GB SSD slice/partition
+      - data-volume:/var/lib/postgresql/data
-      - ${OBMP_DATA_ROOT}/postgres/ts:/var/lib/postgresql/ts      # Chnage this to 500GB SSD slice/partition
+      - ts-volume:/var/lib/postgresql/ts
     # alter_job max_runtime in _timescaledb_config.bgw_job ( https://docs.timescale.com/latest/api#alter_job )
     command: >
       -c max_wal_size=10GB
@@ -130,7 +142,7 @@ services:
   collector:
     restart: unless-stopped
     container_name: obmp-collector
-    image: openbmp/collector:2.2.0
+    image: openbmp/collector:2.2.3
     sysctls:
       - net.ipv4.tcp_keepalive_intvl=30
       - net.ipv4.tcp_keepalive_probes=5
@@ -145,7 +157,7 @@ services:
   psql-app:
     restart: unless-stopped
     container_name: obmp-psql-app
-    image: openbmp/psql-app:2.2.1
+    image: openbmp/psql-app:2.2.2
     sysctls:
       - net.ipv4.tcp_keepalive_intvl=30
       - net.ipv4.tcp_keepalive_probes=5

postgres/Dockerfile

@@ -4,16 +4,16 @@
 #
 #  Build:
 #        DOCKER_BUILDKIT=1 docker build --platform linux/amd64 \
-#            --build-arg VERSION=2.2.0 \
+#            --build-arg VERSION=2.2.1 \
 #            --squash \
-#            -t openbmp/postgres:2.2.0 .
+#            -t openbmp/postgres:2.2.1 .
 #
 #  Run:
 #       docker run --rm -it -p 5432:5432 \
 #              -e POSTGRES_PASSWORD=openbmp \
 #              -e POSTGRES_USER=openbmp \
 #              -e POSTGRES_DB=openbmp \
-#              openbmp/postgres:2.2.0
+#              openbmp/postgres:2.2.1
 
 # -----------------------------------------------
 # stage: Build
@@ -23,7 +23,7 @@
 # -----------------------------------------------
 # stage: Final container
 # -----------------------------------------------
-FROM timescale/timescaledb-ha:pg14-ts2.7-latest
+FROM timescale/timescaledb-ha:pg14-ts2.8-latest
 
 ARG VERSION=0.0.0
 

psql-app/Dockerfile

@@ -17,8 +17,8 @@
 #   Example docker build:
 #     tar -c -C ../../ ./obmp-psql ./obmp-docker/psql-app ./obmp-java-api-message \
 #           | docker buildx build --platform linux/amd64 --progress=plain \
-#                 --build-arg VERSION=2.2.1 \
+#                 --build-arg VERSION=2.2.2 \
-#                 -f obmp-docker/psql-app/Dockerfile -t openbmp/psql-app:2.2.1 -
+#                 -f obmp-docker/psql-app/Dockerfile -t openbmp/psql-app:2.2.2 -
 
 # -----------------------------------------------
 # stage: Build

psql-app/scripts/run

@@ -205,6 +205,8 @@ SETVAR
 # -----------------------------------------------
 upgrade() {
 
+  if [[ -f /config/do_not_init_db ]]; then
+
     if [[ ! -f /config/psql-app-upgraded.2.1.0 ]]; then
       echo "===> Upgrading to 2.1.0"
       /tmp/upgrade/upgrade_2.1.0.sh
@@ -226,7 +228,19 @@ upgrade() {
       echo "===> Done with upgrade"
     fi
 
+   if [[ ! -f /config/psql-app-upgraded.2.2.2 ]]; then
+      echo "===> Upgrading to 2.2.2"
+      /tmp/upgrade/upgrade_2.2.2.sh
+      touch /config/psql-app-upgraded.2.2.2
+      echo "===> Done with upgrade"
+    fi
 
+  else
+      touch /config/psql-app-upgraded.2.1.0
+      touch /config/psql-app-upgraded.2.2.0
+      touch /config/psql-app-upgraded.2.2.1
+      touch /config/psql-app-upgraded.2.2.2
+  fi
 }
 
 

psql-app/upgrade/upgrade_2.2.2.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Upgrade script for L3VPN
+#
+#  Copyright (c) 2022 Cisco Systems, Inc. and Tim Evens.  All rights reserved.
+#
+# Author: Tim Evens <tim@evensweb.com>
+#
+
+source /usr/local/openbmp/pg_profile
+
+echo "==> Upgrading to 2.2.2 SQL ==================================== "
+psql < /tmp/upgrade/upgrade_2.2.2.sql
+echo "==> Done upgrading to 2.2.2 SQL ================================== "
+
+

psql-app/upgrade/upgrade_2.2.2.sql

@@ -0,0 +1,131 @@
+-- -----------------------------------------------------------------------
+-- Copyright (c) 2022 Cisco Systems, Inc. and others.  All rights reserved.
+--
+-- Upgrade form 2.2.1 to 2.2.2
+-- -----------------------------------------------------------------------
+CREATE INDEX IF NOT EXISTS base_attrs_next_hop_idx ON base_attrs (next_hop);
+
+
+CREATE OR REPLACE FUNCTION update_global_ip_rib(max_interval interval DEFAULT '2 hour')
+	RETURNS void AS $$
+DECLARE
+	execution_start timestamptz  := clock_timestamp();
+	insert_count    int;
+	start_time timestamptz := now();
+BEGIN
+
+	select time_bucket('5 minutes', timestamp - interval '5 minute') INTO start_time
+	FROM global_ip_rib order by timestamp desc limit 1;
+
+	IF start_time is null THEN
+		start_time = time_bucket('5 minutes', now() - max_interval);
+		raise INFO '-> Last query time is null, setting last query time within %', max_interval;
+	ELSIF start_time < now() - max_interval THEN
+		start_time = time_bucket('5 minutes', now() - max_interval);
+		raise INFO '-> Last query time is greater than max % time, setting last query time', max_interval;
+	ELSIF start_time > now() THEN
+		start_time = time_bucket('5 minutes', now() - interval '15 minutes');
+		raise INFO '-> Last query time is greater than current time, setting last query time to past 15 minutes';
+	END IF;
+
+	raise INFO 'Start time       : %', execution_start;
+	raise INFO 'Last Query Time  : %', start_time;
+
+	raise INFO '-> Updating changed prefixes ...';
+
+	insert_count = 0;
+
+	INSERT INTO global_ip_rib (prefix,prefix_len,recv_origin_as,
+	                           iswithdrawn,timestamp,first_added_timestamp,num_peers,advertising_peers,withdrawn_peers)
+
+	SELECT r.prefix,
+	       max(r.prefix_len),
+	       r.origin_as,
+	       bool_and(r.iswithdrawn)                                             as isWithdrawn,
+	       max(r.timestamp),
+	       min(r.first_added_timestamp),
+	       count(distinct r.peer_hash_id)                                      as total_peers,
+	       count(distinct r.peer_hash_id) FILTER (WHERE r.iswithdrawn = False) as advertising_peers,
+	       count(distinct r.peer_hash_id) FILTER (WHERE r.iswithdrawn = True)  as withdrawn_peers
+	FROM ip_rib r
+	WHERE
+		(timestamp >= start_time OR first_added_timestamp >= start_time)
+	  AND origin_as != 23456
+	GROUP BY r.prefix, r.origin_as
+	ON CONFLICT (prefix,recv_origin_as)
+		DO UPDATE SET timestamp=excluded.timestamp,
+		              first_added_timestamp=excluded.first_added_timestamp,
+		              iswithdrawn=excluded.iswithdrawn,
+		              num_peers=excluded.num_peers,
+		              advertising_peers=excluded.advertising_peers,
+		              withdrawn_peers=excluded.withdrawn_peers;
+
+	GET DIAGNOSTICS insert_count = row_count;
+	raise INFO 'Rows updated   : %', insert_count;
+	raise INFO 'Duration       : %', clock_timestamp() - execution_start;
+	raise INFO 'Completion time: %', clock_timestamp();
+
+	-- Update IRR
+	raise INFO '-> Updating IRR info';
+	UPDATE global_ip_rib r SET
+		                       irr_origin_as=i.origin_as,
+		                       irr_source=i.source,
+		                       irr_descr=i.descr
+	FROM info_route i
+	WHERE  r.timestamp >= start_time and i.prefix = r.prefix;
+
+	GET DIAGNOSTICS insert_count = row_count;
+	raise INFO 'Rows updated   : %', insert_count;
+	raise INFO 'Duration       : %', clock_timestamp() - execution_start;
+	raise INFO 'Completion time: %', clock_timestamp();
+
+
+	-- Update RPKI entries - Limit query to only update what has changed in interval time
+	--    NOTE: The global_ip_rib table should have current times when first run (new table).
+	--          This will result in this query taking a while. After first run, it shouldn't take
+	--          as long.
+	raise INFO '-> Updating RPKI info';
+	UPDATE global_ip_rib r SET rpki_origin_as=p.origin_as
+	FROM rpki_validator p
+	WHERE r.timestamp >= start_time
+	  AND p.prefix >>= r.prefix
+	  AND r.prefix_len >= p.prefix_len
+	  AND r.prefix_len <= p.prefix_len_max;
+
+	GET DIAGNOSTICS insert_count = row_count;
+	raise INFO 'Rows updated   : %', insert_count;
+	raise INFO 'Duration       : %', clock_timestamp() - execution_start;
+
+
+	raise INFO 'Completion time: %', clock_timestamp();
+
+END;
+$$ LANGUAGE plpgsql;
+
+drop view IF EXISTS v_ls_links CASCADE;
+
+ALTER TABLE ls_links
+	ALTER COLUMN admin_group TYPE bigint,
+	ALTER COLUMN unreserved_bw TYPE varchar(128);
+ALTER TABLE ls_links_log
+	ALTER COLUMN admin_group TYPE bigint,
+	ALTER COLUMN unreserved_bw TYPE varchar(128);
+
+
+CREATE VIEW v_ls_links AS
+SELECT localn.name as Local_Router_Name,remoten.name as Remote_Router_Name,
+       localn.igp_router_id as Local_IGP_RouterId,localn.router_id as Local_RouterId,
+       remoten.igp_router_id Remote_IGP_RouterId, remoten.router_id as Remote_RouterId,
+       localn.seq, localn.bgp_ls_id as bgpls_id,
+       CASE WHEN ln.protocol in ('OSPFv2', 'OSPFv3') THEN localn.ospf_area_id ELSE localn.isis_area_id END as AreaId,
+       ln.mt_id as MT_ID,interface_addr as InterfaceIP,neighbor_addr as NeighborIP,
+       ln.isIPv4,ln.protocol,igp_metric,local_link_id,remote_link_id,admin_group,max_link_bw,max_resv_bw,
+       unreserved_bw,te_def_metric,mpls_proto_mask,srlg,ln.name,ln.timestamp,local_node_hash_id,remote_node_hash_id,
+       localn.igp_router_id as localn_igp_router_id,remoten.igp_router_id as remoten_igp_router_id,
+       ln.base_attr_hash_id as base_attr_hash_id, ln.peer_hash_id as peer_hash_id,
+       CASE WHEN ln.iswithdrawn THEN 'WITHDRAWN' ELSE 'ACTIVE' END as state
+FROM ls_links ln
+	     JOIN ls_nodes localn ON (ln.local_node_hash_id = localn.hash_id
+	AND ln.peer_hash_id = localn.peer_hash_id)
+	     JOIN ls_nodes remoten ON (ln.remote_node_hash_id = remoten.hash_id
+	AND ln.peer_hash_id = remoten.peer_hash_id);