sam/netbox-diode-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix create_message_chunks import error across all collectors

Browse Source 
The installed Diode SDK version does not export create_message_chunks.
Replace chunked ingestion with direct client.ingest() calls.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
sam 2026-02-28 16:49:25 -07:00
parent 427dbf456d
commit 37b92c166a
7 changed files with 35 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
collectors/cml_collector.py
View File

@ -415,8 +415,6 @@ def ingest_entities(entities: list[Entity], dry_run: bool = False) -> None:
log.info("Ingesting %d entities to %s ...", len(entities), target) log.info("Ingesting %d entities to %s ...", len(entities), target)
from netboxlabs.diode.sdk.ingester import create_message_chunks
with DiodeClient( with DiodeClient(
target=target, target=target,
client_id=client_id, client_id=client_id,
@ -424,13 +422,11 @@ def ingest_entities(entities: list[Entity], dry_run: bool = False) -> None:
app_name="cml-collector", app_name="cml-collector",
app_version="0.1.0", app_version="0.1.0",
) as client: ) as client:
chunks = create_message_chunks(entities) resp = client.ingest(entities=entities)
for idx, chunk in enumerate(chunks): if resp.errors:
resp = client.ingest(entities=chunk) log.error("Ingestion errors: %s", resp.errors)
if resp.errors: else:
log.error("Chunk %d errors: %s", idx, resp.errors) log.info("Ingestion successful")
else:
log.info("Chunk %d: %d entities ingested", idx, len(chunk))
def main(): def main():

14
collectors/docker_collector.py
View File

@ -315,8 +315,6 @@ def ingest_entities(entities: list[Entity], dry_run: bool = False) -> None:
log.info("Ingesting %d entities to %s ...", len(entities), target) log.info("Ingesting %d entities to %s ...", len(entities), target)
from netboxlabs.diode.sdk.ingester import create_message_chunks
with DiodeClient( with DiodeClient(
target=target, target=target,
client_id=client_id, client_id=client_id,
@ -324,13 +322,11 @@ def ingest_entities(entities: list[Entity], dry_run: bool = False) -> None:
app_name="docker-collector", app_name="docker-collector",
app_version="0.1.0", app_version="0.1.0",
) as client: ) as client:
chunks = create_message_chunks(entities) resp = client.ingest(entities=entities)
for idx, chunk in enumerate(chunks): if resp.errors:
resp = client.ingest(entities=chunk) log.error("Ingestion errors: %s", resp.errors)
if resp.errors: else:
log.error("Chunk %d errors: %s", idx, resp.errors) log.info("Ingestion successful")
else:
log.info("Chunk %d: %d entities ingested", idx, len(chunk))
def main(): def main():

14
collectors/network_collector.py
View File

@ -1608,8 +1608,6 @@ def ingest_entities(entities: list[Entity], dry_run: bool = False) -> None:
log.info("Ingesting %d entities to %s ...", len(entities), target) log.info("Ingesting %d entities to %s ...", len(entities), target)
from netboxlabs.diode.sdk.ingester import create_message_chunks
with DiodeClient( with DiodeClient(
target=target, target=target,
client_id=client_id, client_id=client_id,
@ -1617,13 +1615,11 @@ def ingest_entities(entities: list[Entity], dry_run: bool = False) -> None:
app_name="network-collector", app_name="network-collector",
app_version="0.1.0", app_version="0.1.0",
) as client: ) as client:
chunks = create_message_chunks(entities) resp = client.ingest(entities=entities)
for idx, chunk in enumerate(chunks): if resp.errors:
resp = client.ingest(entities=chunk) log.error("Ingestion errors: %s", resp.errors)
if resp.errors: else:
log.error("Chunk %d errors: %s", idx, resp.errors) log.info("Ingestion successful")
else:
log.info("Chunk %d: %d entities ingested", idx, len(chunk))
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------

14
collectors/observium_collector.py
View File

@ -337,8 +337,6 @@ def ingest_entities(entities: list[Entity], dry_run: bool = False) -> None:
log.info("Ingesting %d entities to %s ...", len(entities), target) log.info("Ingesting %d entities to %s ...", len(entities), target)
from netboxlabs.diode.sdk.ingester import create_message_chunks
with DiodeClient( with DiodeClient(
target=target, target=target,
client_id=client_id, client_id=client_id,
@ -346,13 +344,11 @@ def ingest_entities(entities: list[Entity], dry_run: bool = False) -> None:
app_name="observium-collector", app_name="observium-collector",
app_version="0.1.0", app_version="0.1.0",
) as client: ) as client:
chunks = create_message_chunks(entities) resp = client.ingest(entities=entities)
for idx, chunk in enumerate(chunks): if resp.errors:
resp = client.ingest(entities=chunk) log.error("Ingestion errors: %s", resp.errors)
if resp.errors: else:
log.error("Chunk %d errors: %s", idx, resp.errors) log.info("Ingestion successful")
else:
log.info("Chunk %d: %d entities ingested", idx, len(chunk))
def main(): def main():

14
collectors/unifi_collector.py
View File

@ -713,8 +713,6 @@ def ingest_entities(entities: list[Entity], dry_run: bool = False) -> None:
log.info("Ingesting %d entities to %s ...", len(entities), target) log.info("Ingesting %d entities to %s ...", len(entities), target)
from netboxlabs.diode.sdk.ingester import create_message_chunks
with DiodeClient( with DiodeClient(
target=target, target=target,
client_id=client_id, client_id=client_id,
@ -722,13 +720,11 @@ def ingest_entities(entities: list[Entity], dry_run: bool = False) -> None:
app_name="unifi-collector", app_name="unifi-collector",
app_version="0.1.0", app_version="0.1.0",
) as client: ) as client:
chunks = create_message_chunks(entities) resp = client.ingest(entities=entities)
for idx, chunk in enumerate(chunks): if resp.errors:
resp = client.ingest(entities=chunk) log.error("Ingestion errors: %s", resp.errors)
if resp.errors: else:
log.error("Chunk %d errors: %s", idx, resp.errors) log.info("Ingestion successful")
else:
log.info("Chunk %d: %d entities ingested", idx, len(chunk))
def main(): def main():

14
collectors/vmware_collector.py
View File

@ -503,8 +503,6 @@ def ingest_entities(entities: list[Entity], dry_run: bool = False) -> None:
log.info("Ingesting %d entities to %s ...", len(entities), target) log.info("Ingesting %d entities to %s ...", len(entities), target)
from netboxlabs.diode.sdk.ingester import create_message_chunks
with DiodeClient( with DiodeClient(
target=target, target=target,
client_id=client_id, client_id=client_id,
@ -512,13 +510,11 @@ def ingest_entities(entities: list[Entity], dry_run: bool = False) -> None:
app_name="vmware-collector", app_name="vmware-collector",
app_version="0.1.0", app_version="0.1.0",
) as client: ) as client:
chunks = create_message_chunks(entities) resp = client.ingest(entities=entities)
for idx, chunk in enumerate(chunks): if resp.errors:
resp = client.ingest(entities=chunk) log.error("Ingestion errors: %s", resp.errors)
if resp.errors: else:
log.error("Chunk %d errors: %s", idx, resp.errors) log.info("Ingestion successful")
else:
log.info("Chunk %d: %d entities ingested", idx, len(chunk))
def main(): def main():

14
collectors/zabbix_collector.py
View File

@ -331,8 +331,6 @@ def ingest_entities(entities: list[Entity], dry_run: bool = False) -> None:
log.info("Ingesting %d entities to %s ...", len(entities), target) log.info("Ingesting %d entities to %s ...", len(entities), target)
from netboxlabs.diode.sdk.ingester import create_message_chunks
with DiodeClient( with DiodeClient(
target=target, target=target,
client_id=client_id, client_id=client_id,
@ -340,13 +338,11 @@ def ingest_entities(entities: list[Entity], dry_run: bool = False) -> None:
app_name="zabbix-collector", app_name="zabbix-collector",
app_version="0.1.0", app_version="0.1.0",
) as client: ) as client:
chunks = create_message_chunks(entities) resp = client.ingest(entities=entities)
for idx, chunk in enumerate(chunks): if resp.errors:
resp = client.ingest(entities=chunk) log.error("Ingestion errors: %s", resp.errors)
if resp.errors: else:
log.error("Chunk %d errors: %s", idx, resp.errors) log.info("Ingestion successful")
else:
log.info("Chunk %d: %d entities ingested", idx, len(chunk))
def main(): def main():