Merge pull request #141 from farsightsec/next

Release 1.2.0

Author: mcrawforddt

.gitignore

@@ -59,3 +59,6 @@ test-suite.log
 tests/test-io
 tests/test-misc
 tests/test-parse
+tests/test-private
+tests/*/*.out
+tests/*/test.sh

COPYRIGHT

@@ -1,6 +1,6 @@
 Most nmsg code is under the following copyright and license:
 
-    Copyright (c) 2023 DomainTools LLC
+    Copyright (c) 2023-2024 DomainTools LLC
     Copyright (c) 2008-2021 by Farsight Security, Inc.
 
     Licensed under the Apache License, Version 2.0 (the "License");

ChangeLog

@@ -1,3 +1,16 @@
+nmsg (1.2.0)
+
+  * Add support for Kafka i/o in JSON and binary forms.
+
+  * Add support for prometheus metrics export.
+
+  * Replace mutex protected counters and flags with stdatomic operations.
+
+  * New command line options: --kafkakey, --readkafka, --writekafka, --promport
+
+  * New functions: nmsg_input_open_kafka_endpoint(), nmsg_input_open_kafka_json(),
+                   nmsg_output_open_kafka_endpoint(), nmsg_output_open_kafka_json()
+
 nmsg (1.1.2)
 
   * Now allow nmsgtool --setgroup, --setoperator, and --setsource to

Makefile.am

@@ -18,7 +18,10 @@ AM_CFLAGS = \
 	$(libprotobuf_c_CFLAGS) \
 	$(libwdns_CFLAGS) \
 	$(libzmq_CFLAGS) \
-	$(json_c_CFLAGS)
+	$(librdkafka_CFLAGS) \
+	$(json_c_CFLAGS) \
+	$(libmicrohttpd_CFLAGS) \
+	$(libprom_CFLAGS)
 AM_LDFLAGS =
 
 EXTRA_DIST += ChangeLog
@@ -51,6 +54,7 @@ EXTRA_DIST += \
 	tests/test-layout-fltmod_plugin.c \
 	tests/test-nmsg_output_set_rate.c \
 	tests/test-parse.c \
+	tests/test-private.c \
 	tests/test-io.c \
 	tests/test-misc.c \
 	tests/udp-checksum-tests/test.sh.in \
@@ -121,26 +125,25 @@ nobase_nodist_include_HEADERS = \
 
 lib_LTLIBRARIES = nmsg/libnmsg.la
 
-VERSION_INFO = 9:1:1
+VERSION_INFO = 10:0:2
 
 nmsg_libnmsg_la_LDFLAGS = \
 	$(AM_LDFLAGS) \
 	-version-info $(VERSION_INFO) \
 	-export-symbols-regex "^(nmsg_[a-z].*)"
-nmsg_libnmsg_la_LIBADD = \
+LIBNMSG_LIB_DEPS = \
 	$(libpcap_LIBS) \
 	$(libprotobuf_c_LIBS) \
 	$(libzmq_LIBS) \
+	$(librdkafka_LIBS) \
 	$(json_c_LIBS)
-nmsg_libnmsg_la_SOURCES = \
-	libmy/crc32c.c libmy/crc32c.h libmy/crc32c-slicing.c libmy/crc32c-sse42.c \
-	libmy/list.h \
-	libmy/my_time.h \
-	libmy/my_rate.c libmy/my_rate.h \
-	libmy/tree.h \
-	libmy/b64_decode.c libmy/b64_decode.h \
-	libmy/b64_encode.c libmy/b64_encode.h \
-	libmy/fast_inet_ntop.c libmy/fast_inet_ntop.h \
+nmsg_libnmsg_la_LIBADD = $(LIBNMSG_LIB_DEPS)
+LIBNMSG_LIB_MODULES = \
+	libmy/crc32c.c libmy/crc32c-slicing.c libmy/crc32c-sse42.c \
+	libmy/my_rate.c \
+	libmy/b64_decode.c \
+	libmy/b64_encode.c \
+	libmy/fast_inet_ntop.c \
 	nmsg/alias.c \
 	nmsg/asprintf.c \
 	nmsg/brate.c \
@@ -161,37 +164,50 @@ nmsg_libnmsg_la_SOURCES = \
 	nmsg/io.c \
 	nmsg/ipdg.c \
 	nmsg/ipreasm.c \
-	nmsg/ipreasm.h \
 	nmsg/msgmodset.c \
 	nmsg/nmsg.c \
-	nmsg/nmsg_json.h \
-	nmsg/nmsg_port_net.h \
 	nmsg/output.c \
 	nmsg/output_json.c \
 	nmsg/output_nmsg.c \
 	nmsg/output_pres.c \
 	nmsg/payload.c \
 	nmsg/pcap_input.c \
-	nmsg/private.h \
 	nmsg/random.c \
 	nmsg/rate.c \
 	nmsg/res.c \
 	nmsg/sock.c \
 	nmsg/strbuf.c \
 	nmsg/timespec.c \
-	nmsg/version.c nmsg/version.h \
+	nmsg/version.c \
 	nmsg/zmqio.c \
+	nmsg/kafkaio.c \
 	nmsg/zbuf.c \
 	nmsg/msgmod/lookup.c \
 	nmsg/msgmod/message.c \
 	nmsg/msgmod/msgmod.c \
 	nmsg/msgmod/transparent.c \
-	nmsg/msgmod/transparent.h \
 	nmsg/msgmod/transparent_json.c \
 	nmsg/msgmod/transparent_message.c \
 	nmsg/msgmod/transparent_module.c \
 	nmsg/msgmod/transparent_payload.c \
 	nmsg/msgmod/transparent_pres.c
+nmsg_libnmsg_la_SOURCES = \
+	libmy/crc32c.h \
+	libmy/list.h \
+	libmy/my_time.h \
+	libmy/my_rate.h \
+	libmy/tree.h \
+	libmy/b64_decode.h \
+	libmy/b64_encode.h \
+	libmy/fast_inet_ntop.h \
+	nmsg/ipreasm.h \
+	nmsg/nmsg_json.h \
+	nmsg/nmsg_port_net.h \
+	nmsg/private.h \
+	nmsg/version.h \
+	nmsg/kafkaio.h \
+	nmsg/msgmod/transparent.h \
+	$(LIBNMSG_LIB_MODULES)
 nodist_nmsg_libnmsg_la_SOURCES = \
 	nmsg/nmsg.pb-c.c \
 	nmsg/nmsg.pb-c.h
@@ -330,7 +346,10 @@ bin_PROGRAMS += src/nmsgtool
 src_nmsgtool_LDADD = \
 	nmsg/libnmsg.la \
 	$(libpcap_LIBS) \
-	$(libzmq_LIBS)
+	$(libzmq_LIBS) \
+	$(librdkafka_LIBS) \
+	$(libmicrohttpd_LIBS) \
+	$(libprom_LIBS)
 src_nmsgtool_SOURCES = \
 	libmy/argv.c \
 	libmy/argv.h \
@@ -346,6 +365,10 @@ src_nmsgtool_SOURCES = \
 	src/rwfile.c \
 	src/unescape.c
 
+if USE_DT_PROM
+src_nmsgtool_SOURCES += src/dt_prom.c \
+                        src/dt_prom.h
+endif
 #
 ##
 ### Tests
@@ -428,10 +451,20 @@ check_PROGRAMS += tests/test-parse
 tests_test_parse_LDADD = nmsg/libnmsg.la
 tests_test_parse_SOURCES = tests/test-parse.c
 
+TESTS += tests/test-private
+check_PROGRAMS += tests/test-private
+tests_test_private_LDFLAGS = -rdynamic
+PRIVATE_TEST_MODULES = $(LIBNMSG_LIB_MODULES:.c=.o)
+tests_test_private_LDADD = \
+	$(PRIVATE_TEST_MODULES) \
+	nmsg/nmsg.pb-c.o \
+	$(LIBNMSG_LIB_DEPS)
+tests_test_private_SOURCES = tests/test-private.c
+
 TESTS += tests/test-io
 check_PROGRAMS += tests/test-io
 tests_test_io_CPPFLAGS = -DSRCDIR="\"$(abs_srcdir)\"" $(AM_CPPFLAGS)
-tests_test_io_LDADD = nmsg/libnmsg.la $(libzmq_LIBS)
+tests_test_io_LDADD = nmsg/libnmsg.la $(libzmq_LIBS) $(librdkafka_LIBS)
 tests_test_io_SOURCES = tests/test-io.c
 
 TESTS += tests/test-misc
@@ -473,14 +506,10 @@ examples_nmsg_callback_SOURCES = examples/nmsg_callback.c
 examples_print_srcip_LDADD = nmsg/libnmsg.la
 examples_print_srcip_SOURCES = examples/print_srcip.c
 
-examples_nmsg_dnsqr2pcap_LDADD = \
-	nmsg/libnmsg.la \
-	$(libpcap_LIBS)
+examples_nmsg_dnsqr2pcap_LDADD = nmsg/libnmsg.la $(libpcap_LIBS)
 examples_nmsg_dnsqr2pcap_SOURCES = examples/nmsg-dnsqr2pcap.c
 
-examples_nmsg_packet2pcap_LDADD = \
-	nmsg/libnmsg.la \
-	$(libpcap_LIBS)
+examples_nmsg_packet2pcap_LDADD = nmsg/libnmsg.la $(libpcap_LIBS)
 examples_nmsg_packet2pcap_SOURCES = examples/nmsg-packet2pcap.c
 
 examples_print_version_LDADD = nmsg/libnmsg.la

README.md

@@ -19,13 +19,19 @@ nmsg has the following external dependencies:
 
 * [zmq](http://zeromq.org/)
 
+* [rdkafka](https://github.com/confluentinc/librdkafka)
+
 * [json-c](https://github.com/json-c/json-c)
 
 * [zlib](http://www.zlib.net/)
 
+* prometheus integration (optional)
+  * [libprometheus](https://github.com/digitalocean/prometheus-client-c)
+  * [libmicrohttpd](https://www.gnu.org/software/libmicrohttpd/)
+
 On Debian systems, the following packages should be installed, if available:
 
-    pkg-config libpcap0.8-dev libprotobuf-c-dev protobuf-c-compiler libzmq3-dev libjson-c-dev zlib1g-dev
+    pkg-config libpcap0.8-dev libprotobuf-c-dev protobuf-c-compiler libzmq3-dev librdkafka-dev libjson-c-dev zlib1g-dev
 
 Note that on Debian systems, binary packages of nmsg and its dependencies are
 available from
@@ -35,7 +41,8 @@ Debian-based systems.
 
 On FreeBSD systems, the following ports should be installed, if available:
 
-    devel/libzmq
+    net/libzmq3
+    net/librdkafka
     devel/json-c
     devel/pkgconf
     devel/protobuf
@@ -54,6 +61,9 @@ script.
 Support for `libzmq` can be disabled by passing the `--without-libzmq` parameter
 to the `configure` script.
 
+Support for `librdkafka` can be disabled by passing the `--without-librdkafka` parameter
+to the `configure` script.
+
 Support for `json-c` can be disabled by passing the `--without-json-c` parameter
 to the `configure` script.
 

configure.ac

@@ -1,8 +1,8 @@
 AC_PREREQ(2.64)
 
 m4_define(nmsg_major_version, 1)
-m4_define(nmsg_minor_version, 1)
-m4_define(nmsg_patchlevel_version, 2)
+m4_define(nmsg_minor_version, 2)
+m4_define(nmsg_patchlevel_version, 0)
 m4_define(nmsg_version,
 	  nmsg_major_version.nmsg_minor_version.nmsg_patchlevel_version)
 m4_define(nmsg_version_number,
@@ -51,6 +51,7 @@ AC_CONFIG_FILES([tests/nmsg-dns-tests/test.sh],
 AC_CONFIG_FILES([tests/nmsg-dnsobs-tests/test.sh],
                 [chmod +x tests/nmsg-dnsobs-tests/test.sh])
 
+AC_SUBST(shlibpath_var)
 AC_CONFIG_FILES([tests/nmsg-dnsqr-tests/test.sh],
                 [chmod +x tests/nmsg-dnsqr-tests/test.sh])
 
@@ -149,6 +150,15 @@ AS_IF([test -z "$PROTOC_C"],
 
 PKG_CHECK_MODULES([libwdns], [libwdns >= 0.12.0])
 
+AC_ARG_WITH([librdkafka], AS_HELP_STRING([--without-librdkafka], [Disable kafka support]))
+if test "x$with_librdkafka" != "xno"; then
+    PKG_CHECK_MODULES([librdkafka], [rdkafka >= 1.1.0])
+    AC_DEFINE([HAVE_LIBRDKAFKA], [1], [Define to 1 if librdkafka support is enabled.])
+    use_librdkafka="true"
+else
+    use_librdkafka="false"
+fi
+
 AC_ARG_WITH([libzmq], AS_HELP_STRING([--without-libzmq], [Disable zmq support]))
 if test "x$with_libzmq" != "xno"; then
     PKG_CHECK_MODULES([libzmq], [libzmq >= 4.2.0])
@@ -167,6 +177,18 @@ else
     use_json_c="false"
 fi
 
+AC_ARG_WITH([prometheus], AS_HELP_STRING([--with-prometheus], [Enable prometheus metrics]))
+if test "x$with_prometheus" == "xyes"; then
+    PKG_CHECK_MODULES([libmicrohttpd], [libmicrohttpd >= 0.9.0])
+    PKG_CHECK_MODULES([libprom], [libprom >= 0.1.0])
+    AC_DEFINE([HAVE_PROMETHEUS], [1], [Define to 1 if prometheus metrics are enabled.])
+    USE_PROMETHEUS="yes"
+    AM_CONDITIONAL([USE_DT_PROM], [true])
+else
+    USE_PROMETHEUS="no"
+    AM_CONDITIONAL([USE_DT_PROM], [false])
+fi
+
 AC_CHECK_HEADER([zlib.h], [], [ AC_MSG_ERROR([required header file not found]) ])
 AC_CHECK_LIB([z], [deflate], [], [ AC_MSG_ERROR([required library not found]) ])
 
@@ -226,7 +248,9 @@ AC_MSG_RESULT([
 
         bigendian:              ${ac_cv_c_bigendian}
         libzmq support:         ${use_libzmq}
+        librdkafka support:     ${use_librdkafka}
         json-c support:         ${use_json_c}
+        prometheus support:     ${USE_PROMETHEUS}
 
         building html docs:     ${DOC_HTML_MSG}
         building manpage docs:  ${DOC_MAN_MSG}

debian/changelog

@@ -1,3 +1,14 @@
+nmsg (1.2.0-1) debian-fsi; urgency=medium
+
+  * Add support for Kafka i/o in JSON and binary forms.
+  * Add support for prometheus metrics export.
+  * Replace mutex protected counters and flags with stdatomic operations.
+  * New command line options: --kafkakey, --readkafka, --writekafka, --promport
+  * New functions: nmsg_input_open_kafka_endpoint(), nmsg_input_open_kafka_json(),
+                   nmsg_output_open_kafka_endpoint(), nmsg_output_open_kafka_json()
+
+ -- Farsight Security Inc <[email protected]>  Wed, 26 Jun 2024 13:56:58 -0400
+
 nmsg (1.1.2-1) debian-fsi; urgency=medium
 
   * Now allow nmsgtool --setgroup, --setoperator, and --setsource to

debian/control

@@ -13,6 +13,7 @@ Build-Depends:
  libprotobuf-c-dev (>= 1.0.1~),
  libwdns-dev (>= 0.12.0~),
  libzmq3-dev (>= 4.2.0~),
+ librdkafka-dev (>= 1.1.0~),
  libjson-c-dev (>= 0.13.0~),
  pkg-config,
  protobuf-c-compiler (>= 1.0.1~),

debian/copyright

@@ -1,7 +1,7 @@
 Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 
 Files: *
-Copyright: 2023 DomainTools LLC
+Copyright: 2023-2024 DomainTools LLC
            2008-2021 by Farsight Security, Inc.
 License: Apache-2.0
  Licensed under the Apache License, Version 2.0 (the "License"); you

debian/libnmsg8.symbols

@@ -28,6 +28,8 @@ libnmsg.so.8 libnmsg8 #MINVER#
  nmsg_input_open_callback@Base 0.7.0
  nmsg_input_open_file@Base 0.5.0
  nmsg_input_open_json@Base 0.10.0
+ nmsg_input_open_kafka_endpoint@Base 1.2.0
+ nmsg_input_open_kafka_json@Base 1.2.0
  nmsg_input_open_null@Base 0.7.0
  nmsg_input_open_pcap@Base 0.5.0
  nmsg_input_open_pres@Base 0.5.0
@@ -130,6 +132,8 @@ libnmsg.so.8 libnmsg8 #MINVER#
  nmsg_output_open_callback@Base 0.5.0
  nmsg_output_open_file@Base 0.5.0
  nmsg_output_open_json@Base 0.10.0
+ nmsg_output_open_kafka_endpoint@Base 1.2.0
+ nmsg_output_open_kafka_json@Base 1.2.0
  nmsg_output_open_pres@Base 0.11.1
  nmsg_output_open_sock@Base 0.5.0
  nmsg_output_open_zmq@Base 0.14.0