Merge pull request #46 from farsightsec/next

Release 1.3.0

Author: alesage

COPYRIGHT

@@ -1,4 +1,4 @@
-Copyright (c) 2012-2018 by Farsight Security, Inc.
+Copyright (c) 2012-2019 by Farsight Security, Inc.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

ChangeLog

@@ -1,7 +1,22 @@
+mtbl (1.3.0)
+
+ * Fix mtbl_reload_now() on filesets with open iterators.
+ * Add support for absolute pathnames in filesets
+ * Make merge function an optional parameter for mergers and filesets to
+   enable unmerged results.
+ * Add dupsort function to sort unmerged results based on data.
+ * Add filename filter option to filesets. This provides functionality similar
+   to mtbl_fileset_partition() but resilient against fileset reloads.
+ * Add mtbl_fileset_dup() to open an existing fileset with different options.
+
+ -- Farsight Security, Inc. <[email protected]>  Fri, 29 Mar 2019 12:13:17 -0500
+
 mtbl (1.2.1)
 
  * Fix libtool version number.
 
+ -- Farsight Security, Inc. <[email protected]>  Fri May 25 17:49:08 2018 -0500
+
 mtbl (1.2.0)
 
  * Prevent fileset reloading when the fileset has iterators open.

Makefile.am

@@ -23,7 +23,7 @@ AM_LDFLAGS =
 ##
 #
 
-LIBMTBL_VERSION_INFO=1:2:0
+LIBMTBL_VERSION_INFO=2:0:1
 
 include_HEADERS = mtbl/mtbl.h
 lib_LTLIBRARIES = mtbl/libmtbl.la

configure.ac

@@ -1,6 +1,6 @@
 AC_PREREQ(2.64)
 AC_INIT([mtbl],
-        [1.2.1],
+        [1.3.0],
         [https://github.com/farsightsec/mtbl/issues],
         [mtbl],
         [https://github.com/farsightsec/mtbl])

libmy/heap.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012 by Farsight Security, Inc.
+ * Copyright (c) 2012, 2019 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -25,19 +25,21 @@ VECTOR_GENERATE(ptrvec, void *);
 struct heap {
 	ptrvec			*vec;
 	heap_compare_func	cmp;
+	void			*clos;
 };
 
 static inline int
-cmp_wrapper(heap_compare_func cmp, const void *a, const void *b)
+cmp_wrapper(heap_compare_func cmp, const void *a, const void *b, void *clos)
 {
-	return ((cmp(a, b) < 0) ? 1 : 0);
+	return ((cmp(a, b, clos) < 0) ? 1 : 0);
 }
 
 struct heap *
-heap_init(heap_compare_func cmp)
+heap_init(heap_compare_func cmp, void *clos)
 {
 	struct heap *h = my_calloc(1, sizeof(*h));
 	h->cmp = cmp;
+	h->clos = clos;
 	h->vec = ptrvec_init(1);
 	return (h);
 }
@@ -70,7 +72,7 @@ siftdown(struct heap *h, size_t startpos, size_t pos)
 	while (pos > startpos) {
 		size_t parentpos = (pos - 1) >> 1;
 		void *parent = ptrvec_value(h->vec, parentpos);
-		int cmp = cmp_wrapper(h->cmp, newitem, parent);
+		int cmp = cmp_wrapper(h->cmp, newitem, parent, h->clos);
 		if (cmp == -1)
 			return (-1);
 		if (cmp == 0)
@@ -95,7 +97,8 @@ siftup(struct heap *h, size_t pos)
 		if (rightpos < endpos) {
 			int cmp = cmp_wrapper(h->cmp,
 					      ptrvec_value(h->vec, childpos),
-					      ptrvec_value(h->vec, rightpos));
+					      ptrvec_value(h->vec, rightpos),
+					      h->clos);
 			if (cmp == -1)
 				return (-1);
 			if (cmp == 0)

libmy/heap.h

@@ -3,9 +3,9 @@
 
 struct heap;
 
-typedef int (*heap_compare_func)(const void *a, const void *b);
+typedef int (*heap_compare_func)(const void *a, const void *b, void *clos);
 
-struct heap *heap_init(heap_compare_func);
+struct heap *heap_init(heap_compare_func, void *);
 void heap_destroy(struct heap **);
 void heap_reset(struct heap *);
 void heap_clip(struct heap *, size_t);

man/mtbl_dump.1

@@ -1,13 +1,13 @@
 '\" t
 .\"     Title: mtbl_dump
 .\"    Author: [FIXME: author] [see http://docbook.sf.net/el/author]
-.\" Generator: DocBook XSL Stylesheets v1.78.1 <http://docbook.sf.net/>
-.\"      Date: 07/17/2015
+.\" Generator: DocBook XSL Stylesheets v1.79.1 <http://docbook.sf.net/>
+.\"      Date: 04/01/2019
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "MTBL_DUMP" "1" "07/17/2015" "\ \&" "\ \&"
+.TH "MTBL_DUMP" "1" "04/01/2019" "\ \&" "\ \&"
 .\" -----------------------------------------------------------------
 .\" * Define some portability stuff
 .\" -----------------------------------------------------------------

man/mtbl_fileset.3

@@ -2,12 +2,12 @@
 .\"     Title: mtbl_fileset
 .\"    Author: [FIXME: author] [see http://docbook.sf.net/el/author]
 .\" Generator: DocBook XSL Stylesheets v1.79.1 <http://docbook.sf.net/>
-.\"      Date: 05/31/2018
+.\"      Date: 03/27/2019
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "MTBL_FILESET" "3" "05/31/2018" "\ \&" "\ \&"
+.TH "MTBL_FILESET" "3" "03/27/2019" "\ \&" "\ \&"
 .\" -----------------------------------------------------------------
 .\" * Define some portability stuff
 .\" -----------------------------------------------------------------
@@ -41,6 +41,11 @@ mtbl_fileset_init(const char *\fR\fB\fIfname\fR\fR\fB, const struct mtbl_fileset
 .fi
 .sp
 .nf
+\fBstruct mtbl_fileset *
+mtbl_fileset_dup(struct mtbl_fileset *\fR\fB\fIorig\fR\fR\fB, const struct mtbl_fileset_options *\fR\fB\fIfopt\fR\fR\fB);\fR
+.fi
+.sp
+.nf
 \fBvoid
 mtbl_fileset_destroy(struct mtbl_fileset **\fR\fB\fIf\fR\fR\fB);\fR
 .fi
@@ -91,6 +96,22 @@ mtbl_fileset_options_set_merge_func(
 .sp
 .nf
 \fBvoid
+mtbl_fileset_options_set_dupsort_func(
+        struct mtbl_fileset_options *\fR\fB\fIfopt\fR\fR\fB,
+        mtbl_dupsort_func \fR\fB\fIfp\fR\fR\fB,
+        void *\fR\fB\fIclos\fR\fR\fB);\fR
+.fi
+.sp
+.nf
+\fBvoid
+mtbl_fileset_options_set_filename_filter_func(
+        struct mtbl_fileset_options *\fR\fB\fIfopt\fR\fR\fB,
+        mtbl_filename_filter_func \fR\fB\fIfp\fR\fR\fB,
+        void *\fR\fB\fIclos\fR\fR\fB);\fR
+.fi
+.sp
+.nf
+\fBvoid
 mtbl_fileset_options_set_reload_interval(
         struct mtbl_fileset_options *\fR\fB\fIfopt\fR\fR\fB,
         uint32_t \fR\fB\fIreload_interval\fR\fR\fB);\fR
@@ -107,6 +128,8 @@ Because the MTBL format does not allow duplicate keys, the caller must provide a
 .sp
 \fBmtbl_fileset\fR objects are created with the \fBmtbl_fileset_init\fR() function, which requires the path to a "setfile", \fIfname\fR, and a non\-NULL \fIfopt\fR argument which has been configured with a merge function \fIfp\fR\&. \fBmtbl_fileset_source\fR() should then be called in order to consume output via the \fBmtbl_source\fR(3) interface\&.
 .sp
+The \fBmtbl_fileset_dup\fR() function, which requires a \fBmtbl_fileset\fR object and a non\-NULL \fBopt\fR argument, creates a duplicate \fBmtbl_fileset\fR object that shares the underlying fileset, but with different fileset options, as specified\&.
+.sp
 Accesses via the \fBmtbl_source\fR(3) interface will implicitly check for updates to the setfile\&. However, it may be necessary to explicitly call the \fBmtbl_fileset_reload\fR() function in order to check for updates, especially if files are being removed from the setfile and the \fBmtbl_source\fR is infrequently accessed\&.
 .sp
 The \fBmtbl_fileset_reload\fR() function avoids checking for updates more frequently than every \fIreload_interval\fR seconds\&. If \fBreload_interval\fR is set to \fBMTBL_FILESET_RELOAD_INTERVAL_NEVER\fR, then \fBmtbl_fileset_reload\fR() function will only load the fileset once\&. The \fBmtbl_fileset_reload_now\fR() function can be called to bypass the \fIreload_interval\fR check\&.
@@ -130,6 +153,28 @@ See \fBmtbl_merger\fR(3)\&. An \fBmtbl_merger\fR object is used internally for t
 .nr an-break-flag 1
 .br
 .ps +1
+\fBdupsort_func\fR
+.RS 4
+.sp
+See \fBmtbl_merger\fR(3)\&. Used to sort the entries with duplicate keys during the merge process based on their data\&.
+.RE
+.sp
+.it 1 an-trap
+.nr an-no-space-flag 1
+.nr an-break-flag 1
+.br
+.ps +1
+\fBfname_filter_func\fR
+.RS 4
+.sp
+Used to filter specific files by name from a fileset\&. If the function returns \fBfalse\fR, the file\(cqs data will not be included in the results returned by any iterators on the fileset\&.
+.RE
+.sp
+.it 1 an-trap
+.nr an-no-space-flag 1
+.nr an-break-flag 1
+.br
+.ps +1
 \fBreload_interval\fR
 .RS 4
 .sp

man/mtbl_fileset.3.txt

@@ -14,6 +14,10 @@ Fileset objects:
 ^struct mtbl_fileset *
 mtbl_fileset_init(const char *'fname', const struct mtbl_fileset_options *'fopt');^
 
+[verse]
+^struct mtbl_fileset *
+mtbl_fileset_dup(struct mtbl_fileset *'orig', const struct mtbl_fileset_options *'fopt');^
+
 [verse]
 ^void
 mtbl_fileset_destroy(struct mtbl_fileset **'f');^
@@ -55,6 +59,20 @@ mtbl_fileset_options_set_merge_func(
         mtbl_merge_func 'fp',
         void *'clos');^
 
+[verse]
+^void
+mtbl_fileset_options_set_dupsort_func(
+        struct mtbl_fileset_options *'fopt',
+        mtbl_dupsort_func 'fp',
+        void *'clos');^
+
+[verse]
+^void
+mtbl_fileset_options_set_filename_filter_func(
+        struct mtbl_fileset_options *'fopt',
+        mtbl_filename_filter_func 'fp',
+        void *'clos');^
+
 [verse]
 ^void
 mtbl_fileset_options_set_reload_interval(
@@ -96,6 +114,10 @@ which has been configured with a merge function _fp_. ^mtbl_fileset_source^()
 should then be called in order to consume output via the ^mtbl_source^(3)
 interface.
 
+The ^mtbl_fileset_dup^() function, which requires a ^mtbl_fileset^ object
+and a non-NULL ^opt^ argument, creates a duplicate ^mtbl_fileset^ object
+that shares the underlying fileset, but with different fileset options, as specified.
+
 Accesses via the ^mtbl_source^(3) interface will implicitly check for updates
 to the setfile. However, it may be necessary to explicitly call the
 ^mtbl_fileset_reload^() function in order to check for updates, especially if
@@ -121,6 +143,13 @@ mergers in an inconsistent state.
 See ^mtbl_merger^(3). An ^mtbl_merger^ object is used internally for the
 external sort.
 
+==== dupsort_func ====
+See ^mtbl_merger^(3). Used to sort the entries with duplicate keys during the merge process based on their data.
+
+==== fname_filter_func ====
+Used to filter specific files by name from a fileset. If the function returns ^false^, the file's data
+will not be included in the results returned by any iterators on the fileset.
+
 ==== reload_interval ====
 Specifies the interval between checks for updates to the setfile, in seconds.
 Defaults to 60 seconds.  ^MTBL_FILESET_RELOAD_INTERVAL_NEVER^ is a special value that indicates to never reload the fileset.

man/mtbl_iter.3

@@ -2,12 +2,12 @@
 .\"     Title: mtbl_iter
 .\"    Author: [FIXME: author] [see http://docbook.sf.net/el/author]
 .\" Generator: DocBook XSL Stylesheets v1.79.1 <http://docbook.sf.net/>
-.\"      Date: 02/17/2017
+.\"      Date: 03/29/2019
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "MTBL_ITER" "3" "02/17/2017" "\ \&" "\ \&"
+.TH "MTBL_ITER" "3" "03/29/2019" "\ \&" "\ \&"
 .\" -----------------------------------------------------------------
 .\" * Define some portability stuff
 .\" -----------------------------------------------------------------
@@ -48,7 +48,7 @@ mtbl_iter_destroy(struct mtbl_iter **\fR\fB\fIit\fR\fR\fB);\fR
 .nf
 \fBmtbl_res
 mtbl_iter_seek(struct mtbl_iter *\fR\fB\fIit\fR\fR\fB,
-        const uint8_t **\fR\fB\fIkey\fR\fR\fB, size_t *\fR\fB\fIlen_key\fR\fR\fB);\fR
+        const uint8_t *\fR\fB\fIkey\fR\fR\fB, size_t \fR\fB\fIlen_key\fR\fR\fB);\fR
 .fi
 .SH "DESCRIPTION"
 .sp