Skip to contentSkip to conversation

ulogd V2 improvements, round 2

PATCH Applied [email protected] 4 participants 31 messages Feb 2, 2008 – Feb 20, 2008 netfilter

lore
Share & tools
lore
#0 lore raw

cover ulogd V2 improvements, round 2

Hi,

this is a follow-up to my previous ulogd V2 improvements patchset send to you as an RFC.

The changes to the previous set of patches are rather small, namely the code of the NFCT plugin contains two small changes after feedback from Pablo Neira.

Because all those changes were strictly chronological, with some of the changes confusingly reverted later, I decided to stuff the NFCT-related changes into one, as well as for the SQLITE3 related ones. Therefore the number of patches reduces from 30 to 15.

After the feedback I've got I'll suggest to apply all the patches without the last NFCT one. It is applied just as an RFC.

My intention is to turn the output keys of the NFCT plugin compatible to the former one, and also add a configuration option to control whether to propagate one dataset per flow or two (as it was before).

Because the NFCT-related changes depend on a patchlet to libnetfilter-conntrack which might actually not be applied, I'll suggest to convert the code base to libnl sooner. Any objections?

Any feedback welcome.

/holger

#1 lore raw

PATCH 1/15 Add NACCT output plugin

Hi,
Content-Disposition: inline; filename=ulogd-NACCT-plugin.diff

output/Makefile.am +4 -1
Index: ulogd-netfilter/output/Makefile.am
===================================================================
@@ -4,7 +4,8 @@ LIBS=""
 SUBDIRS= pcap mysql pgsql sqlite3
 
 pkglib_LTLIBRARIES = ulogd_output_LOGEMU.la ulogd_output_SYSLOG.la \
-		     ulogd_output_OPRINT.la ulogd_output_IPFIX.la
+		     ulogd_output_OPRINT.la ulogd_output_IPFIX.la \
+			 ulogd_output_NACCT.la
 
 ulogd_output_LOGEMU_la_SOURCES = ulogd_output_LOGEMU.c
 ulogd_output_LOGEMU_la_LDFLAGS = -module
@@ -18,3 +19,5 @@ ulogd_output_OPRINT_la_LDFLAGS = -module
 ulogd_output_IPFIX_la_SOURCES = ulogd_output_IPFIX.c
 ulogd_output_IPFIX_la_LDFLAGS = -module
 
+ulogd_output_NACCT_la_SOURCES = ulogd_output_NACCT.c
+ulogd_output_NACCT_la_LDFLAGS = -module
output/ulogd_output_NACCT.c new file +206 -0

206 lines not on this page · raw message

maintainer Applied
#2 lore raw

Re: [ULOGD 01/15] Add NACCT output plugin

Applied. Thanks Holger.

Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
#3 lore raw

PATCH 2/15 common.h: added

In reply to [email protected]

Hi,
Content-Disposition: inline; filename=ulogd-add-common-code.diff

include/ulogd/common.h new file +50 -0
Index: ulogd-netfilter/include/ulogd/common.h
===================================================================
@@ -0,0 +1,50 @@
+/*
+ * common.h
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ * Holger Eitzenberger <[email protected]>  Astaro AG 2008
+ */
+#ifndef COMMON_H
+#define COMMON_H
+
+#include <stdlib.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <string.h>
+#include <errno.h>
+#include <assert.h>
+
+#define min(x, y) ({ \
+        typeof(x) _x = (x);  typeof(y) _y = (y); \
+        _x < _y ? _x : _y; })
+#define max(x, y) ({ \
+        typeof(x) _x = (x);  typeof(y) _y = (y); \
+        _x > _y ? _x : _y; })
+
+#define SEC		* 1
+#define MIN		* 60 SEC
+#define HOUR	* 60 MIN
+#define DAY		* 24 HOUR
+
+#define __unused				__attribute__((unused))
+#define __fmt_printf(i, first)	__attribute__((format (printtf,(i),(f))))
+
+#ifdef DEBUG
+#define pr_debug(fmt, ...)		ulogd_log(ULOGD_DEBUG, fmt, ## __VA_ARGS__)
+#else
+#define pr_debug(fmt, ...)
+#endif /* DEBUG */
+
+#endif /* COMMON_H */
#4 lore raw

Re: [ULOGD 02/15] common.h: added

Applied with minor glitches. Please, make sure your lines split at 80 chars. Added minor missing change in include/ulogd/Makefile.am. Thanks.

Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
#5 lore raw

PATCH 3/15 Replace timer code by working version

In reply to [email protected]

Hi,
Content-Disposition: inline; filename=ulogd-timer-handling.diff

Replace existing timer code by simple and more importantly working version. Current resolution is one second, which may be easily extended if need be.

include/ulogd/ulogd.h +18 -4
Index: ulogd-netfilter/include/ulogd/ulogd.h
===================================================================
@@ -240,17 +240,31 @@ void ulogd_unregister_fd(struct ulogd_fd
 int ulogd_select_main();
 
 /***********************************************************************
- * timer handling
+ * timer handling (timer.c)
  ***********************************************************************/
+#define TIMER_F_USED			0x0001
+#define TIMER_F_PERIODIC		0x0002
 
 struct ulogd_timer {
 	struct llist_head list;
-	struct timeval expires;
-	void (*cb)(void *data);
-	void *data;
+	unsigned expires;			/* seconds */
+	unsigned ival;				/* seconds */
+	unsigned flags;
+	void (* cb)(struct ulogd_timer *);
+	void *data;					/* usually (ulogd_pluginstance *) */
 };
 
+extern struct timeval tv_now;
+extern struct timeval tv_now_local;
+
+#define t_now			tv_now.tv_sec
+#define t_now_local		tv_now_local.tv_sec
+
+int ulogd_timer_init(void);
+int ulogd_timer_run(void);
 int ulogd_register_timer(struct ulogd_timer *timer);
 void ulogd_unregister_timer(struct ulogd_timer *timer);
+void ulogd_timer_schedule(void);
+int ulogd_timer_handle(void);
 
 #endif /* _ULOGD_H */
src/timer.c +74 -99

223 lines not on this page · raw message

#6 lore raw

Re: [ULOGD 03/15] Replace timer code by working version

Quoted 3 lines · [email protected] wrote:

Replace existing timer code by simple and more importantly working version. Current resolution is one second, which may be easily extended if need be.

I want to apply this patch since it fixes the current timer code. Although I don't like it since setitimer() wakes up the daemon every 1 seconds even if there is nothing to do but OK, we can fix this later.

Quoted 10 lines
(quoted diff) +1 -0
@@ -59,6 +60,7 @@
 {
 	struct ulogd_fd *ufd;
 	fd_set readset, writeset, exceptset;
+	struct timeval tv = { .tv_sec = 1, };
 	int i;
 
 	FD_ZERO(&readset);
@@ -77,7 +79,13 @@
 			FD_SET(ufd->fd, &exceptset);

Review line 88

Quoted 10 lines
(quoted diff) +7 -1
 	}
 
-	i = select(maxfd+1, &readset, &writeset, &exceptset, NULL);
+ again:
+	i = select(maxfd+1, &readset, &writeset, &exceptset, &tv);
+	if (i < 0) {
+		if (errno == EINTR)
+			goto again;
+	}
+

Hm, but why select also wakes up every 1 second? I'll continue applying your patches when I get a reply on this (I cannot apply anyway since the others depend on this).

BTW, this patch depends on 5/15. Please, your patches must compile without dependencies.

$ make
[...]
/home/pablo/SVN-netfilter/branches/ulog/ulogd2/src/ulogd.c:835:
undefined reference to `ulogd_timer_check_n_run'
Quoted 3 lines · Holger Heitzenberger

One of the changes herein is the usage of the pthread library. This is strictly necessary because some plugins might (and SQLITE3 *does*) use pthreads.

I have grep the source code looking for pthread stuff but I didn't find it, you mean that sqlite3 would need it in the future?

Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
#7 lore raw

PATCH 4/15 Add IFI list

In reply to [email protected]

Hi,
Content-Disposition: inline; filename=ulogd-ifi.diff

IFI implements a list of up-to-date network interfaces with MAC addresses, mostly used for logging and accounting. It is therefore quite redudant to the IFINDEX plugin, but IMO this should go into ulogd core and not into a plugin.

A future step might be to drop IFINDEX plugin.

include/ulogd/ifi.h new file +43 -0
Index: ulogd-netfilter/include/ulogd/ifi.h
===================================================================
@@ -0,0 +1,43 @@
+/*
+ * ifi.h
+ *
+ * Maintain a list of network interfaces.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ *
+ * Holger Eitzenberger <[email protected]>  Astaro AG, 2007.
+ */
+#ifndef IFI_H
+#define IFI_H
+
+#include <ulogd/linuxlist.h>
+#include <net/if.h>
+
+
+struct ifi {
+	struct llist_head link;
+	unsigned idx;			/* interface index */
+	unsigned used : 1;
+	unsigned flags;
+	char name[IFNAMSIZ];
+	unsigned char lladdr[6];
+};
+
+int ifi_init(void);
+void ifi_fini(void);
+
+struct ifi *ifi_find_by_idx(unsigned);
+
+#endif /* IFI_H */
src/ifi.c new file +379 -0

379 lines not on this page · raw message

src/Makefile.am +1 -1
Index: ulogd-netfilter/src/Makefile.am
===================================================================
@@ -5,5 +5,5 @@ AM_CPPFLAGS = $(all_includes) -I$(top_sr
 
 sbin_PROGRAMS = ulogd
 
-ulogd_SOURCES = ulogd.c select.c timer.c conffile.c
+ulogd_SOURCES = ulogd.c ifi.c select.c timer.c conffile.c
 ulogd_LDFLAGS = -export-dynamic
src/ulogd.c +6 -0
Index: ulogd-netfilter/src/ulogd.c
===================================================================
@@ -63,6 +63,9 @@
 #include <syslog.h>
 #include <ulogd/conffile.h>
 #include <ulogd/ulogd.h>
+#include <ulogd/ifi.h>
+
+
 #ifdef DEBUG
 #define DEBUGP(format, args...) fprintf(stderr, format, ## args)
 #else
@@ -983,6 +986,9 @@ int main(int argc, char* argv[])
 	ulogd_log(ULOGD_INFO, 
 		  "initialization finished, entering main loop\n");
 
+	if (ifi_init() < 0)
+		exit(EXIT_FAILURE);
+
 	ulogd_main_loop();
 
 	/* hackish, but result is the same */
#8 lore raw

Re: [ULOGD 04/15] Add IFI list

src/ifi.c

Quoted 4 lines · [email protected] wrote:

Index: ulogd-netfilter/src/ifi.c
===================================================================

src/ifi.c new file +0 -0

AFAICS, this file implements something similar to the ifname2index API which lives in libnfnetlink. Why don't you that instead? See the utils/iftest.c file under libnfnetlink for an example.

Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
#9 lore raw

Re: [ULOGD 04/15] Add IFI list

Quoted 3 lines · Pablo Neira Ayuso

AFAICS, this file implements something similar to the ifname2index API which lives in libnfnetlink. Why don't you that instead? See the utils/iftest.c file under libnfnetlink for an example.

The version of libnfnetlink I used initially did not have that, nor would I ever expect for libnfnetlink to provide that functionality.

So I would leave it as-is in order to be able to build ulogd against older librarires.

#10 lore raw

Re: [ULOGD 04/15] Add IFI list

Quoted 8 lines · Holger Eitzenberger
Quoted 3 lines · Pablo Neira Ayuso

AFAICS, this file implements something similar to the ifname2index API which lives in libnfnetlink. Why don't you that instead? See the utils/iftest.c file under libnfnetlink for an example.

The version of libnfnetlink I used initially did not have that, nor would I ever expect for libnfnetlink to provide that functionality.

libnfnetlink provides that since July 2007 because other libraries, such as libnetfilter_queue need such feature. There was a discussion way back and I decided to it there.

Quoted 2 lines

So I would leave it as-is in order to be able to build ulogd against older librarires.

Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
#11 lore raw

PATCH 5/15 Add signalling subsystem

In reply to [email protected]

Hi,
Content-Disposition: inline; filename=ulogd-fix-signal-handling.diff

This patch adds the concept of synchronous and asynchronous signal handlers to ulogd, where 'synchronous' just means to be synchronous to the underlying IO multiplexer.

Will later be used by plugins like SQLITE3 and NFCT.

One of the changes herein is the usage of the pthread library. This is strictly necessary because some plugins might (and SQLITE3 *does*) use pthreads.

src/Makefile.am +2 -2
Index: ulogd-netfilter/src/Makefile.am
===================================================================
@@ -5,5 +5,5 @@ AM_CPPFLAGS = $(all_includes) -I$(top_sr
 
 sbin_PROGRAMS = ulogd
 
-ulogd_SOURCES = ulogd.c ifi.c select.c timer.c conffile.c
-ulogd_LDFLAGS = -export-dynamic
+ulogd_SOURCES = ulogd.c ifi.c select.c timer.c signal.c conffile.c
+ulogd_LDFLAGS = -export-dynamic -lpthread
src/select.c +10 -1
Index: ulogd-netfilter/src/select.c
===================================================================
@@ -23,6 +23,7 @@
 
 #include <fcntl.h>
 #include <ulogd/ulogd.h>
+#include <ulogd/common.h>
 #include <ulogd/linuxlist.h>
 
 static int maxfd = 0;
@@ -59,6 +60,7 @@ int ulogd_select_main()
 {
 	struct ulogd_fd *ufd;
 	fd_set readset, writeset, exceptset;
+	struct timeval tv = { .tv_sec = 1, };
 	int i;
 
 	FD_ZERO(&readset);
@@ -77,7 +79,13 @@ int ulogd_select_main()
 			FD_SET(ufd->fd, &exceptset);
 	}
 
-	i = select(maxfd+1, &readset, &writeset, &exceptset, NULL);
+ again:
+	i = select(maxfd+1, &readset, &writeset, &exceptset, &tv);
+	if (i < 0) {
+		if (errno == EINTR)
+			goto again;
+	}
+
 	if (i > 0) {
 		/* call registered callback functions */
 		llist_for_each_entry(ufd, &ulogd_fds, list) {
@@ -96,5 +104,6 @@ int ulogd_select_main()
 				ufd->cb(ufd->fd, flags, ufd->data);
 		}
 	}
+
 	return i;
 }
src/ulogd.c +86 -48

232 lines not on this page · raw message

include/ulogd/signal.h new file +46 -0

46 lines not on this page · raw message

src/signal.c new file +193 -0

193 lines not on this page · raw message

#12 lore raw

Re: [ULOGD 05/15] Add signalling subsystem

Quoted 3 lines · [email protected] wrote:

This patch adds the concept of synchronous and asynchronous signal handlers to ulogd, where 'synchronous' just means to be synchronous to the underlying IO multiplexer.

I just committed a patch that reworks the timer framework to make it synchronous with select(). Since we don't have asynchronous timers anymore, I think that we can just block signaling during file descriptors handling since ulogd would receive not often. AFAICS, this infrastructure is nice but it was mainly targeted to the annoying SIGALRM.

Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
#13 lore raw

Re: [ULOGD 05/15] Add signalling subsystem

Quoted 9 lines · Pablo Neira Ayuso
Quoted 3 lines

This patch adds the concept of synchronous and asynchronous signal handlers to ulogd, where 'synchronous' just means to be synchronous to the underlying IO multiplexer.

I just committed a patch that reworks the timer framework to make it synchronous with select(). Since we don't have asynchronous timers anymore, I think that we can just block signaling during file descriptors handling since ulogd would receive not often. AFAICS, this infrastructure is nice but it was mainly targeted to the annoying SIGALRM.

Hi Pablo,

comparing your patch with the one I provided in my last patch collection I can say that your patch is quite larger due to the usage of the red-black trees in the timer code. The number of timers in ulogd is depending of your configuration of course, but with my current configuration (NFLOG, NFCT, SQLITE3) I have currently three timers, wow.

Note that with your patch you basically remove the possibility for plugins to have timers which are asynchronous. It's therefore less flexible for future users.

Also note that libraries such as libevent do it quite similar than provided in my patch.

My patch has the intention of providing a flexible infrastructure for plugins, which room for future improvements (such as red-black trees if there are hundreds of timers). Some of my later patches I have enqueued locally base on those changes, but that's my problem.

You commented on some of those patches, with a quite positive statement to my initial post of this patch. Also, Eric gave a GO on all patches despite the last NFCT patch, which I promised to rework for compatibility reason and based on your suggestions.

I accept the fact that you apparently like red-black trees and they definitely have their use-cases, but looking at typical ulogd configurations and numbers of timers in ulogd I can say that red-black trees just for timer usage seem to me like overkill.

Since you are in the habit of favoring your own patches against the ones I provide, even without giving a chance to modify my patch, I simply consider doing a fork of ulogd. Otherwise I'll loose many of the work I've enqueued locally.

Patrick?

/holger

Signature · 5 lines
Holger Eitzenberger <[email protected]> | Software Developer
Astaro AG | www.astaro.de | Phone +49-721-25516-246 | Fax -200

Download your free 30 day trial version of the award-winning Astaro
Security Gateway solution here: https://my.astaro.com/download/
#14 lore raw

Re: [ULOGD 05/15] Add signalling subsystem

Quoted 22 lines · Holger Eitzenberger
Quoted 9 lines · Pablo Neira Ayuso
Quoted 3 lines

This patch adds the concept of synchronous and asynchronous signal handlers to ulogd, where 'synchronous' just means to be synchronous to the underlying IO multiplexer.

I just committed a patch that reworks the timer framework to make it synchronous with select(). Since we don't have asynchronous timers anymore, I think that we can just block signaling during file descriptors handling since ulogd would receive not often. AFAICS, this infrastructure is nice but it was mainly targeted to the annoying SIGALRM.

Hi Pablo,

comparing your patch with the one I provided in my last patch collection I can say that your patch is quite larger due to the usage of the red-black trees in the timer code. The number of timers in ulogd is depending of your configuration of course, but with my current configuration (NFLOG, NFCT, SQLITE3) I have currently three timers, wow.

Note that with your patch you basically remove the possibility for plugins to have timers which are asynchronous. It's therefore less flexible for future users.

Also note that libraries such as libevent do it quite similar than provided in my patch.

My patch has the intention of providing a flexible infrastructure for plugins, which room for future improvements (such as red-black trees if there are hundreds of timers). Some of my later patches I have enqueued locally base on those changes, but that's my problem.

Quoted 7 lines

You commented on some of those patches, with a quite positive statement to my initial post of this patch. Also, Eric gave a GO on all patches despite the last NFCT patch, which I promised to rework for compatibility reason and based on your suggestions.

I accept the fact that you apparently like red-black trees and they definitely have their use-cases, but looking at typical ulogd configurations and numbers of timers in ulogd I can say that red-black trees just for timer usage seem to me like overkill.

Since you are in the habit of favoring your own patches against the ones I provide, even without giving a chance to modify my patch, I simply consider doing a fork of ulogd. Otherwise I'll loose many of the work I've enqueued locally.

Patrick?

I'm not sure I understand the problem fully, looking at both patches it seems to me that Pablo's patch could be extended to provide similar functionality to yours without much effort.

I agree that simply committing replacement patches without discussion when its known that further work depends on a patch is not a nice way of working together. I did not see any discussion related to this patch except the minor complaint about polling once per second in the first posting. Could someone explain the problem here please?

#15 lore raw

Re: [ULOGD 05/15] Add signalling subsystem

In reply to Holger Eitzenberger
Quoted 9 lines · Holger Eitzenberger

comparing your patch with the one I provided in my last patch collection I can say that your patch is quite larger due to the usage of the red-black trees in the timer code. The number of timers in ulogd is depending of your configuration of course, but with my current configuration (NFLOG, NFCT, SQLITE3) I have currently three timers, wow.

Note that with your patch you basically remove the possibility for plugins to have timers which are asynchronous. It's therefore less flexible for future users.

Please, could you tell me why my timer approach is less flexible?

Quoted 2 lines

Also note that libraries such as libevent do it quite similar than provided in my patch.

Indeed. This was the intention of my patch, to make ulogd event-driven and so timers synchronous using select() which simplifies the timer infrastructure instead of using SIGALRM.

Quoted 9 lines

My patch has the intention of providing a flexible infrastructure for plugins, which room for future improvements (such as red-black trees if
there are hundreds of timers). Some of my later patches I have
enqueued locally base on those changes, but that's my problem.

You commented on some of those patches, with a quite positive statement to my initial post of this patch. Also, Eric gave a GO on all patches despite the last NFCT patch, which I promised to rework for compatibility reason and based on your suggestions.

I have barely touched NFCT. I'm still waiting for your NFCT patch.

Quoted 4 lines

I accept the fact that you apparently like red-black trees and they definitely have their use-cases, but looking at typical ulogd configurations and numbers of timers in ulogd I can say that red-black trees just for timer usage seem to me like overkill.

We can easily implement list-based timers using the same API. You only have to use the current timer API in your patches.

Quoted 3 lines

Since you are in the habit of favoring your own patches against the ones I provide, even without giving a chance to modify my patch, I simply consider doing a fork of ulogd.

You still have the chance to modify your patches. I'm not disregarding your work.

Quoted 1 line

Otherwise I'll loose many of the work I've enqueued locally.

I have kept lots of patches locally in my whole life that I had rework lots of times to get them into mainline...

Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
#16 lore raw

PATCH 6/15 Conffile cleanup, use common pr_debug()

In reply to [email protected]

Hi,
Content-Disposition: inline; filename=ulogd-config-changes.diff

src/conffile.c +15 -20
Index: ulogd-netfilter/src/conffile.c
===================================================================
@@ -17,17 +17,10 @@
  *  along with this program; if not, write to the Free Software
  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
-
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
+#include <ulogd/ulogd.h>
+#include <ulogd/common.h>
 #include <ulogd/conffile.h>
 
-#ifdef DEBUG_CONF
-#define DEBUGC(format, args...) fprintf(stderr, format, ## args)
-#else
-#define DEBUGC(format, args...)
-#endif
 
 /* points to config entry with error */
 struct config_entry *config_errce = NULL;
@@ -101,6 +94,8 @@ int config_register_file(const char *fil
 	if (fname)
 		return 1;
 
+	pr_debug("%s: registered config file '%s'\n", __func__, file);
+
 	fname = (char *) malloc(strlen(file)+1);
 	if (!fname)
 		return -ERROOM;
@@ -121,12 +116,12 @@ int config_parse_file(const char *sectio
 	char linebuf[LINE_LEN+1];
 	char *line = linebuf;
 
+	pr_debug("%s: section='%s' file='%s'\n", __func__, section, fname);
+
 	cfile = fopen(fname, "r");
 	if (!cfile)
 		return -ERROPEN;
 
-	DEBUGC("parsing section [%s]\n", section);
-
 	/* Search for correct section */
 	while (fgets(line, LINE_LEN, cfile)) {
 		char wordbuf[LINE_LEN];
@@ -137,7 +132,7 @@ int config_parse_file(const char *sectio
 
 		if (!(wordend = get_word(line, " \t\n[]", (char *) wordbuf)))
 			continue;
-		DEBUGC("word: \"%s\"\n", wordbuf);
+		pr_debug("word: \"%s\"\n", wordbuf);
 		if (!strcmp(wordbuf, section)) {
 			found = 1;
 			break;
@@ -156,7 +151,7 @@ int config_parse_file(const char *sectio
 		char wordbuf[LINE_LEN];
 		char *wordend;
 		
-		DEBUGC("line read: %s\n", line);
+		pr_debug("line read: %s\n", line);
 		if (*line == '#')
 			continue;
 
@@ -164,14 +159,14 @@ int config_parse_file(const char *sectio
 			continue;
 
 		if (wordbuf[0] == '[' ) {
-			DEBUGC("Next section '%s' encountered\n", wordbuf);
+			pr_debug("Next section '%s' encountered\n", wordbuf);
 			break;
 		}
 
-		DEBUGC("parse_file: entering main loop\n");
+		pr_debug("parse_file: entering main loop\n");
 		for (i = 0; i < kset->num_ces; i++) {
 			struct config_entry *ce = &kset->ces[i];
-			DEBUGC("parse main loop, key: %s\n", ce->key);
+			pr_debug("parse main loop, key: %s\n", ce->key);
 			if (strcmp(ce->key, (char *) &wordbuf)) {
 				continue;
 			}
@@ -181,7 +176,7 @@ int config_parse_file(const char *sectio
 
 			if (ce->hit && !(ce->options & CONFIG_OPT_MULTI))
 			{
-				DEBUGC("->ce-hit and option not multi!\n");
+				pr_debug("->ce-hit and option not multi!\n");
 				config_errce = ce;
 				err = -ERRMULT;
 				goto cpf_error;
@@ -205,15 +200,15 @@ int config_parse_file(const char *sectio
 			}
 			break;
 		}
-		DEBUGC("parse_file: exiting main loop\n");
+		pr_debug("parse_file: exiting main loop\n");
 	}
 
 
 	for (i = 0; i < kset->num_ces; i++) {
 		struct config_entry *ce = &kset->ces[i];
-		DEBUGC("ce post loop, ce=%s\n", ce->key);
+		pr_debug("ce post loop, ce=%s\n", ce->key);
 		if ((ce->options & CONFIG_OPT_MANDATORY) && (ce->hit == 0)) {
-			DEBUGC("Mandatory config directive \"%s\" not found\n",
+			pr_debug("Mandatory config directive \"%s\" not found\n",
 				ce->key);
 			config_errce = ce;
 			err = -ERRMAND;
maintainer Applied
#17 lore raw

Re: [ULOGD 06/15] Conffile cleanup, use common pr_debug()

Applied. Thanks Holger.

Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
#18 lore raw

PATCH 7/15 Renice to -1 on startup

In reply to [email protected]

Hi,
Content-Disposition: inline; filename=ulogd-renice-on-startup.diff

Thus possibly preventing e.g. ctnetlink from overruns on busy sites.

TODO: make this conditional

src/ulogd.c +2 -0
Index: ulogd-netfilter/src/ulogd.c
===================================================================
@@ -1004,6 +1004,8 @@ main(int argc, char* argv[])
 		}
 	}
 
+	nice(-1);
+
 	if (daemonize){
 		if (fork()) {
 			exit(0);
#19 lore raw

Re: [ULOGD 07/15] Renice to -1 on startup

Quoted 3 lines · [email protected] wrote:

Thus possibly preventing e.g. ctnetlink from overruns on busy sites.

TODO: make this conditional

Indeed, a patch for that would be nice. Applied. Thanks.

Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
#20 lore raw

PATCH 8/15 Initial round to make plugins reconfigurable

In reply to [email protected]

Hi,
Content-Disposition: inline; filename=ulogd-allow-plugins-to-be-reconfigurable.diff

In order to make ulogd fully reconfigurable without restarting this patch does some introductory work letting plugins be able to be flagged as reconfigurable.

include/ulogd/ulogd.h +6 -0
Index: ulogd-netfilter/include/ulogd/ulogd.h
===================================================================
@@ -125,6 +125,10 @@ struct ulogd_keyset {
 
 struct ulogd_pluginstance_stack;
 struct ulogd_pluginstance;
+
+/* plugin flags */
+#define ULOGD_PF_RECONF			0x00000001
+
 struct ulogd_plugin {
 	/* global list of plugins */
 	struct llist_head list;
@@ -135,6 +139,8 @@ struct ulogd_plugin {
 	/* ID for this plugin (dynamically assigned) */
 	unsigned int id;
 
+	unsigned flags;
+
 	struct ulogd_keyset input;
 	struct ulogd_keyset output;
 
src/ulogd.c +81 -0

99 lines not on this page · raw message

#21 lore raw

PATCH 9/15 llist: add llist_for_each_prev_safe()

In reply to [email protected]

Hi,
Content-Disposition: inline; filename=ulogd-common-add-llist_for_each_prev_safe.diff

include/ulogd/linuxlist.h +10 -0
Index: ulogd-netfilter/include/ulogd/linuxlist.h
===================================================================
@@ -245,6 +245,16 @@ static inline void llist_splice_init(str
         	pos = pos->prev, prefetch(pos->prev))
         	
 /**
+ * llist_for_each_prev_safe - iterate over llist backwards safe against removal
+ * @pos:	the type * to use as a loop counter.
+ * @n:		another type * to use as temporary storage
+ * @head:	the head for your llist.
+ */
+#define llist_for_each_prev_safe(pos, n, head) \
+    for (pos = (head)->prev, n = pos->prev; pos != (head); \
+        pos = n, n = pos->prev)
+
+/**
  * llist_for_each_safe	-	iterate over a llist safe against removal of llist entry
  * @pos:	the &struct llist_head to use as a loop counter.
  * @n:		another &struct llist_head to use as temporary storage
#22 lore raw

PATCH 10/15 Improve select performance

In reply to [email protected]

Hi,
Content-Disposition: inline; filename=ulogd-select-performance-improvement.diff

The previous code consumed quite lots of CPU cycles because of inefficiently handling fd_sets.

This patch corrects that.

src/select.c +90 -51
Index: ulogd-netfilter/src/select.c
===================================================================
@@ -24,86 +24,125 @@
 #include <fcntl.h>
 #include <ulogd/ulogd.h>
 #include <ulogd/common.h>
+#include <ulogd/signal.h>
 #include <ulogd/linuxlist.h>
 
-static int maxfd = 0;
+static fd_set readset, writeset, exceptset;
+static int maxfd = -1;
 static LLIST_HEAD(ulogd_fds);
 
-int ulogd_register_fd(struct ulogd_fd *fd)
+
+static int
+set_nonblock(int fd)
 {
 	int flags;
 
-	/* make FD nonblocking */
-	flags = fcntl(fd->fd, F_GETFL);
-	if (flags < 0)
+	if ((flags = fcntl(fd, F_GETFL)) < 0)
 		return -1;
+
 	flags |= O_NONBLOCK;
-	flags = fcntl(fd->fd, F_SETFL, flags);
-	if (flags < 0)
+
+	if ((flags = fcntl(fd, F_SETFL, flags)) < 0)
+		return -1;
+
+	return 0;
+}
+
+int
+ulogd_register_fd(struct ulogd_fd *ufd)
+{
+	if (set_nonblock(ufd->fd) < 0)
 		return -1;
 
-	/* Register FD */
-	if (fd->fd > maxfd)
-		maxfd = fd->fd;
+	if (ufd->when & ULOGD_FD_READ)
+		FD_SET(ufd->fd, &readset);
+	
+	if (ufd->when & ULOGD_FD_WRITE)
+		FD_SET(ufd->fd, &writeset);
+	
+	if (ufd->when & ULOGD_FD_EXCEPT)
+		FD_SET(ufd->fd, &exceptset);
+
+	if (ufd->fd > maxfd)
+		maxfd = ufd->fd;
 
-	llist_add_tail(&fd->list, &ulogd_fds);
+	llist_add_tail(&ufd->list, &ulogd_fds);
 
 	return 0;
 }
 
-void ulogd_unregister_fd(struct ulogd_fd *fd)
+void
+ulogd_unregister_fd(struct ulogd_fd *ufd)
 {
-	llist_del(&fd->list);
+	if (ufd->when & ULOGD_FD_READ)
+		FD_CLR(ufd->fd, &readset);
+	
+	if (ufd->when & ULOGD_FD_WRITE)
+		FD_CLR(ufd->fd, &writeset);
+	
+	if (ufd->when & ULOGD_FD_EXCEPT)
+		FD_CLR(ufd->fd, &exceptset);
+
+	llist_del(&ufd->list);
+
+	maxfd = -1;
+	llist_for_each_entry(ufd, &ulogd_fds, list) {
+		assert(ufd->fd >= 0);
+
+		if (ufd->fd > maxfd)
+			maxfd = ufd->fd;
+	}
 }
 
-int ulogd_select_main()
+/* ulogd_dispatch() - dispatch events */
+int
+ulogd_dispatch(void)
 {
-	struct ulogd_fd *ufd;
-	fd_set readset, writeset, exceptset;
-	struct timeval tv = { .tv_sec = 1, };
-	int i;
-
-	FD_ZERO(&readset);
-	FD_ZERO(&writeset);
-	FD_ZERO(&exceptset);
+	fd_set rds_tmp, wrs_tmp, exs_tmp;
+	sigset_t curr_ss;
 
-	/* prepare read and write fdsets */
-	llist_for_each_entry(ufd, &ulogd_fds, list) {
-		if (ufd->when & ULOGD_FD_READ)
-			FD_SET(ufd->fd, &readset);
+	ulogd_get_sigset(&curr_ss);
 
-		if (ufd->when & ULOGD_FD_WRITE)
-			FD_SET(ufd->fd, &writeset);
+	pthread_sigmask(SIG_UNBLOCK, &curr_ss, NULL);
 
-		if (ufd->when & ULOGD_FD_EXCEPT)
-			FD_SET(ufd->fd, &exceptset);
-	}
+	for (;;) {
+		struct ulogd_fd *ufd;
+		int n;
 
- again:
-	i = select(maxfd+1, &readset, &writeset, &exceptset, &tv);
-	if (i < 0) {
-		if (errno == EINTR)
-			goto again;
-	}
+	again:
+		rds_tmp = readset;
+		wrs_tmp = writeset;
+		exs_tmp = exceptset;
 
-	if (i > 0) {
-		/* call registered callback functions */
-		llist_for_each_entry(ufd, &ulogd_fds, list) {
-			int flags = 0;
-
-			if (FD_ISSET(ufd->fd, &readset))
-				flags |= ULOGD_FD_READ;
+		n = select(maxfd+1, &rds_tmp, &wrs_tmp, &exs_tmp, NULL);
+		if (n < 0) {
+			if (errno == EINTR)
+				goto again;
 
-			if (FD_ISSET(ufd->fd, &writeset))
-				flags |= ULOGD_FD_WRITE;
+			ulogd_log(ULOGD_FATAL, "select: %m\n");
 
-			if (FD_ISSET(ufd->fd, &exceptset))
-				flags |= ULOGD_FD_EXCEPT;
+			break;
+		}
 
-			if (flags)
-				ufd->cb(ufd->fd, flags, ufd->data);
+		if (n > 0) {
+			/* call registered callback functions */
+			llist_for_each_entry(ufd, &ulogd_fds, list) {
+				int flags = 0;
+
+				if (FD_ISSET(ufd->fd, &rds_tmp))
+					flags |= ULOGD_FD_READ;
+
+				if (FD_ISSET(ufd->fd, &wrs_tmp))
+					flags |= ULOGD_FD_WRITE;
+
+				if (FD_ISSET(ufd->fd, &exs_tmp))
+					flags |= ULOGD_FD_EXCEPT;
+
+				if (flags)
+					ufd->cb(ufd->fd, flags, ufd->data);
+			}
 		}
 	}
 
-	return i;
+	return 0;
 }
src/ulogd.c +1 -27
Index: ulogd-netfilter/src/ulogd.c
===================================================================
@@ -727,32 +727,6 @@ out_buf:
 	return ret;
 }
 	
-
-static int ulogd_main_loop(void)
-{
-	sigset_t curr;
-	int ret = 0;
-
-	ulogd_get_sigset(&curr);
-
-	pthread_sigmask(SIG_UNBLOCK, &curr, NULL);
-
-	for (;;) {
-		ret = ulogd_select_main();
-		if (ret == 0) 
-			continue;
-
-		if (ret < 0) {
-			ulogd_log(ULOGD_ERROR, "select returned %s\n",
-					  strerror(errno));
-			break;
-		}
-	}
-
-	return ret;
-}
-
-/* open the logfile */
 static int logfile_open(const char *name)
 {
 	if (name)
@@ -1112,7 +1086,7 @@ main(int argc, char* argv[])
 
 	ulogd_log(ULOGD_INFO, "entering main loop\n");
 
-	ulogd_main_loop();
+	ulogd_dispatch();
 
 	return 0;
 }
include/ulogd/ulogd.h +1 -1
Index: ulogd-netfilter/include/ulogd/ulogd.h
===================================================================
@@ -243,7 +243,7 @@ struct ulogd_fd {
 
 int ulogd_register_fd(struct ulogd_fd *ufd);
 void ulogd_unregister_fd(struct ulogd_fd *ufd);
-int ulogd_select_main();
+int ulogd_dispatch(void);
 
 /***********************************************************************
  * timer handling (timer.c)
#23 lore raw

Re: [ULOGD 10/15] Improve select performance

Quoted 2 lines · [email protected] wrote:

The previous code consumed quite lots of CPU cycles because of inefficiently handling fd_sets.

I have applied the following patch which is based on your improvement.

I'll check the other patches that are targeted to make plugins reconfigurable as soon as I have some spare time, whatever help in the merge process with current SVN would be appreciated and would surely speed up the development. Thanks.

Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
select.c +34 -20
Index: src/select.c
===================================================================
@@ -26,6 +26,7 @@
 #include <ulogd/linuxlist.h>
 
 static int maxfd = 0;
+static fd_set readset, writeset, exceptset;
 static LLIST_HEAD(ulogd_fds);
 
 int ulogd_register_fd(struct ulogd_fd *fd)
@@ -41,6 +42,15 @@
 	if (flags < 0)
 		return -1;
 
+	if (fd->when & ULOGD_FD_READ)
+		FD_SET(fd->fd, &readset);
+
+	if (fd->when & ULOGD_FD_WRITE)
+		FD_SET(fd->fd, &writeset);
+
+	if (fd->when & ULOGD_FD_EXCEPT)
+		FD_SET(fd->fd, &exceptset);
+
 	/* Register FD */
 	if (fd->fd > maxfd)
 		maxfd = fd->fd;
@@ -52,44 +62,48 @@
 
 void ulogd_unregister_fd(struct ulogd_fd *fd)
 {
+	if (fd->when & ULOGD_FD_READ)
+		FD_CLR(fd->fd, &readset);
+
+	if (fd->when & ULOGD_FD_WRITE)
+		FD_CLR(fd->fd, &writeset);
+
+	if (fd->when & ULOGD_FD_EXCEPT)
+		FD_CLR(fd->fd, &exceptset);
+
 	llist_del(&fd->list);
+
+	/* Improvement: recalculate maxfd iif fd->fd == maxfd */
+	maxfd = -1;
+	llist_for_each_entry(fd, &ulogd_fds, list) {
+		if (fd->fd > maxfd)
+			maxfd = fd->fd;
+	}
 }
 
 int ulogd_select_main(struct timeval *tv)
 {
 	struct ulogd_fd *ufd;
-	fd_set readset, writeset, exceptset;
+	fd_set rds_tmp, wrs_tmp, exs_tmp;
 	int i;
 
-	FD_ZERO(&readset);
-	FD_ZERO(&writeset);
-	FD_ZERO(&exceptset);
+	rds_tmp = readset;
+	wrs_tmp = writeset;
+	exs_tmp = exceptset;
 
-	/* prepare read and write fdsets */
-	llist_for_each_entry(ufd, &ulogd_fds, list) {
-		if (ufd->when & ULOGD_FD_READ)
-			FD_SET(ufd->fd, &readset);
-
-		if (ufd->when & ULOGD_FD_WRITE)
-			FD_SET(ufd->fd, &writeset);
-
-		if (ufd->when & ULOGD_FD_EXCEPT)
-			FD_SET(ufd->fd, &exceptset);
-	}
-
-	i = select(maxfd+1, &readset, &writeset, &exceptset, tv);
+	i = select(maxfd+1, &rds_tmp, &wrs_tmp, &exs_tmp, tv);
 	if (i > 0) {
 		/* call registered callback functions */
 		llist_for_each_entry(ufd, &ulogd_fds, list) {
 			int flags = 0;
 
-			if (FD_ISSET(ufd->fd, &readset))
+			if (FD_ISSET(ufd->fd, &rds_tmp))
 				flags |= ULOGD_FD_READ;
 
-			if (FD_ISSET(ufd->fd, &writeset))
+			if (FD_ISSET(ufd->fd, &wrs_tmp))
 				flags |= ULOGD_FD_WRITE;
 
-			if (FD_ISSET(ufd->fd, &exceptset))
+			if (FD_ISSET(ufd->fd, &exs_tmp))
 				flags |= ULOGD_FD_EXCEPT;
 
 			if (flags)
#24 lore raw

PATCH 11/15 Add set_sockbuf_len()

In reply to [email protected]

Hi,
Content-Disposition: inline; filename=ulogd-add-set_sockbuf_len.diff

include/ulogd/common.h +2 -0
Index: ulogd-netfilter/include/ulogd/common.h
===================================================================
@@ -47,4 +47,6 @@
 #define pr_debug(fmt, ...)
 #endif /* DEBUG */
 
+int set_sockbuf_len(int fd, int rcv_len, int snd_len);
+
 #endif /* COMMON_H */
src/common.c new file +52 -0

52 lines not on this page · raw message

src/Makefile.am +1 -1
Index: ulogd-netfilter/src/Makefile.am
===================================================================
@@ -5,5 +5,5 @@ AM_CPPFLAGS = $(all_includes) -I$(top_sr
 
 sbin_PROGRAMS = ulogd
 
-ulogd_SOURCES = ulogd.c ifi.c select.c timer.c signal.c conffile.c
+ulogd_SOURCES = ulogd.c ifi.c select.c timer.c signal.c common.c conffile.c
 ulogd_LDFLAGS = -export-dynamic -lpthread
#25 lore raw

Re: [ULOGD 11/15] Add set_sockbuf_len()

Quoted 7 lines · [email protected] wrote:
src/common.c new file +7 -0
+	if (snd_len > 0) {
+		ret = setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &snd_len,
+						 sizeof(snd_len));
+		if (ret < 0) {
+			ulogd_log(ULOGD_ERROR, "setsockopt: SO_SNDBUF: %m\n");
+			return -1;
+		}

src/common.c line 49

Quoted 10 lines
src/common.c new file +10 -0
+	}
+
+	if (rcv_len > 0) {
+		ret = setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &rcv_len,
+						 sizeof(rcv_len));
+		if (ret < 0) {
+			ulogd_log(ULOGD_ERROR, "setsockopt: SO_RCVBUF: %m\n");
+			return -1;
+		}
+	}

I guess that this is meant to increase the size of the netlink buffers, isn't this? However, where's the client of this code?

Please, next time put the new code with its client so I can easily notice what you want to do by looking at a single patch, otherwise I'll have to look at the entire patchset to search for its client. A short rationale on the benefits of the patch would also help.

Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
#26 lore raw

PATCH 12/15 Introduce global state, skip some stacks during reconfiguration

In reply to [email protected]

Hi,
Content-Disposition: inline; filename=ulogd-fix-races-during-reconf.diff

Some plugins are not reconfigurable, therefore skips entire stacks if there is at least one plugin which is not reconfigurable.

Also introduce a global state to counter some pathological reconfiguration issues.

src/ulogd.c +80 -21
Index: ulogd-netfilter/src/ulogd.c
===================================================================
@@ -83,6 +83,7 @@ static FILE *logfile = NULL;		/* logfile
 static char *ulogd_configfile = ULOGD_CONFIGFILE;
 static char *ulogd_logfile = ULOGD_LOGFILE_DEFAULT;
 static FILE syslog_dummy;
+static enum GlobalState state;
 
 /* linked list for all registered plugins */
 static LLIST_HEAD(ulogd_plugins);
@@ -129,6 +130,19 @@ static struct config_keyset ulogd_kset =
 #define loglevel_ce	ulogd_kset.ces[2]
 #define stack_ce	ulogd_kset.ces[3]
 
+
+void
+ulogd_set_state(enum GlobalState s)
+{
+	state = s;
+}
+
+enum GlobalState
+ulogd_get_state(void)
+{
+	return state;
+}
+
 /***********************************************************************
  * UTILITY FUNCTIONS FOR PLUGINS
  ***********************************************************************/
@@ -195,8 +209,8 @@ int ulogd_wildcard_inputkeys(struct ulog
 
 	/* first pass: count keys */
 	llist_for_each_entry(pi_cur, &stack->list, list) {
-		ulogd_log(ULOGD_DEBUG, "iterating over pluginstance '%s'\n",
-			  pi_cur->id);
+		ulogd_log(ULOGD_DEBUG, "%s: iterating over pluginstance '%s'\n",
+				  upi->id, pi_cur->id);
 		num_keys += pi_cur->plugin->output.num_keys;
 	}
 
@@ -209,8 +223,12 @@ int ulogd_wildcard_inputkeys(struct ulog
 	llist_for_each_entry(pi_cur, &stack->list, list) {
 		int i;
 
-		for (i = 0; i < pi_cur->plugin->output.num_keys; i++)
+		for (i = 0; i < pi_cur->plugin->output.num_keys; i++) {
+			pr_debug("%s: copy key '%s' from plugin '%s'\n", upi->id,
+					 pi_cur->plugin->output.keys[i].name, pi_cur->id);
+
 			upi->input.keys[index++] = pi_cur->output.keys[i];
+		}
 	}
 
 	upi->input.num_keys = num_keys;
@@ -650,6 +668,10 @@ static int create_stack(const char *opti
 	}
 	INIT_LLIST_HEAD(&stack->list);
 
+	/* By default a stack is reconfigurable unless there is a plugin
+	   which is not reconfigurable */
+	stack->flags = ULOGD_SF_RECONF;
+
 	ulogd_log(ULOGD_DEBUG, "building new pluginstance stack (%s):\n",
 		  option);
 
@@ -698,6 +720,9 @@ static int create_stack(const char *opti
 			
 		ulogd_log(ULOGD_DEBUG, "pushing `%s' on stack\n", pl->name);
 		llist_add_tail(&pi->list, &stack->list);
+
+		if ((pi->plugin->flags & ULOGD_PF_RECONF) == 0)
+			stack->flags &= ~ULOGD_SF_RECONF;
 	}
 
 	/* PASS 2: resolve key connections from bottom to top of stack */
@@ -788,10 +813,20 @@ static int parse_conffile(const char *se
 	return 1;
 }
 
+/**
+ * Call callback on each ulogd_pluginstance.
+ * @arg cb      Callback to call for each ulogd_pluginstance.
+ * @arg arg     Argument to pass to each call to argument.
+ *
+ * Call callback on each ulogd_pluginstance, skipping stacks which are
+ * not reconfigurable.
+ *
+ * @return Number of calls.
+ */
 static int
 for_each_pluginstance(int (* cb)(struct ulogd_pluginstance *,
 								 struct ulogd_pluginstance_stack *,
-								 void *), void *arg)
+								 unsigned long), unsigned long arg)
 {
 	struct ulogd_pluginstance_stack *stack;
 	int sum = 0;
@@ -801,6 +836,9 @@ for_each_pluginstance(int (* cb)(struct 
 	llist_for_each_entry(stack, &ulogd_pi_stacks, stack_list) {
 		struct ulogd_pluginstance *pi;
 
+		if ((stack->flags & ULOGD_SF_RECONF) == 0)
+			continue;
+
 		llist_for_each_entry(pi, &stack->list, list) {
 			int ret;
 
@@ -824,14 +862,15 @@ enum ReconfOp
 
 static int
 _do_reconf(struct ulogd_pluginstance *pi,
-		   struct ulogd_pluginstance_stack *stack, void *arg)
+		   struct ulogd_pluginstance_stack *stack, unsigned long arg)
 {
-	enum ReconfOp op = (unsigned)arg;
+	enum ReconfOp op = arg;
 	int ret = 0;
 
 	assert(pi != NULL);
 
-	if ((pi->plugin->flags & ULOGD_PF_RECONF) == 0)
+	/* check whether stack is reconfigurable */
+	if ((stack->flags & ULOGD_SF_RECONF) == 0)
 		return 0;
 
 	switch (op) {
@@ -839,16 +878,12 @@ _do_reconf(struct ulogd_pluginstance *pi
 		ret = pi->plugin->stop(pi);
 		break;
 
-	case CONFIGURE:
-		ulogd_pluginstance_reset_cfg(pi);
-		ret = pi->plugin->configure(pi, stack);
-		break;
-
 	case START:
 		ret = pi->plugin->start(pi);
 		break;
 
 	default:
+		ulogd_log(ULOGD_ERROR, "%s: invalid op '%d'\n", __func__, op);
 		return -1;
 	}
 
@@ -863,25 +898,43 @@ _do_reconf(struct ulogd_pluginstance *pi
 static int
 reconfigure_plugins(void)
 {
+	struct ulogd_pluginstance_stack *stack;
+
 	ulogd_log(ULOGD_INFO, "reconfiguring plugins\n");
 
-	if (for_each_pluginstance(_do_reconf, (void *)STOP) < 0)
-		abort();
+	ulogd_set_state(GS_INITIALIZING);
 
-	if (for_each_pluginstance(_do_reconf, (void *)CONFIGURE) < 0)
-		abort();
+	/* Skip stacks which contain at least one plugin which is not
+	   reconfigurable.  This a workaround until all plugins are
+	   reconfigurable. */
+	if (for_each_pluginstance(_do_reconf, STOP) < 0)
+		goto err;
 
-	if (for_each_pluginstance(_do_reconf, (void *)START) < 0)
-		abort();
+	llist_for_each_entry(stack, &ulogd_pi_stacks, stack_list) {
+		if ((stack->flags & ULOGD_SF_RECONF) == 0)
+			continue;
+
+		if (create_stack_resolve_keys(stack) < 0)
+			goto err;
+	}
+
+	if (for_each_pluginstance(_do_reconf, START) < 0)
+		goto err;
+
+	ulogd_set_state(GS_RUNNING);
 
 	return 0;
+
+ err:
+ 	ulogd_log(ULOGD_FATAL, "failed to reconfigure\n");
+ 	return -1;
 }
 
 static int
 _do_signal(struct ulogd_pluginstance *pi,
-		   struct ulogd_pluginstance_stack *stack, void *arg)
+		   struct ulogd_pluginstance_stack *stack, unsigned long arg)
 {
-	int signo = (int) arg;
+	int signo = (int)arg;
 
 	if (pi->plugin->signal) {
 		pi->plugin->signal(pi, signo);
@@ -897,6 +950,9 @@ sync_sig_handler(int signo)
 {
 	pr_debug("%s: signal '%d' received\n", __func__, signo);
 
+	if (ulogd_get_state() != GS_RUNNING)
+		return;
+
 	switch (signo) {
 	case SIGHUP:
 		reconfigure_plugins();
@@ -913,7 +969,7 @@ sync_sig_handler(int signo)
 		break;
 	}
 
-	for_each_pluginstance(_do_signal, (void *) signo);
+	for_each_pluginstance(_do_signal, (unsigned long)signo);
 }
 
 static void
@@ -965,6 +1021,7 @@ main(int argc, char* argv[])
 	uid_t uid = 0;
 	gid_t gid = 0;
 
+	ulogd_set_state(GS_INITIALIZING);
 
 	while ((argch = getopt_long(argc, argv, "c:dh::Vu:", opts, NULL)) != -1) {
 		switch (argch) {
@@ -1086,6 +1143,8 @@ main(int argc, char* argv[])
 
 	ulogd_log(ULOGD_INFO, "entering main loop\n");
 
+	ulogd_set_state(GS_RUNNING);
+
 	ulogd_dispatch();
 
 	return 0;
include/ulogd/ulogd.h +12 -0
Index: ulogd-netfilter/include/ulogd/ulogd.h
===================================================================
@@ -189,17 +189,29 @@ struct ulogd_pluginstance {
 	char private[0];
 };
 
+/* stack flags */
+#define ULOGD_SF_RECONF			0x00000001 /* stack is reconfigurable */
+
 struct ulogd_pluginstance_stack {
 	/* global list of pluginstance stacks */
 	struct llist_head stack_list;
 	/* list of plugins in this stack */
 	struct llist_head list;
 	char *name;
+	unsigned flags;
 };
 
 /***********************************************************************
  * PUBLIC INTERFACE 
  ***********************************************************************/
+enum GlobalState {
+	GS_INVAL = 0,
+	GS_INITIALIZING,			/* also reconfiguration */
+	GS_RUNNING,
+};
+
+void ulogd_set_state(enum GlobalState);
+enum GlobalState ulogd_get_state(void);
 
 void ulogd_propagate_results(struct ulogd_pluginstance *pi);
 
#27 lore raw

PATCH 13/15 llist: turn poisoning off by default

In reply to [email protected]

Hi,
Content-Disposition: inline; filename=ulogd-llist-poison-debug-only.diff

include/ulogd/linuxlist.h +2 -0
Index: ulogd-netfilter/include/ulogd/linuxlist.h
===================================================================
@@ -117,8 +117,10 @@ static inline void __llist_del(struct ll
 static inline void llist_del(struct llist_head *entry)
 {
 	__llist_del(entry->prev, entry->next);
+#ifdef ULOGD_LLIST_DEBUG
 	entry->next = LLIST_POISON1;
 	entry->prev = LLIST_POISON2;
+#endif /* ULOGD_LLIST_DEBUG */
 }
 
 /**
#28 lore raw

PATCH 14/15 SQLITE3: port to ulogd 2.00, mostly a rewrite

In reply to [email protected]

Hi,
Content-Disposition: inline; filename=ulogd-SQLITE3-plugin.diff

NOTE

As described in

http://www.sqlite.org/cvstrac/wiki?p=CorruptionFollowingBusyError

some versions of SQLITE contain an error when transactions other than EXCLUSIVE are used. If a SQL commands to create a transaction or inside a transaction ever gets SQLITE_BUSY a ROLLBACK has to be done, as otherwise a corrupt database might be the result.

output/sqlite3/ulogd_output_SQLITE3.c +683 -268
Index: ulogd-netfilter-stuffed/output/sqlite3/ulogd_output_SQLITE3.c
===================================================================
@@ -1,4 +1,3 @@
-#if 0
 /*
  * ulogd output plugin for logging to a SQLITE database
  *
@@ -26,389 +25,805 @@
  *
  *  2005-02-09 Harald Welte <[email protected]>:
  *  	- port to ulogd-1.20 
+ *
+ *  Holger Eitzenberger <[email protected]>  Astaro AG, 2007
+ *  	- port to ulogd-2.00
  */
 
-#include <stdlib.h>
-#include <string.h>
-#include <arpa/inet.h>
 #include <ulogd/ulogd.h>
 #include <ulogd/conffile.h>
+#include <ulogd/common.h>
+#include <ulogd/linuxlist.h>
+#include <arpa/inet.h>
 #include <sqlite3.h>
 
-#ifdef DEBUG_SQLITE3
-#define DEBUGP(x, args...)	fprintf(stderr, x, ## args)
-#else
-#define DEBUGP(x, args...)
-#endif
+#define PFX		"SQLITE3: "
+
+/* config defaults */
+#define CFG_BUFFER_DEFAULT		100
+#define CFG_TIMER_DEFAULT		1 SEC
+#define CFG_MAX_BACKLOG_DEFAULT	0		/* unlimited */
+
+
+#define SQLITE3_BUSY_TIMEOUT 300
+
+/* number of colums we have (really should be configurable) */
+#define DB_NUM_COLS	10
 
-struct _field {
+
+/* map DB column to ulogd key */
+struct col {
 	char name[ULOGD_MAX_KEYLEN];
-	unsigned int id;
-	struct _field *next;
+	struct ulogd_key *key;
+};
+
+struct row {
+	struct llist_head link;
+	uint32_t ip_saddr;
+	uint32_t ip_daddr;
+	unsigned char ip_proto;
+	unsigned l4_dport;
+	unsigned raw_in_pktlen;
+	unsigned raw_in_pktcount;
+	unsigned raw_out_pktlen;
+	unsigned raw_out_pktcount;
+	unsigned flow_start_sec;
+	unsigned flow_duration;
 };
 
-/* the database handle we are using */
-static sqlite3 *dbh;
+#define RKEY(key)	((key)->u.source)
 
-/* a linked list of the fields the table has */
-static struct _field *fields;
 
-/* buffer for our insert statement */
-static char *stmt;
+struct sqlite3_priv {
+	sqlite3 *dbh;				/* database handle we are using */
+	char *stmt;
+	sqlite3_stmt *p_stmt;
+	int buffer_size;
 
-/* pointer to the final prepared statement */
-static sqlite3_stmt *p_stmt;
+	struct ulogd_timer timer;
 
-/* number of statements to buffer before we commit */
-static int buffer_size;
+	struct col cols[DB_NUM_COLS];
 
-/* number of statements currently in the buffer */
-static int buffer_ctr;
+	/* our backlog buffer */
+	struct llist_head rows;
+	int num_rows;
+	int max_backlog;
 
-/* our configuration directives */
-static config_entry_t db_ce = { 
-	.key = "db", 
-	.type = CONFIG_TYPE_STRING,
-	.options = CONFIG_OPT_MANDATORY,
-};
+	time_t commit_time;
 
-static config_entry_t table_ce = { 
-	.next = &db_ce, 
-	.key = "table",
-	.type = CONFIG_TYPE_STRING,
-	.options = CONFIG_OPT_MANDATORY,
+	unsigned disable : 1;
+	unsigned overlimit_msg : 1;
 };
 
-static config_entry_t buffer_ce = { 
-	.next = &table_ce,
-	.key = "buffer",
-	.type = CONFIG_TYPE_INT,
-	.options = CONFIG_OPT_MANDATORY,
+
+static struct config_keyset sqlite3_kset = {
+	.num_ces = 6,
+	.ces = {
+		{
+			.key = "db",
+			.type = CONFIG_TYPE_STRING,
+			.options = CONFIG_OPT_MANDATORY,
+		},
+		{
+			.key = "table",
+			.type = CONFIG_TYPE_STRING,
+			.options = CONFIG_OPT_MANDATORY,
+		},
+		{
+			.key = "buffer",
+			.type = CONFIG_TYPE_INT,
+			.options = CONFIG_OPT_NONE,
+			.u.value = CFG_BUFFER_DEFAULT,
+		},
+		{
+			.key = "timer",
+			.type = CONFIG_TYPE_INT,
+			.options = CONFIG_OPT_NONE,
+			.u.value = CFG_TIMER_DEFAULT,
+		},
+		{
+			.key = "max-backlog",
+			.type = CONFIG_TYPE_INT,
+			.options = CONFIG_OPT_NONE,
+			.u.value = CFG_MAX_BACKLOG_DEFAULT,
+		},
+		{
+			.key = "disable",
+			.type = CONFIG_TYPE_INT,
+			.options = CONFIG_OPT_NONE,
+			.u.value = 0,
+		},
+	},
 };
 
-/* our main output function, called by ulogd */
-static int _sqlite3_output(ulog_iret_t *result)
+#define db_ce(pi)		(pi)->config_kset->ces[0].u.string
+#define table_ce(pi)	(pi)->config_kset->ces[1].u.string
+#define buffer_ce(pi)	(pi)->config_kset->ces[2].u.value
+#define timer_ce(pi)	(pi)->config_kset->ces[3].u.value
+#define max_backlog_ce(pi)	(pi)->config_kset->ces[4].u.value
+#define disable_ce(pi)	(pi)->config_kset->ces[5].u.value
+
+
+#define SQL_CREATE_STR \
+		"create table daily(ip_saddr integer, ip_daddr integer, " \
+		"ip_protocol integer, l4_dport integer, raw_in_pktlen integer, " \
+		"raw_in_pktcount integer, raw_out_pktlen integer, " \
+		"raw_out_pktcount integer, flow_start_sec integer, " \
+		"flow_duration integer)"
+
+
+static struct row *
+row_new(void)
 {
-	struct _field *f;
-	ulog_iret_t *res;
-	int col_counter;
-#ifdef IP_AS_STRING
-	char *ipaddr;
-	struct in_addr addr;
-#endif
-
-	col_counter = 1;
-	for (f = fields; f; f = f->next) {
-		res = keyh_getres(f->id);
-
-		if (!res) {
-			ulogd_log(ULOGD_NOTICE,
-				"no result for %s ?!?\n", f->name);
-		}
-			
-		if (!res || !IS_VALID((*res))) {
-			/* no result, pass a null */
-			sqlite3_bind_null(p_stmt, col_counter);
-			col_counter++;
-			continue;
-		}
-		
-		switch (res->type) {
-			case ULOGD_RET_INT8:
-				sqlite3_bind_int(p_stmt,col_counter,res->value.i8);
-				break;
-			case ULOGD_RET_INT16:
-				sqlite3_bind_int(p_stmt,col_counter,res->value.i16);
-				break;
-			case ULOGD_RET_INT32:
-				sqlite3_bind_int(p_stmt,col_counter,res->value.i32);
-				break;
-			case ULOGD_RET_INT64:
-				sqlite3_bind_int64(p_stmt,col_counter,res->value.i64);
-				break;
-			case ULOGD_RET_UINT8:
-				sqlite3_bind_int(p_stmt,col_counter,res->value.ui8);
-				break;
-			case ULOGD_RET_UINT16:
-				sqlite3_bind_int(p_stmt,col_counter,res->value.ui16);
-				break;
-			case ULOGD_RET_IPADDR:
-#ifdef IP_AS_STRING
-				memset(&addr, 0, sizeof(addr));
-				addr.s_addr = ntohl(res->value.ui32);
-				ipaddr = inet_ntoa(addr);
-				sqlite3_bind_text(p_stmt,col_counter,ipaddr,strlen(ipaddr),SQLITE_STATIC);
-                                break;
-#endif /* IP_AS_STRING */
-			/* EVIL: fallthrough when logging IP as u_int32_t */
-			case ULOGD_RET_UINT32:
-				sqlite3_bind_int(p_stmt,col_counter,res->value.ui32);
-				break;
-			case ULOGD_RET_UINT64:
-				sqlite3_bind_int64(p_stmt,col_counter,res->value.ui64);
-				break;
-			case ULOGD_RET_BOOL:
-				sqlite3_bind_int(p_stmt,col_counter,res->value.b);
-				break;
-			case ULOGD_RET_STRING:
-				sqlite3_bind_text(p_stmt,col_counter,res->value.ptr,strlen(res->value.ptr),SQLITE_STATIC);
-				break;
-			default:
-				ulogd_log(ULOGD_NOTICE,
-					"unknown type %d for %s\n",
-					res->type, res->key);
-				break;
-		} 
-
-		col_counter++;
-	}
-
-	/* now we have created our statement, insert it */
-
-	if (sqlite3_step(p_stmt) == SQLITE_DONE) {
-		sqlite3_reset(p_stmt);
-		buffer_ctr++;
-	} else {
-		ulogd_log(ULOGD_ERROR, "sql error during insert: %s\n",
-				sqlite3_errmsg(dbh));
-		return 1;
-	}
+	struct row *row;
+
+	if ((row = calloc(1, sizeof(struct row))) == NULL)
+		ulogd_error("%s: out of memory\n", __func__);
+
+	return row;
+}
+
 
-	/* commit all of the inserts to the database, ie flush buffer */
-	if (buffer_ctr >= buffer_size) {
-		if (sqlite3_exec(dbh,"commit",NULL,NULL,NULL) != SQLITE_OK)
-			ulogd_log(ULOGD_ERROR,"unable to commit records to db.");
+static inline void
+__row_del(struct sqlite3_priv *priv, struct row *row)
+{
+	assert(row != NULL);
 
-		if (sqlite3_exec(dbh,"begin deferred",NULL,NULL,NULL) != SQLITE_OK)
-			ulogd_log(ULOGD_ERROR,"unable to begin a new transaction.");
+	free(row);
+}
+
+
+static void
+row_del(struct sqlite3_priv *priv, struct row *row)
+{
+	llist_del(&row->link);
 
-		buffer_ctr = 0;
-		DEBUGP("committing.\n");
+	__row_del(priv, row);
+
+	priv->num_rows--;
+}
+
+
+static int
+row_add(struct sqlite3_priv *priv, struct row *row)
+{
+	if (priv->max_backlog && priv->num_rows >= priv->max_backlog) {
+		if (!priv->overlimit_msg) {
+			ulogd_error(PFX "over max-backlog limit, dropping rows\n");
+
+			priv->overlimit_msg = 1;
+		}
+
+		__row_del(priv, row);
+
+		return -1;
 	}
 
+	llist_add_tail(&row->link, &priv->rows);
+
+	priv->num_rows++;
+
 	return 0;
 }
 
+/* set_commit_time() - set time for next try on locked database
+ *
+ * The database is effectively locked in between.
+ */
+static void
+set_commit_time(const struct ulogd_pluginstance *pi)
+{
+	struct sqlite3_priv *priv = (void *)pi->private;
+
+	priv->commit_time = t_now + 1;
+
+	pr_debug("%s: commit time %d\n", __func__, priv->commit_time);
+}
+
 #define _SQLITE3_INSERTTEMPL   "insert into X (Y) values (Z)"
 
-/* create the static part of our insert statement */
-static int _sqlite3_createstmt(void)
+/* create static part of our insert statement */
+static int
+db_createstmt(struct ulogd_pluginstance *pi)
 {
-	struct _field *f;
-	unsigned int size;
+	struct sqlite3_priv *priv = (void *)pi->private;
 	char buf[ULOGD_MAX_KEYLEN];
 	char *underscore;
 	char *stmt_pos;
-	int col_count;
 	int i;
 
-	if (stmt) {
-		ulogd_log(ULOGD_NOTICE, "createstmt called, but stmt"
-			" already existing\n");	
-		return 1;
-	}
-
-	/* caclulate the size for the insert statement */
-	size = strlen(_SQLITE3_INSERTTEMPL) + strlen(table_ce.u.string);
+	if (priv->stmt != NULL)
+		free(priv->stmt);
 
-	DEBUGP("initial size: %u\n", size);
-
-	col_count = 0;
-	for (f = fields; f; f = f->next) {
-		/* we need space for the key and a comma, and a ? */
-		size += strlen(f->name) + 3;
-		DEBUGP("size is now %u since adding %s\n",size,f->name);
-		col_count++;
+	if ((priv->stmt = calloc(1, 1024)) == NULL) {
+		ulogd_error(PFX "out of memory\n");
+		return -1;
 	}
 
-	DEBUGP("there were %d columns\n",col_count);
-	DEBUGP("after calc name length: %u\n",size);
-
-	ulogd_log(ULOGD_DEBUG, "allocating %u bytes for statement\n", size);
-
-	stmt = (char *) malloc(size);
+	sprintf(priv->stmt, "insert into %s (", table_ce(pi));
+	stmt_pos = priv->stmt + strlen(priv->stmt);
 
-	if (!stmt) {
-		ulogd_log(ULOGD_ERROR, "OOM!\n");
-		return 1;
-	}
+	for (i = 0; i < DB_NUM_COLS; i++) {
+		struct col *col = &priv->cols[i];
 
-	sprintf(stmt, "insert into %s (", table_ce.u.string);
-	stmt_pos = stmt + strlen(stmt);
+		/* convert name */
+		strncpy(buf, col->name, ULOGD_MAX_KEYLEN);
 
-	for (f = fields; f; f = f->next) {
-		strncpy(buf, f->name, ULOGD_MAX_KEYLEN);	
 		while ((underscore = strchr(buf, '.')))
 			*underscore = '_';
+
 		sprintf(stmt_pos, "%s,", buf);
-		stmt_pos = stmt + strlen(stmt);
+		stmt_pos = priv->stmt + strlen(priv->stmt);
 	}
 
 	*(stmt_pos - 1) = ')';
 
 	sprintf(stmt_pos, " values (");
-	stmt_pos = stmt + strlen(stmt);
+	stmt_pos = priv->stmt + strlen(priv->stmt);
 
-	for (i = 0; i < col_count - 1; i++) {
+	for (i = 0; i < DB_NUM_COLS - 1; i++) {
 		sprintf(stmt_pos,"?,");
 		stmt_pos += 2;
 	}
 
 	sprintf(stmt_pos, "?)");
-	ulogd_log(ULOGD_DEBUG, "stmt='%s'\n", stmt);
+	ulogd_log(ULOGD_DEBUG, "%s: stmt='%s'\n", pi->id, priv->stmt);
 
-	DEBUGP("about to prepare statement.\n");
+	pr_debug("about to prepare statement.\n");
 
-	sqlite3_prepare(dbh,stmt,-1,&p_stmt,0);
-
-	DEBUGP("statement prepared.\n");
-
-	if (!p_stmt) {
-		ulogd_log(ULOGD_ERROR,"unable to prepare statement");
+	sqlite3_prepare(priv->dbh, priv->stmt, -1, &priv->p_stmt, 0);
+	if (priv->p_stmt == NULL) {
+		ulogd_error(PFX "prepare: %s\n", sqlite3_errmsg(priv->dbh));
 		return 1;
 	}
 
+	pr_debug("%s: statement prepared.\n", pi->id);
+
 	return 0;
 }
 
 
-/* length of "select * from \0" */
-#define SQLITE_SELECT_LEN 15
+static struct ulogd_key *
+ulogd_find_key(struct ulogd_pluginstance *pi, const char *name)
+{
+	int i;
+
+	for (i = 0; i < pi->input.num_keys; i++) {
+		if (strcmp(pi->input.keys[i].name, name) == 0)
+			return &pi->input.keys[i];
+	}
+
+	return NULL;
+}
+
+#define SELECT_ALL_STR			"select * from "
+#define SELECT_ALL_LEN			sizeof(SELECT_ALL_STR)
 
-/* find out which columns the table has */
-static int _sqlite3_get_columns(const char *table)
+static int
+db_count_cols(struct ulogd_pluginstance *pi, sqlite3_stmt **stmt)
 {
+	struct sqlite3_priv *priv = (void *)pi->private;
+	char query[SELECT_ALL_LEN + CONFIG_VAL_STRING_LEN] = SELECT_ALL_STR;
+
+	strncat(query, table_ce(pi), LINE_LEN);
+
+	if (sqlite3_prepare(priv->dbh, query, -1, stmt, 0) != SQLITE_OK) {
+		return -1;
+	}
+
+	return sqlite3_column_count(*stmt);
+}
+
+
+static int
+db_create_tbl(struct ulogd_pluginstance *pi)
+{
+	struct sqlite3_priv *priv = (void *)pi->private;
+	char *errmsg;
+	int ret;
+
+	sqlite3_exec(priv->dbh, "drop table daily", NULL, NULL, NULL);
+
+	ret = sqlite3_exec(priv->dbh, SQL_CREATE_STR, NULL, NULL, &errmsg);
+	if (ret != SQLITE_OK) {
+		ulogd_error(PFX "create table: %s\n", errmsg);
+		sqlite3_free(errmsg);
+
+		return -1;
+	}
+
+	return 0;
+}
+
+
+/* initialize DB, possibly creating it */
+static int
+db_init(struct ulogd_pluginstance *pi)
+{
+	struct sqlite3_priv *priv = (void *)pi->private;
 	char buf[ULOGD_MAX_KEYLEN];
-	char query[SQLITE_SELECT_LEN + CONFIG_VAL_STRING_LEN] = "select * from \0";
 	char *underscore;
-	struct _field *f;
 	sqlite3_stmt *schema_stmt;
-	int column;
-	int result;
-	int id;
+	int num_cols, i;
 
-	if (!dbh)
-		return 1;
+	if (priv->dbh == NULL)
+		return -1;
 
-	strncat(query,table,LINE_LEN);
-	
-	result = sqlite3_prepare(dbh,query,-1,&schema_stmt,0);
-	
-	if (result != SQLITE_OK)
-		return 1;
+	num_cols = db_count_cols(pi, &schema_stmt);
+	if (num_cols != DB_NUM_COLS) {
+		ulogd_log(ULOGD_INFO, "%s: (re)creating database\n", pi->id);
+
+		if (db_create_tbl(pi) < 0)
+			return -1;
+
+		num_cols = db_count_cols(pi, &schema_stmt);
+	}
+
+	assert(num_cols == DB_NUM_COLS);
+
+	for (i = 0; i < DB_NUM_COLS; i++) {
+		struct col *col = &priv->cols[i];
+
+		strncpy(buf, sqlite3_column_name(schema_stmt, i), ULOGD_MAX_KEYLEN);
 
-	for (column = 0; column < sqlite3_column_count(schema_stmt); column++) {
 		/* replace all underscores with dots */
-		strncpy(buf, sqlite3_column_name(schema_stmt,column), ULOGD_MAX_KEYLEN);
-		while ((underscore = strchr(buf, '_')))
+		while ((underscore = strchr(buf, '_')) != NULL)
 			*underscore = '.';
 
-		DEBUGP("field '%s' found: ", buf);
+		pr_debug("column '%s' found\n", buf);
 
-		if (!(id = keyh_getid(buf))) {
-			DEBUGP(" no keyid!\n");
-			continue;
+		strncpy(col->name, buf, ULOGD_MAX_KEYLEN);
+
+		if ((col->key = ulogd_find_key(pi, buf)) == NULL) {
+			printf(PFX "%s: key not found\n", buf);
+			return -1;
 		}
+	}
+
+	ulogd_log(ULOGD_INFO, "%s: database opened\n", pi->id);
+
+	if (sqlite3_finalize(schema_stmt) != SQLITE_OK) {
+		ulogd_error(PFX "sqlite_finalize: %s\n",
+					sqlite3_errmsg(priv->dbh));
+		return -1;
+	}
+
+	return 0;
+}
+
+
+static void
+db_reset(struct ulogd_pluginstance *pi)
+{
+	struct sqlite3_priv *priv = (void *)pi->private;
+
+	sqlite3_finalize(priv->p_stmt);
+
+	sqlite3_close(priv->dbh);
+	priv->dbh = NULL;
+
+	free(priv->stmt);
+	priv->stmt = NULL;
+}
+
+
+static int
+db_start(struct ulogd_pluginstance *pi)
+{
+	struct sqlite3_priv *priv = (void *)pi->private;
 
-		DEBUGP("keyid %u\n", id);
+	ulogd_log(ULOGD_DEBUG, "%s: opening database connection\n", pi->id);
 
-		/* prepend it to the linked list */
-		f = (struct _field *) malloc(sizeof *f);
-		if (!f) {
-			ulogd_log(ULOGD_ERROR, "OOM!\n");
-			return 1;
+	if (sqlite3_open(db_ce(pi), &priv->dbh) != SQLITE_OK) {
+		ulogd_error(PFX "%s\n", sqlite3_errmsg(priv->dbh));
+		return -1;
+	}
+
+	/* set the timeout so that we don't automatically fail
+	   if the table is busy */
+	sqlite3_busy_timeout(priv->dbh, SQLITE3_BUSY_TIMEOUT);
+
+	/* read the fieldnames to know which values to insert */
+	if (db_init(pi) < 0)
+		return -1;
+
+	/* initialize our buffer size and counter */
+	priv->buffer_size = buffer_ce(pi);
+
+	priv->max_backlog = max_backlog_ce(pi);
+
+	/* create and prepare the actual insert statement */
+	db_createstmt(pi);
+
+	return 0;
+}
+
+/* db_err() - handle database errors */
+static int
+db_err(struct ulogd_pluginstance *pi, int ret)
+{
+	struct sqlite3_priv *priv = (void *)pi->private;
+
+	pr_debug("%s: ret=%d (errcode %d)\n", __func__, ret,
+			 sqlite3_errcode(priv->dbh));
+
+	assert(ret != SQLITE_OK && ret != SQLITE_DONE);
+
+	if (ret == SQLITE_BUSY || ret == SQLITE_LOCKED)
+		set_commit_time(pi);
+	else {
+		switch (sqlite3_errcode(priv->dbh)) {
+		case SQLITE_LOCKED:
+		case SQLITE_BUSY:
+			set_commit_time(pi);
+			break;
+
+		case SQLITE_SCHEMA:
+			if (priv->stmt) {
+				sqlite3_finalize(priv->p_stmt);
+
+				db_createstmt(pi);
+
+				ulogd_log(ULOGD_INFO, "%s: database schema changed\n",
+						  pi->id);
+			}
+			break;
+
+		default:
+			ulogd_error("%s: transaction: %s\n", pi->id,
+						sqlite3_errmsg(priv->dbh));
+			break;
 		}
-		strncpy(f->name, buf, ULOGD_MAX_KEYLEN);
-		f->id = id;
-		f->next = fields;
-		fields = f;	
 	}
 
-	sqlite3_finalize(schema_stmt);
+	sqlite3_exec(priv->dbh, "rollback", NULL, NULL, NULL);
+
+	/* no sqlit3_clear_bindings(), as an unbind will be done implicitely
+	   on next bind. */
+	if (priv->p_stmt != NULL)
+		sqlite3_reset(priv->p_stmt);
+
 	return 0;
 }
 
-/** 
- * make connection and select database 
- * returns 0 if database failed to open.
- */
-static int _sqlite3_open_db(char *db_file)
+static int
+db_add_row(struct ulogd_pluginstance *pi, const struct row *row)
 {
-	DEBUGP("opening database.\n");
-	return sqlite3_open(db_file,&dbh);
+	struct sqlite3_priv *priv = (void *)pi->private;
+	int db_col = 1, ret;
+
+	do {
+		ret = sqlite3_bind_int64(priv->p_stmt, db_col++, row->ip_saddr);
+		if (ret != SQLITE_OK)
+			break;
+
+		ret = sqlite3_bind_int64(priv->p_stmt, db_col++, row->ip_daddr);
+		if (ret != SQLITE_OK)
+			break;
+
+		ret = sqlite3_bind_int(priv->p_stmt, db_col++, row->ip_proto);
+		if (ret != SQLITE_OK)
+			break;
+
+		ret = sqlite3_bind_int(priv->p_stmt, db_col++, row->l4_dport);
+		if (ret != SQLITE_OK)
+			break;
+
+		ret = sqlite3_bind_int(priv->p_stmt, db_col++, row->raw_in_pktlen);
+		if (ret != SQLITE_OK)
+			break;
+
+		ret = sqlite3_bind_int64(priv->p_stmt, db_col++, row->raw_in_pktcount);
+		if (ret != SQLITE_OK)
+			break;
+
+		ret = sqlite3_bind_int(priv->p_stmt, db_col++, row->raw_out_pktlen);
+		if (ret != SQLITE_OK)
+			break;
+
+		ret = sqlite3_bind_int64(priv->p_stmt, db_col++,
+								 row->raw_out_pktcount);
+		if (ret != SQLITE_OK)
+			break;
+
+		ret = sqlite3_bind_int(priv->p_stmt, db_col++, row->flow_start_sec);
+		if (ret != SQLITE_OK)
+			break;
+
+		ret = sqlite3_bind_int(priv->p_stmt, db_col++, row->flow_duration);
+		if (ret != SQLITE_OK)
+			break;
+
+		if ((ret = sqlite3_step(priv->p_stmt)) == SQLITE_DONE) {
+			/* no sqlite3_clear_bindings(), as an unbind will be
+			   implicetely done before next bind. */
+			sqlite3_reset(priv->p_stmt);
+
+			return 0;
+		}
+
+		/* according to the documentation sqlite3_step() always returns a
+		   generic SQLITE_ERROR.  In order to find out the cause of the
+		   error you have to call sqlite3_reset() or sqlite3_finalize(). */
+		ret = sqlite3_reset(priv->p_stmt);
+	} while (0);
+
+	return db_err(pi, ret);
 }
 
-/* give us an opportunity to close the database down properly */
-static void _sqlite3_fini(void)
+/* delete_rows() - delete rows from the tail of the list */
+static int
+delete_rows(struct ulogd_pluginstance *pi, int rows)
 {
-	DEBUGP("cleaning up db connection\n");
+	struct sqlite3_priv *priv = (void *)pi->private;
+	struct llist_head *curr, *tmp;
+
+    llist_for_each_prev_safe(curr, tmp, &priv->rows) {
+		struct row *row = container_of(curr, struct row, link);
 
-	/* free up our prepared statements so we can close the db */
-	if (p_stmt) {
-		sqlite3_finalize(p_stmt);
-		DEBUGP("prepared statement finalized\n");
+		if (rows-- == 0)
+			break;
+
+		row_del(priv, row);
 	}
 
-	if (dbh) {
-		int result;
-		/* flush the remaining insert statements to the database. */
-		result = sqlite3_exec(dbh,"commit",NULL,NULL,NULL);
+	return 0;
+}
+
+/*
+  db_commit_rows()
+
+  RETURN
+    >0	rows commited
+    0	locked
+   -1	error
+*/
+static int
+db_commit_rows(struct ulogd_pluginstance *pi)
+{
+	struct sqlite3_priv *priv = (void *)pi->private;
+	struct row *row;
+	int ret, rows = 0, max_commit;
+
+	ret = sqlite3_exec(priv->dbh, "begin immediate transaction", NULL,
+					   NULL, NULL);
+	if (ret != SQLITE_OK)
+		return db_err(pi, ret);
+
+	/* Limit number of rows to commit.  Note that currently three times
+	   buffer_size is a bit arbitrary and therefore might be adjusted in
+	   the future. */
+	max_commit = max(3 * priv->buffer_size, 1024);
 
-		if (result != SQLITE_OK)
-			ulogd_log(ULOGD_ERROR,"unable to commit remaining records to db.");
+	llist_for_each_entry_reverse(row, &priv->rows, link) {
+		if (++rows > max_commit)
+			break;
 
-		sqlite3_close(dbh);
-		DEBUGP("database file closed\n");
+		if (db_add_row(pi, row) < 0)
+			return db_err(pi, ret);
 	}
+
+	ret = sqlite3_exec(priv->dbh, "commit", NULL, NULL, NULL);
+	if (ret != SQLITE_OK)
+		return db_err(pi, ret);
+
+	sqlite3_reset(priv->p_stmt);
+	
+	pr_debug("%s: commited %d/%d rows\n", pi->id, rows, priv->num_rows);
+
+	delete_rows(pi, rows);
+	
+	if (priv->commit_time >= t_now)
+		priv->commit_time = 0;		/* release commit lock */
+	
+	if (priv->overlimit_msg)
+		priv->overlimit_msg = 0;
+
+	return rows;
 }
 
-#define _SQLITE3_BUSY_TIMEOUT 300
 
-static int _sqlite3_init(void)
+/* our main output function, called by ulogd */
+static int
+sqlite3_interp(struct ulogd_pluginstance *pi)
 {
-	/* have the opts parsed */
-	config_parse_file("SQLITE3", &buffer_ce);
+	struct sqlite3_priv *priv = (void *)pi->private;
+	struct col *cols = priv->cols;
+	struct row *row;
+
+	if ((row = row_new()) == NULL)
+		return ULOGD_IRET_ERR;
+
+	row->ip_saddr = RKEY(cols[0].key)->u.value.ui32;
+	row->ip_daddr = RKEY(cols[1].key)->u.value.ui32;
+	row->ip_proto = RKEY(cols[2].key)->u.value.ui8;
+	row->l4_dport = RKEY(cols[3].key)->u.value.ui16;
+	row->raw_in_pktlen = RKEY(cols[4].key)->u.value.ui32;
+	row->raw_in_pktcount = RKEY(cols[5].key)->u.value.ui32;
+	row->raw_out_pktlen = RKEY(cols[6].key)->u.value.ui32;
+	row->raw_out_pktcount = RKEY(cols[7].key)->u.value.ui32;
+	row->flow_start_sec = RKEY(cols[9].key)->u.value.ui32;
+	row->flow_duration = RKEY(cols[10].key)->u.value.ui32;
 
-	if (_sqlite3_open_db(db_ce.u.string)) {
-		ulogd_log(ULOGD_ERROR, "can't open the database file\n");
-		return 1;
+	if (row_add(priv, row) < 0)
+		return ULOGD_IRET_OK;
+
+	if (priv->num_rows >= priv->buffer_size && priv->commit_time == 0)
+		db_commit_rows(pi);
+
+	return ULOGD_IRET_OK;
+}
+
+
+static void
+sqlite_timer_cb(struct ulogd_timer *t)
+{
+	struct ulogd_pluginstance *pi = t->data;
+	struct sqlite3_priv *priv = (void *)pi->private;
+	int rows;
+
+	pr_debug("%s: timer=%p\n", __func__, t);
+
+	if (priv->commit_time != 0 && priv->commit_time > t_now)
+		return;
+
+	if (priv->num_rows == 0)
+		return;
+
+	rows = db_commit_rows(pi);
+
+	ulogd_log(ULOGD_DEBUG, "%s: rows=%d commited=%d\n", pi->id,
+			  priv->num_rows, rows);
+}
+
+
+static int
+sqlite3_configure(struct ulogd_pluginstance *pi,
+				  struct ulogd_pluginstance_stack *stack)
+{
+	struct sqlite3_priv *priv = (void *)pi->private;
+
+	memset(priv, 0, sizeof(struct sqlite3_priv));
+	
+	config_parse_file(pi->id, pi->config_kset);
+
+	if (ulogd_wildcard_inputkeys(pi) < 0)
+		return -1;
+
+	if (db_ce(pi) == NULL) {
+		ulogd_error("%s: configure: no database specified\n", pi->id);
+		return -1;
 	}
 
-	/* set the timeout so that we don't automatically fail
-         * if the table is busy. */
-	sqlite3_busy_timeout(dbh, _SQLITE3_BUSY_TIMEOUT);
+	if (table_ce(pi) == NULL) {
+		ulogd_error("%s: configure: no table specified\n", pi->id);
+		return -1;
+	}
 
-	/* read the fieldnames to know which values to insert */
-	if (_sqlite3_get_columns(table_ce.u.string)) {
-		ulogd_log(ULOGD_ERROR, "unable to get sqlite columns\n");
-		return 1;
+	if (timer_ce(pi) <= 0) {
+		ulogd_error("%s: configure: invalid timer value\n", pi->id);
+		return -1;
 	}
 
-	/* initialize our buffer size and counter */
-	buffer_size = buffer_ce.u.value;
-	buffer_ctr = 0;
+	if (max_backlog_ce(pi)) {
+		if (max_backlog_ce(pi) <= buffer_ce(pi)) {
+			ulogd_error("%s: configure: invalid max-backlog value\n",
+						pi->id);
+			return -1;
+		}
+	}
 
-	DEBUGP("Have a buffer size of : %d\n", buffer_size);
+	priv->max_backlog = max_backlog_ce(pi);
+	priv->disable = disable_ce(pi);
 
-	if (sqlite3_exec(dbh,"begin deferred",NULL,NULL,NULL) != SQLITE_OK)
-		ulogd_log(ULOGD_ERROR,"can't create a new transaction\n");
+	pr_debug("%s: db='%s' table='%s' timer=%d max-backlog=%d\n", pi->id,
+			 db_ce(pi), table_ce(pi), timer_ce(pi), max_backlog_ce(pi));
 
-	/* create and prepare the actual insert statement */
-	_sqlite3_createstmt();
+	return 0;
+}
+
+
+static int
+sqlite3_start(struct ulogd_pluginstance *pi)
+{
+	struct sqlite3_priv *priv = (void *)pi->private;
+
+	pr_debug("%s: pi=%p\n", __func__, pi);
+
+	if (priv->disable) {
+		ulogd_log(ULOGD_NOTICE, "%s: disabled\n", pi->id);
+		return 0;
+	}
+
+	priv->num_rows = 0;
+	INIT_LLIST_HEAD(&priv->rows);
+
+	if (db_start(pi) < 0)
+		return -1;
+
+	/* init timer */
+	priv->timer.cb = sqlite_timer_cb;
+	priv->timer.ival = timer_ce(pi);
+	priv->timer.flags = TIMER_F_PERIODIC;
+	priv->timer.data = pi;
+
+	if (ulogd_register_timer(&priv->timer) < 0)
+		return -1;
+
+	ulogd_log(ULOGD_INFO, "%s: started\n", pi->id);
 
 	return 0;
 }
 
-static ulog_output_t _sqlite3_plugin = { 
-	.name = "sqlite3", 
-	.output = &_sqlite3_output, 
-	.init = &_sqlite3_init,
-	.fini = &_sqlite3_fini,
-};
 
-void _init(void) 
+/* give us an opportunity to close the database down properly */
+static int
+sqlite3_stop(struct ulogd_pluginstance *pi)
 {
-	register_output(&_sqlite3_plugin);
+	struct sqlite3_priv *priv = (void *)pi->private;
+
+	pr_debug("%s: pi=%p\n", __func__, pi);
+
+	if (priv->disable)
+		return 0;				/* wasn't started */
+
+	if (priv->dbh == NULL)
+		return 0;				/* already stopped */
+
+	db_reset(pi);
+
+	return 0;
 }
 
-#endif
+
+static void
+sqlite3_signal(struct ulogd_pluginstance *pi, int sig)
+{
+	struct sqlite3_priv *priv = (void *)pi->private;
+
+	switch (sig) {
+	case SIGUSR1:
+		if (priv->dbh != NULL) {
+			db_reset(pi);
+
+			if (db_start(pi) < 0) {
+				ulogd_log(ULOGD_FATAL, "%s: database reset failed\n", pi->id);
+				exit(EXIT_FAILURE);
+			}
+		}
+		break;
+
+	default:
+		break;
+	}
+}
+
+
+static struct ulogd_plugin sqlite3_plugin = { 
+	.name = "SQLITE3",
+	.flags = ULOGD_PF_RECONF,
+	.input = {
+		.type = ULOGD_DTYPE_PACKET | ULOGD_DTYPE_FLOW,
+	},
+	.output = {
+		.type = ULOGD_DTYPE_SINK,
+	},
+	.config_kset = &sqlite3_kset,
+	.priv_size = sizeof(struct sqlite3_priv),
+	.configure = sqlite3_configure,
+	.start = sqlite3_start,
+	.stop = sqlite3_stop,
+	.signal = sqlite3_signal,
+	.interp = sqlite3_interp,
+	.version = ULOGD_VERSION,
+};
+
+static void init(void) __attribute__((constructor));
+
+static void
+init(void) 
+{
+	ulogd_register_plugin(&sqlite3_plugin);
+
+	ulogd_log(ULOGD_INFO, "using Sqlite version %s\n", sqlite3_libversion());
+}
#29 lore raw

PATCH 15/15 NFCT: rework and let it scale

In reply to [email protected]

Hi,
Content-Disposition: inline; filename=ulogd-NFCT-plugin.diff

Also implement garbage collection to account for the fact that netlink messages are sometimes lost (ENOBUFS) on busy sites. This is done by implementing a tuple cache (indexed by tuple) and a sequence cache and going regularly over part of both caches if need be.

input/flow/linux_jhash.h new file +146 -0
Index: ulogd-netfilter-stuffed/input/flow/linux_jhash.h
===================================================================
@@ -0,0 +1,146 @@
+#ifndef _LINUX_JHASH_H
+#define _LINUX_JHASH_H
+
+/* jhash.h: Jenkins hash support.
+ *
+ * Copyright (C) 1996 Bob Jenkins ([email protected])
+ *
+ * http://burtleburtle.net/bob/hash/
+ *
+ * These are the credits from Bob's sources:
+ *
+ * lookup2.c, by Bob Jenkins, December 1996, Public Domain.
+ * hash(), hash2(), hash3, and mix() are externally useful functions.
+ * Routines to test the hash are included if SELF_TEST is defined.
+ * You can use this free for any purpose.  It has no warranty.
+ *
+ * Copyright (C) 2003 David S. Miller ([email protected])
+ *
+ * I've modified Bob's hash to be useful in the Linux kernel, and
+ * any bugs present are surely my fault.  -DaveM
+ */
+
+#define u32		uint32_t
+#define u8		uint8_t
+
+/* NOTE: Arguments are modified. */
+#define __jhash_mix(a, b, c) \
+{ \
+  a -= b; a -= c; a ^= (c>>13); \
+  b -= c; b -= a; b ^= (a<<8); \
+  c -= a; c -= b; c ^= (b>>13); \
+  a -= b; a -= c; a ^= (c>>12);  \
+  b -= c; b -= a; b ^= (a<<16); \
+  c -= a; c -= b; c ^= (b>>5); \
+  a -= b; a -= c; a ^= (c>>3);  \
+  b -= c; b -= a; b ^= (a<<10); \
+  c -= a; c -= b; c ^= (b>>15); \
+}
+
+/* The golden ration: an arbitrary value */
+#define JHASH_GOLDEN_RATIO	0x9e3779b9
+
+/* The most generic version, hashes an arbitrary sequence
+ * of bytes.  No alignment or length assumptions are made about
+ * the input key.
+ */
+static inline u32 jhash(const void *key, u32 length, u32 initval)
+{
+	u32 a, b, c, len;
+	const u8 *k = key;
+
+	len = length;
+	a = b = JHASH_GOLDEN_RATIO;
+	c = initval;
+
+	while (len >= 12) {
+		a += (k[0] +((u32)k[1]<<8) +((u32)k[2]<<16) +((u32)k[3]<<24));
+		b += (k[4] +((u32)k[5]<<8) +((u32)k[6]<<16) +((u32)k[7]<<24));
+		c += (k[8] +((u32)k[9]<<8) +((u32)k[10]<<16)+((u32)k[11]<<24));
+
+		__jhash_mix(a,b,c);
+
+		k += 12;
+		len -= 12;
+	}
+
+	c += length;
+	switch (len) {
+	case 11: c += ((u32)k[10]<<24);
+	case 10: c += ((u32)k[9]<<16);
+	case 9 : c += ((u32)k[8]<<8);
+	case 8 : b += ((u32)k[7]<<24);
+	case 7 : b += ((u32)k[6]<<16);
+	case 6 : b += ((u32)k[5]<<8);
+	case 5 : b += k[4];
+	case 4 : a += ((u32)k[3]<<24);
+	case 3 : a += ((u32)k[2]<<16);
+	case 2 : a += ((u32)k[1]<<8);
+	case 1 : a += k[0];
+	};
+
+	__jhash_mix(a,b,c);
+
+	return c;
+}
+
+/* A special optimized version that handles 1 or more of u32s.
+ * The length parameter here is the number of u32s in the key.
+ */
+static inline u32 jhash2(u32 *k, u32 length, u32 initval)
+{
+	u32 a, b, c, len;
+
+	a = b = JHASH_GOLDEN_RATIO;
+	c = initval;
+	len = length;
+
+	while (len >= 3) {
+		a += k[0];
+		b += k[1];
+		c += k[2];
+		__jhash_mix(a, b, c);
+		k += 3; len -= 3;
+	}
+
+	c += length * 4;
+
+	switch (len) {
+	case 2 : b += k[1];
+	case 1 : a += k[0];
+	};
+
+	__jhash_mix(a,b,c);
+
+	return c;
+}
+
+
+/* A special ultra-optimized versions that knows they are hashing exactly
+ * 3, 2 or 1 word(s).
+ *
+ * NOTE: In partilar the "c += length; __jhash_mix(a,b,c);" normally
+ *       done at the end is not done here.
+ */
+static inline u32 jhash_3words(u32 a, u32 b, u32 c, u32 initval)
+{
+	a += JHASH_GOLDEN_RATIO;
+	b += JHASH_GOLDEN_RATIO;
+	c += initval;
+
+	__jhash_mix(a, b, c);
+
+	return c;
+}
+
+static inline u32 jhash_2words(u32 a, u32 b, u32 initval)
+{
+	return jhash_3words(a, b, 0, initval);
+}
+
+static inline u32 jhash_1word(u32 a, u32 initval)
+{
+	return jhash_3words(a, 0, 0, initval);
+}
+
+#endif /* _LINUX_JHASH_H */
input/flow/ulogd_inpflow_NFCT.c +902 -277

1,376 lines not on this page · raw message

#30 lore raw
In reply to [email protected]
Quoted 3 lines · [email protected] wrote:

Because the NFCT-related changes depend on a patchlet to libnetfilter-conntrack which might actually not be applied, I'll suggest to convert the code base to libnl sooner. Any objections?

No objections with using libnl but, as I told you in one of my previous emails, you can remove your small patch dependency by using libnfnetlink and libnetfilter_conntrack correctly.

Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers