[ULOGD RFC PATCH 0/34]
PATCH 24/34 Fix IPv4 output.
In reply to Eric LeblondFrom: Eric leblond <[email protected]>
Fix a bug in IPv4 output of IP2BIN module.
---
:100644 100644 7790cbd... 92bfa88... M filter/ulogd_filter_IP2BIN.c
filter/ulogd_filter_IP2BIN.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-)
filter/ulogd_filter_IP2BIN.c +2 -0
@@ -136,6 +136,7 @@ static char *ip2bin(struct ulogd_key* inp, int index, char family)char tmp[IPADDR_LENGTH ];unsigned char *addr8;struct in6_addr *addr; +struct in6_addr ip4_addr;char *buffer;int i, written;
@@ -145,6 +146,7 @@ static char *ip2bin(struct ulogd_key* inp, int index, char family)break ;case AF_INET :/* Convert IPv4 to IPv4 in IPv6 */ + addr = &ip4_addr;uint32_to_ipv6 (GET_VALUE (inp, index).ui32, addr);break ;default :
Signature · 1 line
1.5.2.5
PATCH 6/34 New version of SQL schema.
In reply to Eric LeblondFrom: Pierre Chifflier <[email protected]>
Add insert functions for the PostgreSQL version (read instructions).
---
:100644 100644 cbec234... ccbb8e8... M doc/mysql-ulogd2.sql
:100644 100644 61356b3... 016abc8... M doc/pgsql-ulogd2.sql
doc/mysql-ulogd2.sql | 18 --- doc/pgsql-ulogd2.sql | 282 +++++++++++++++++++++++++++++++++++++------------ 2 files changed, 213 insertions(+), 87 deletions(-)
doc/mysql-ulogd2.sql +0 -18
@@ -658,24 +658,6 @@ $$ delimiter ; -- suppressing tuples -DROP PROCEDURE IF EXISTS DELETE_CT_TUPLE; -delimiter $$ -CREATE PROCEDURE DELETE_CT_TUPLE( - IN _packet_id bigint unsigned - ) -SQL SECURITY INVOKER -COMMENT 'Delete a tuple from conntrack' -BEGIN - -- remember : table with most constraints first - DELETE FROM ct_icmp WHERE ct_icmp._icmp_id = _packet_id; - DELETE FROM ct_l4 WHERE ct_l4._l4_id = _packet_id; - DELETE FROM ct_tuple WHERE ct_tuple._tuple_id = _packet_id; -END -$$ - -delimiter ; - - DROP PROCEDURE IF EXISTS DELETE_CT_FLOW; delimiter $$ CREATE PROCEDURE DELETE_CT_FLOW(
doc/pgsql-ulogd2.sql +213 -69
335 lines not on this page · raw message
Signature · 1 line
1.5.2.5
PATCH 22/34 Fix description and indenting.
In reply to Eric LeblondFrom: Eric leblond <[email protected]>
---
:100644 100644 68a4ed7... 8a67cbe... M filter/ulogd_filter_IP2STR.c
filter/ulogd_filter_IP2STR.c | 6 ++---- 1 files changed, 2 insertions(+), 4 deletions(-)
filter/ulogd_filter_IP2STR.c +2 -4
@@ -1,6 +1,6 @@/* ulogd_filter_IP2STR.c, Version $Revision: 1500 $ * -* ulogd interpreter plugin for ifindex to ifname conversion +* ulogd interpreter plugin for internal IP storage format to string conversion * * (C) 2008 by Eric Leblond <[email protected]> *
@@ -119,7 +119,6 @@ static struct ulogd_key ip2str_keys[] = {#define GET_FLAGS (res, x) (res[x].u.source->flags)#define pp_is_valid (res, x) (res[x].u.source && (GET_FLAGS(res, x) & ULOGD_RETF_VALID)) -static char *ip2str (struct ulogd_key * inp,int index,char family) {char tmp[IPADDR_LENGTH ];
@@ -152,8 +151,7 @@ static int interp_ip2str(struct ulogd_pluginstance *pi)/* Iter on all addr fields */ for (i =START_KEY ; i <MAX_KEY ; i++) {if (pp_is_valid (inp, i)) { - ret[i-1 ].u.value.ptr =ip2str (inp, i, - oob_family); + ret[i-1 ].u.value.ptr =ip2str (inp, i, oob_family); ret[i-1 ].flags |=ULOGD_RETF_VALID ; } }
Signature · 1 line
1.5.2.5
PATCH 21/34 Add IP2BIN module: convert IP address to binary string.
In reply to Eric LeblondFrom: Eric leblond <[email protected]>
This module convert IP from internal notation to a string in binary notation which is used by the MySQL output plugin.
---
:100644 100644 94a14cd... 51bcc3f... M filter/Makefile.am
:000000 100644 0000000... 7790cbd... A filter/ulogd_filter_IP2BIN.c
filter/Makefile.am | 5 +- filter/ulogd_filter_IP2BIN.c | 229 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 233 insertions(+), 1 deletions(-)
filter/Makefile.am +4 -1
@@ -6,7 +6,7 @@ noinst_HEADERS = rtnl.h iftable.h pkglib_LTLIBRARIES = ulogd_filter_IFINDEX.la ulogd_filter_PWSNIFF.la \ ulogd_filter_PRINTPKT.la ulogd_filter_PRINTFLOW.la \ - ulogd_filter_IP2STR.la + ulogd_filter_IP2STR.la ulogd_filter_IP2BIN.la ulogd_filter_IFINDEX_la_SOURCES = ulogd_filter_IFINDEX.c rtnl.c iftable.c ulogd_filter_IFINDEX_la_LDFLAGS = -module
@@ -17,6 +17,9 @@ ulogd_filter_PWSNIFF_la_LDFLAGS = -module ulogd_filter_IP2STR_la_SOURCES = ulogd_filter_IP2STR.c ulogd_filter_IP2STR_la_LDFLAGS = -module +ulogd_filter_IP2BIN_la_SOURCES = ulogd_filter_IP2BIN.c +ulogd_filter_IP2BIN_la_LDFLAGS = -module + ulogd_filter_PRINTPKT_la_SOURCES = ulogd_filter_PRINTPKT.c ../util/printpkt.c ulogd_filter_PRINTPKT_la_LDFLAGS = -module
filter/ulogd_filter_IP2BIN.c new file +229 -0
229 lines not on this page · raw message
Signature · 1 line
1.5.2.5
PATCH 32/34 Add state extension.
In reply to Eric LeblondFrom: Eric leblond <[email protected]>
This patch adds an state extension to SQL schema. This can be used to store the information about the packet being dropped or accepted.
---
:100644 100644 eec6e69... d42d216... M doc/mysql-ulogd2.sql
doc/mysql-ulogd2.sql | 28 ++++++++++++++++++++++++++++ 1 files changed, 28 insertions(+), 0 deletions(-)
doc/mysql-ulogd2.sql +28 -0
@@ -191,6 +191,7 @@ CREATE SQL SECURITY INVOKER VIEW `ulog` AS FROM ulog2 LEFT JOIN tcp ON ulog2._id = tcp._tcp_id LEFT JOIN udp ON ulog2._id = udp._udp_id LEFT JOIN icmp ON ulog2._id = icmp._icmp_id LEFT JOIN mac ON ulog2._id = mac._mac_id; + -- shortcuts DROP VIEW IF EXISTS `view_tcp_quad`; CREATE SQL SECURITY INVOKER VIEW `view_tcp_quad` AS
@@ -301,6 +302,21 @@ INSERT INTO ip_proto (_proto_id,proto_name,proto_desc) VALUES (41,'ipv6','Internet Protocol, version 6'), (58,'ipv6-icmp','ICMP for IPv6'); +-- State +DROP TABLE IF EXISTS `state_t`; +CREATE TABLE `state_t` ( + `_state_id` bigint unsigned NOT NULL, + state tinyint(3) unsigned +) ENGINE=INNODB; + +ALTER TABLE state_t ADD UNIQUE KEY `_state_id` (`_state_id`); +ALTER TABLE state_t ADD KEY `index_state_id` (`_state_id`); +ALTER TABLE state_t ADD KEY `state` (`state`); +ALTER TABLE state_t ADD FOREIGN KEY (_state_id) REFERENCES ulog2 (_id); + +INSERT INTO _extensions (ext_name,table_name,join_name) VALUES + ('state','state_t','_state_id'); + -- NuFW specific DROP TABLE IF EXISTS `nufw`;
@@ -324,6 +340,18 @@ CREATE SQL SECURITY INVOKER VIEW `view_nufw` AS INSERT INTO _extensions (ext_name,table_name,join_name) VALUES ('nufw','nufw','_nufw_id'); +-- nufw view (nulog) +DROP VIEW IF EXISTS `nulog`; +-- CREATE SQL SECURITY INVOKER VIEW `ulog` AS +-- SELECT * FROM ulog2 INNER JOIN tcp ON ulog2._id = tcp._tcp_id INNER JOIN udp ON ulog2._id = udp._udp_id +-- INNER JOIN icmp ON ulog2._id = icmp._icmp_id INNER JOIN mac ON ulog2._id = mac._mac_id; +CREATE SQL SECURITY INVOKER VIEW `nulog` AS + SELECT * FROM ulog2 LEFT JOIN tcp ON ulog2._id = tcp._tcp_id LEFT JOIN udp ON ulog2._id = udp._udp_id + LEFT JOIN icmp ON ulog2._id = icmp._icmp_id LEFT JOIN mac ON ulog2._id = mac._mac_id + LEFT JOIN nufw ON ulog2._id = nufw._nufw_id LEFT JOIN state_t ON ulog2._id = state_t._state_id; + + + -- Procedures DROP PROCEDURE IF EXISTS ULOG2_DROP_FOREIGN_KEYS;
Signature · 1 line
1.5.2.5
PATCH 20/34 Mark ID as inactive (sequence in pg schema)
In reply to Eric LeblondFrom: Pierre Chifflier <[email protected]>
---
:100644 100644 6697b64... 0882357... M output/pgsql/ulogd_output_PGSQL.c
output/pgsql/ulogd_output_PGSQL.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-)
output/pgsql/ulogd_output_PGSQL.c +2 -1
@@ -196,7 +196,8 @@ static int get_columns_pgsql(struct ulogd_pluginstance *upi)strncpy (upi->input.keys[i].name, buf,ULOGD_MAX_KEYLEN ); } -/* FIXME: id? */ +/* ID is a sequence */ + upi->input.keys[0 ].flags |=ULOGD_KEYF_INACTIVE ;PQclear (pi->pgres);return 0 ;
Signature · 1 line
1.5.2.5
PATCH 18/34 Update schema for PostgreSQL.
In reply to Eric LeblondFrom: Pierre Chifflier <[email protected]>
This patch add _str suffix to inet types (needed after IP2STR introduction)
---
:100644 100644 016abc8... d6e6efd... M doc/pgsql-ulogd2.sql
doc/pgsql-ulogd2.sql | 46 +++++++++++++++++++++++----------------------- 1 files changed, 23 insertions(+), 23 deletions(-)
doc/pgsql-ulogd2.sql +23 -23
@@ -44,8 +44,8 @@ CREATE TABLE ulog2 ( oob_mark integer default NULL, oob_in varchar(32) default NULL, oob_out varchar(32) default NULL, - ip_saddr inet default NULL, - ip_daddr inet default NULL, + ip_saddr_str inet default NULL, + ip_daddr_str inet default NULL, ip_protocol smallint default NULL, ip_tos smallint default NULL, ip_ttl smallint default NULL,
@@ -58,8 +58,8 @@ CREATE TABLE ulog2 ( ) WITH (OIDS=FALSE); CREATE INDEX ulog2_timestamp ON ulog2(timestamp); -CREATE INDEX ulog2_ip_saddr ON ulog2(ip_saddr); -CREATE INDEX ulog2_ip_daddr ON ulog2(ip_daddr); +CREATE INDEX ulog2_ip_saddr ON ulog2(ip_saddr_str); +CREATE INDEX ulog2_ip_daddr ON ulog2(ip_daddr_str); CREATE TABLE mac ( _mac_id bigint PRIMARY KEY UNIQUE NOT NULL,
@@ -136,10 +136,10 @@ CREATE OR REPLACE VIEW ulog AS -- shortcuts CREATE OR REPLACE VIEW view_tcp_quad AS - SELECT ulog2._id,ulog2.ip_saddr,tcp.tcp_sport,ulog2.ip_daddr,tcp.tcp_dport FROM ulog2 INNER JOIN tcp ON ulog2._id = tcp._tcp_id; + SELECT ulog2._id,ulog2.ip_saddr_str,tcp.tcp_sport,ulog2.ip_daddr_str,tcp.tcp_dport FROM ulog2 INNER JOIN tcp ON ulog2._id = tcp._tcp_id; CREATE OR REPLACE VIEW view_udp_quad AS - SELECT ulog2._id,ulog2.ip_saddr,udp.udp_sport,ulog2.ip_daddr,udp.udp_dport FROM ulog2 INNER JOIN udp ON ulog2._id = udp._udp_id; + SELECT ulog2._id,ulog2.ip_saddr_str,udp.udp_sport,ulog2.ip_daddr_str,udp.udp_dport FROM ulog2 INNER JOIN udp ON ulog2._id = udp._udp_id; -- -- conntrack
@@ -148,15 +148,15 @@ DROP SEQUENCE IF EXISTS ulog2_ct__ct_id_seq; CREATE SEQUENCE ulog2_ct__ct_id_seq; CREATE TABLE ulog2_ct ( _ct_id bigint PRIMARY KEY UNIQUE NOT NULL DEFAULT nextval('ulog2_ct__ct_id_seq'), - orig_ip_saddr inet default NULL, - orig_ip_daddr inet default NULL, + orig_ip_saddr_str inet default NULL, + orig_ip_daddr_str inet default NULL, orig_ip_protocol smallint default NULL, orig_l4_sport integer default NULL, orig_l4_dport integer default NULL, orig_bytes bigint default 0, orig_packets bigint default 0, - reply_ip_saddr inet default NULL, - reply_ip_daddr inet default NULL, + reply_ip_saddr_str inet default NULL, + reply_ip_daddr_str inet default NULL, reply_ip_protocol smallint default NULL, reply_l4_sport integer default NULL, reply_l4_dport integer default NULL,
@@ -172,10 +172,10 @@ CREATE TABLE ulog2_ct ( state smallint default 0 ) WITH (OIDS=FALSE); -CREATE INDEX ulog2_ct_orig_ip_saddr ON ulog2_ct(orig_ip_saddr); -CREATE INDEX ulog2_ct_orig_ip_daddr ON ulog2_ct(orig_ip_daddr); -CREATE INDEX ulog2_ct_reply_ip_saddr ON ulog2_ct(reply_ip_saddr); -CREATE INDEX ulog2_ct_reply_ip_daddr ON ulog2_ct(reply_ip_daddr); +CREATE INDEX ulog2_ct_orig_ip_saddr ON ulog2_ct(orig_ip_saddr_str); +CREATE INDEX ulog2_ct_orig_ip_daddr ON ulog2_ct(orig_ip_daddr_str); +CREATE INDEX ulog2_ct_reply_ip_saddr ON ulog2_ct(reply_ip_saddr_str); +CREATE INDEX ulog2_ct_reply_ip_daddr ON ulog2_ct(reply_ip_daddr_str); CREATE INDEX ulog2_ct_orig_l4_sport ON ulog2_ct(orig_l4_sport); CREATE INDEX ulog2_ct_orig_l4_dport ON ulog2_ct(orig_l4_dport); CREATE INDEX ulog2_ct_reply_l4_sport ON ulog2_ct(reply_l4_sport);
@@ -263,13 +263,13 @@ CREATE OR REPLACE FUNCTION INSERT_IP_PACKET( IN oob_mark integer, IN oob_in varchar(32), IN oob_out varchar(32), - IN ip_saddr inet, - IN ip_daddr inet, + IN ip_saddr_str inet, + IN ip_daddr_str inet, IN ip_protocol smallint ) RETURNS bigint AS $$ INSERT INTO ulog2 (oob_time_sec,oob_time_usec,oob_prefix,oob_mark, - oob_in,oob_out,ip_saddr,ip_daddr,ip_protocol) + oob_in,oob_out,ip_saddr_str,ip_daddr_str,ip_protocol) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9); SELECT currval('ulog2__id_seq'); $$ LANGUAGE SQL SECURITY INVOKER;
@@ -282,8 +282,8 @@ CREATE OR REPLACE FUNCTION INSERT_IP_PACKET_FULL( IN oob_mark integer, IN oob_in varchar(32), IN oob_out varchar(32), - IN ip_saddr inet, - IN ip_daddr inet, + IN ip_saddr_str inet, + IN ip_daddr_str inet, IN ip_protocol smallint, IN ip_tos smallint, IN ip_ttl smallint,
@@ -295,7 +295,7 @@ CREATE OR REPLACE FUNCTION INSERT_IP_PACKET_FULL( ) RETURNS bigint AS $$ INSERT INTO ulog2 (oob_time_sec,oob_time_usec,oob_prefix,oob_mark, - oob_in,oob_out,ip_saddr,ip_daddr,ip_protocol, + oob_in,oob_out,ip_saddr_str,ip_daddr_str,ip_protocol, ip_tos,ip_ttl,ip_totlen,ip_ihl,ip_csum,ip_id,ip_fragoff) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16); SELECT currval('ulog2__id_seq');
@@ -371,8 +371,8 @@ CREATE OR REPLACE FUNCTION INSERT_PACKET_FULL( IN oob_mark integer, IN oob_in varchar(32), IN oob_out varchar(32), - IN ip_saddr inet, - IN ip_daddr inet, + IN ip_saddr_str inet, + IN ip_daddr_str inet, IN ip_protocol smallint, IN ip_tos smallint, IN ip_ttl smallint,
@@ -498,4 +498,4 @@ $$ LANGUAGE SQL SECURITY INVOKER; -- Add foreign keys to tables SELECT ULOG2_ADD_FOREIGN_KEYS(); - +-- Pierre Chifflier <chifflier AT inl DOT fr>
Signature · 1 line
1.5.2.5
PATCH 16/34 Introduce IP2STR module which convert IP to string.
In reply to Eric LeblondFrom: Eric leblond <[email protected]>
This module is a generic module which is used to convert an IP from internal representation to string representation. This is a task needed by several modules like printpkt or pgsql. This module factorizes the code.
---
:100644 100644 b3207f9... 94a14cd... M filter/Makefile.am
:000000 100644 0000000... 68a4ed7... A filter/ulogd_filter_IP2STR.c
filter/Makefile.am | 6 +- filter/ulogd_filter_IP2STR.c | 198 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 203 insertions(+), 1 deletions(-)
filter/Makefile.am +5 -1
@@ -5,7 +5,8 @@ INCLUDES = $(all_includes) -I$(top_srcdir)/include noinst_HEADERS = rtnl.h iftable.h pkglib_LTLIBRARIES = ulogd_filter_IFINDEX.la ulogd_filter_PWSNIFF.la \ - ulogd_filter_PRINTPKT.la ulogd_filter_PRINTFLOW.la + ulogd_filter_PRINTPKT.la ulogd_filter_PRINTFLOW.la \ + ulogd_filter_IP2STR.la ulogd_filter_IFINDEX_la_SOURCES = ulogd_filter_IFINDEX.c rtnl.c iftable.c ulogd_filter_IFINDEX_la_LDFLAGS = -module
@@ -13,6 +14,9 @@ ulogd_filter_IFINDEX_la_LDFLAGS = -module ulogd_filter_PWSNIFF_la_SOURCES = ulogd_filter_PWSNIFF.c ulogd_filter_PWSNIFF_la_LDFLAGS = -module +ulogd_filter_IP2STR_la_SOURCES = ulogd_filter_IP2STR.c +ulogd_filter_IP2STR_la_LDFLAGS = -module + ulogd_filter_PRINTPKT_la_SOURCES = ulogd_filter_PRINTPKT.c ../util/printpkt.c ulogd_filter_PRINTPKT_la_LDFLAGS = -module
filter/ulogd_filter_IP2STR.c new file +198 -0
198 lines not on this page · raw message
Signature · 1 line
1.5.2.5
PATCH 1/34 Introduce new SQL schema.
In reply to Eric LeblondFrom: Eric leblond <[email protected]>
This patch adds new SQL schema for MySQL and PGsql. The goal is to improve the one line per entry format. There is no more a big table with all fields because this sort of storage is causing bad performance (databases don't like to have a lot of NULL fields to store).
Main changes are :
* Add new schema for MySQL and PGsql
* Use call to configurable procedure in SQL OUTPUT modules
* Arguments of a procedure are given by the list of fields of a selected table
---
:000000 100644 0000000... cbec234... A doc/mysql-ulogd2.sql
:000000 100644 0000000... 61356b3... A doc/pgsql-ulogd2.sql
:100644 100644 94752ae... 94cdbcb... M include/ulogd/db.h
:100644 100644 16d3d1a... d43f1fd... M ulogd.conf.in
:100644 100644 65d0f39... 1702acc... M util/db.c
doc/mysql-ulogd2.sql | 745 ++++++++++++++++++++++++++++++++++++++++++++++++++ doc/pgsql-ulogd2.sql | 357 ++++++++++++++++++++++++ include/ulogd/db.h | 8 +- ulogd.conf.in | 7 + util/db.c | 23 +-- 5 files changed, 1120 insertions(+), 20 deletions(-)
doc/mysql-ulogd2.sql new file +745 -0
@@ -0,0 +1,745 @@ +-- general notes: +-- - tables are split using the protocol +-- - keys are created outside the table, when possible +-- - foreign keys (constraints) are added using ULOG2_ADD_FOREIGN_KEYS() +-- - some procedures for maintainance are provided (suppressing entries, compressing tables, running ~VACUUM) +-- - security is set to INVOKER, which means the permissions of the connected client are used. To create an abstraction layer, DEFINER could be used (with precautions on DELETE ..) + + +-- (most constraint) ulog2_ct >> tcp,udp,icmp >> ulog2 (least constraint) + + +DROP TABLE IF EXISTS `_format`; +CREATE TABLE `_format` ( + `version` int(4) NOT NULL +) ENGINE=INNODB; + +INSERT INTO _format (version) VALUES (3); + +-- this table could be used to know which user-defined tables are linked +-- to ulog +DROP TABLE IF EXISTS `_extensions`; +CREATE TABLE `_extensions` ( + `ext_id` int(8) unsigned NOT NULL auto_increment, + `ext_name` varchar(64) NOT NULL, + `table_name` varchar(64) NOT NULL, + `join_name` varchar(64) NOT NULL, + UNIQUE KEY `ext_id` (`ext_id`) +) ENGINE=INNODB; + +DROP TABLE IF EXISTS `mac`; +DROP TABLE IF EXISTS `tcp`; +DROP TABLE IF EXISTS `udp`; +DROP TABLE IF EXISTS `icmp`; +DROP TABLE IF EXISTS `nufw`; +DROP TABLE IF EXISTS `ulog2_ct`; +DROP TABLE IF EXISTS `ct_tuple`; +DROP TABLE IF EXISTS `ct_l4`; +DROP TABLE IF EXISTS `ct_icmp`; +DROP TABLE IF EXISTS `ulog2`; + +CREATE TABLE `ulog2` ( + `_id` bigint unsigned NOT NULL auto_increment, + `oob_time_sec` int(10) unsigned default NULL, + `oob_time_usec` int(10) unsigned default NULL, + `oob_prefix` varchar(32) default NULL, + `oob_mark` int(10) unsigned default NULL, + `oob_in` varchar(32) default NULL, + `oob_out` varchar(32) default NULL, + `ip_saddr` binary(16) default NULL, + `ip_daddr` binary(16) default NULL, + `ip_protocol` tinyint(3) unsigned default NULL, + `ip_tos` tinyint(3) unsigned default NULL, + `ip_ttl` tinyint(3) unsigned default NULL, + `ip_totlen` smallint(5) unsigned default NULL, + `ip_ihl` tinyint(3) unsigned default NULL, + `ip_csum` smallint(5) unsigned default NULL, + `ip_id` smallint(5) unsigned default NULL, + `ip_fragoff` smallint(5) unsigned default NULL, + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP, + UNIQUE KEY `key_id` (`_id`) +) ENGINE=INNODB COMMENT='Table for IP packets'; + +ALTER TABLE ulog2 ADD KEY `index_id` (`_id`); +ALTER TABLE ulog2 ADD KEY `timestamp` (`timestamp`); +ALTER TABLE ulog2 ADD KEY `ip_saddr` (`ip_saddr`); +ALTER TABLE ulog2 ADD KEY `ip_daddr` (`ip_daddr`); +-- This index does not seem very useful: +-- ALTER TABLE ulog2 ADD KEY `oob_time_sec` (`oob_time_sec`); + +CREATE TABLE `mac` ( + `_mac_id` bigint unsigned NOT NULL, + `mac_saddr` binary(12) default NULL, + `mac_daddr` binary(12) default NULL, + `mac_protocol` smallint(5) default NULL +) ENGINE=INNODB; + +ALTER TABLE mac ADD UNIQUE KEY `_mac_id` (`_mac_id`); +ALTER TABLE mac ADD KEY `mac_saddr` (`mac_saddr`); +ALTER TABLE mac ADD KEY `mac_daddr` (`mac_daddr`); +ALTER TABLE mac ADD KEY `index_mac_id` (`_mac_id`); + +CREATE TABLE `tcp` ( + `_tcp_id` bigint unsigned NOT NULL, + `tcp_sport` int(5) unsigned default NULL, + `tcp_dport` int(5) unsigned default NULL, + `tcp_seq` int(10) unsigned default NULL, + `tcp_ackseq` int(10) unsigned default NULL, + `tcp_window` int(5) unsigned default NULL, + `tcp_urg` tinyint(4) default NULL, + `tcp_urgp` int(5) unsigned default NULL, + `tcp_ack` tinyint(4) default NULL, + `tcp_psh` tinyint(4) default NULL, + `tcp_rst` tinyint(4) default NULL, + `tcp_syn` tinyint(4) default NULL, + `tcp_fin` tinyint(4) default NULL +) ENGINE=INNODB; + +ALTER TABLE tcp ADD UNIQUE KEY `_tcp_id` (`_tcp_id`); +ALTER TABLE tcp ADD KEY `index_tcp_id` (`_tcp_id`); +ALTER TABLE tcp ADD KEY `tcp_sport` (`tcp_sport`); +ALTER TABLE tcp ADD KEY `tcp_dport` (`tcp_dport`); + + +CREATE TABLE `udp` ( + `_udp_id` bigint unsigned NOT NULL, + `udp_sport` int(5) unsigned default NULL, + `udp_dport` int(5) unsigned default NULL, + `udp_len` int(5) unsigned default NULL +) ENGINE=INNODB; + +ALTER TABLE udp ADD UNIQUE KEY `_udp_id` (`_udp_id`); +ALTER TABLE udp ADD KEY `index_udp_id` (`_udp_id`); +ALTER TABLE udp ADD KEY `udp_sport` (`udp_sport`); +ALTER TABLE udp ADD KEY `udp_dport` (`udp_dport`); + +CREATE TABLE `icmp` ( + `_icmp_id` bigint unsigned NOT NULL, + `icmp_type` tinyint(3) unsigned default NULL, + `icmp_code` tinyint(3) unsigned default NULL, + `icmp_echoid` smallint(5) unsigned default NULL, + `icmp_echoseq` smallint(5) unsigned default NULL, + `icmp_gateway` int(10) unsigned default NULL, + `icmp_fragmtu` smallint(5) unsigned default NULL +) ENGINE=INNODB; + +ALTER TABLE icmp ADD UNIQUE KEY `key_icmp_id` (`_icmp_id`); +ALTER TABLE icmp ADD KEY `index_icmp_id` (`_icmp_id`); + + +-- views + +DROP VIEW IF EXISTS `view_tcp`; +CREATE SQL SECURITY INVOKER VIEW `view_tcp` AS + SELECT * FROM ulog2 INNER JOIN tcp ON ulog2._id = tcp._tcp_id; + +-- alternate form: +-- select * from ulog2 where ulog2._id in (select tcp._tcp_id from tcp where tcp._tcp_id is not null); + +DROP VIEW IF EXISTS `view_udp`; +CREATE SQL SECURITY INVOKER VIEW `view_udp` AS + SELECT * FROM ulog2 INNER JOIN udp ON ulog2._id = udp._udp_id; + +DROP VIEW IF EXISTS `view_icmp`; +CREATE SQL SECURITY INVOKER VIEW `view_icmp` AS + SELECT * FROM ulog2 INNER JOIN icmp ON ulog2._id = icmp._icmp_id; + +-- ulog view +DROP VIEW IF EXISTS `ulog`; +CREATE SQL SECURITY INVOKER VIEW `ulog` AS + SELECT * FROM ulog2 INNER JOIN tcp ON ulog2._id = tcp._tcp_id INNER JOIN udp ON ulog2._id = udp._udp_id + INNER JOIN icmp ON ulog2._id = icmp._icmp_id INNER JOIN mac ON ulog2._id = mac._mac_id; + +-- shortcuts +DROP VIEW IF EXISTS `view_tcp_quad`; +CREATE SQL SECURITY INVOKER VIEW `view_tcp_quad` AS + SELECT ulog2._id,ulog2.ip_saddr,tcp.tcp_sport,ulog2.ip_daddr,tcp.tcp_dport FROM ulog2 INNER JOIN tcp ON ulog2._id = tcp._tcp_id; + +DROP VIEW IF EXISTS `view_udp_quad`; +CREATE SQL SECURITY INVOKER VIEW `view_udp_quad` AS + SELECT ulog2._id,ulog2.ip_saddr,udp.udp_sport,ulog2.ip_daddr,udp.udp_dport FROM ulog2 INNER JOIN udp ON ulog2._id = udp._udp_id; + + + +-- conntrack + +CREATE TABLE `ulog2_ct` ( + `_ct_id` bigint unsigned NOT NULL auto_increment, + `orig_ip_saddr` binary(16) default NULL, + `orig_ip_daddr` binary(16) default NULL, + `orig_ip_protocol` tinyint(3) unsigned default NULL, + `orig_l4_sport` int(5) default NULL, + `orig_l4_dport` int(5) default NULL, + `orig_bytes` bigint default 0, + `orig_packets` bigint default 0, + `reply_ip_saddr` binary(16) default NULL, + `reply_ip_daddr` binary(16) default NULL, + `reply_ip_protocol` tinyint(3) unsigned default NULL, + `reply_l4_sport` int(5) default NULL, + `reply_l4_dport` int(5) default NULL, + `reply_bytes` bigint default 0, + `reply_packets` bigint default 0, + `icmp_code` tinyint(3) default NULL, + `icmp_type` tinyint(3) default NULL, + `ct_mark` bigint default 0, + `flow_start_sec` int(10) default 0, + `flow_start_usec` int(10) default 0, + `flow_end_sec` int(10) default 0, + `flow_end_usec` int(10) default 0, + `state` tinyint(3) unsigned default 0, + + UNIQUE KEY `_ct_id` (`_ct_id`) +) ENGINE=INNODB; + +ALTER TABLE ulog2_ct ADD KEY `index_ct_id` (`_ct_id`); +ALTER TABLE ulog2_ct ADD KEY `orig_ip_saddr` (`orig_ip_saddr`); +ALTER TABLE ulog2_ct ADD KEY `orig_ip_daddr` (`orig_ip_daddr`); +ALTER TABLE ulog2_ct ADD KEY `orig_ip_protocol` (`orig_ip_protocol`); +ALTER TABLE ulog2_ct ADD KEY `orig_l4_dport` (`orig_l4_dport`); +ALTER TABLE ulog2_ct ADD KEY `orig_l4_sport` (`orig_l4_sport`); +ALTER TABLE ulog2_ct ADD KEY `reply_ip_saddr` (`reply_ip_saddr`); +ALTER TABLE ulog2_ct ADD KEY `reply_ip_daddr` (`reply_ip_daddr`); +ALTER TABLE ulog2_ct ADD KEY `reply_ip_protocol` (`reply_ip_protocol`); +ALTER TABLE ulog2_ct ADD KEY `reply_l4_dport` (`reply_l4_dport`); +ALTER TABLE ulog2_ct ADD KEY `reply_l4_sport` (`reply_l4_sport`); +ALTER TABLE ulog2_ct ADD KEY `state` (`state`); +ALTER TABLE ulog2_ct ADD KEY `orig_tuple` (`orig_ip_saddr`, `orig_ip_daddr`, `orig_ip_protocol`, + `orig_l4_sport`, `orig_l4_dport`); +ALTER TABLE ulog2_ct ADD KEY `reply_tuple` (`reply_ip_saddr`, `reply_ip_daddr`, `reply_ip_protocol`, + `reply_l4_sport`, `reply_l4_dport`); + +DROP VIEW IF EXISTS `conntrack`; +CREATE SQL SECURITY INVOKER VIEW `conntrack` AS + SELECT _ct_id, + orig_ip_saddr, + orig_ip_daddr, + orig_ip_protocol, + orig_l4_sport, + orig_l4_dport, + orig_bytes AS orig_raw_pktlen, + orig_packets AS orig_raw_pktcount, + reply_ip_saddr, + reply_ip_daddr, + reply_ip_protocol, + reply_l4_sport, + reply_l4_dport, + reply_bytes AS reply_raw_pktlen, + reply_packets AS reply_raw_pktcount, + icmp_code, + icmp_type, + ct_mark, + flow_start_sec, + flow_start_usec, + flow_end_sec, + flow_end_usec FROM ulog2_ct WHERE state != 0; + +-- Helper table +DROP TABLE IF EXISTS `ip_proto`; +CREATE TABLE `ip_proto` ( + `_proto_id` int(10) unsigned NOT NULL, + `proto_name` varchar(16) default NULL, + `proto_desc` varchar(255) default NULL +) ENGINE=INNODB; + +ALTER TABLE ip_proto ADD UNIQUE KEY `_proto_id` (`_proto_id`); + +-- see files /etc/protocols +-- or /usr/share/nmap/nmap-protocols +INSERT INTO ip_proto (_proto_id,proto_name,proto_desc) VALUES + (0,'ip','internet protocol, pseudo protocol number'), + (1,'icmp','internet control message protocol'), + (2,'igmp','Internet Group Management'), + (3,'ggp','gateway-gateway protocol'), + (4,'ipencap','IP encapsulated in IP (officially \'IP\')'), + (5,'st','ST datagram mode'), + (6,'tcp','transmission control protocol'), + (17,'udp','user datagram protocol'), + (41,'ipv6','Internet Protocol, version 6'), + (58,'ipv6-icmp','ICMP for IPv6'); + +-- NuFW specific + +DROP TABLE IF EXISTS `nufw`; +CREATE TABLE `nufw` ( + `_nufw_id` bigint unsigned NOT NULL, + `username` varchar(30) default NULL, + `user_id` smallint(5) unsigned default NULL, + `client_os` varchar(100) default NULL, + `client_app` varchar(256) default NULL +) ENGINE=INNODB; + +ALTER TABLE nufw ADD UNIQUE KEY `_nufw_id` (`_nufw_id`); +ALTER TABLE nufw ADD KEY `index_nufw_id` (`_nufw_id`); +ALTER TABLE nufw ADD KEY `user_id` (`user_id`); +ALTER TABLE nufw ADD FOREIGN KEY (_nufw_id) REFERENCES ulog2 (_id); + +DROP VIEW IF EXISTS `view_nufw`; +CREATE SQL SECURITY INVOKER VIEW `view_nufw` AS + SELECT * FROM ulog2 INNER JOIN nufw ON ulog2._id = nufw._nufw_id; + +INSERT INTO _extensions (ext_name,table_name,join_name) VALUES + ('nufw','nufw','_nufw_id'); + +-- Procedures + +DROP PROCEDURE IF EXISTS ULOG2_DROP_FOREIGN_KEYS; +delimiter $$ +CREATE PROCEDURE ULOG2_DROP_FOREIGN_KEYS( + ) +SQL SECURITY INVOKER +COMMENT 'Drop constraints for ulog2 tables' +BEGIN + -- remember : table with most constraints first + ALTER TABLE icmp DROP FOREIGN KEY _icmp_id; + ALTER TABLE udp DROP FOREIGN KEY _udp_id; + ALTER TABLE tcp DROP FOREIGN KEY _tcp_id; +END +$$ +delimiter ; + +DROP PROCEDURE IF EXISTS ULOG2_ADD_FOREIGN_KEYS; +delimiter $$ +CREATE PROCEDURE ULOG2_ADD_FOREIGN_KEYS( + ) +SQL SECURITY INVOKER +COMMENT 'Add constraints for ulog2 tables' +BEGIN + -- remember : table with least constraints first + ALTER TABLE tcp ADD CONSTRAINT _tcp_id FOREIGN KEY (_tcp_id) REFERENCES ulog2 (_id); + ALTER TABLE udp ADD CONSTRAINT _udp_id FOREIGN KEY (_udp_id) REFERENCES ulog2 (_id); + ALTER TABLE icmp ADD CONSTRAINT _icmp_id FOREIGN KEY (_icmp_id) REFERENCES ulog2 (_id); +END +$$ +delimiter ; + +delimiter $$ +DROP FUNCTION IF EXISTS INSERT_IP_PACKET; +CREATE FUNCTION INSERT_IP_PACKET( + _oob_time_sec int(10) unsigned, + _oob_time_usec int(10) unsigned, + _oob_prefix varchar(32), + _oob_mark int(10) unsigned, + _oob_in varchar(32), + _oob_out varchar(32), + _ip_saddr int(16), + _ip_daddr int(16), + _ip_protocol tinyint(3) unsigned + ) RETURNS bigint unsigned +SQL SECURITY INVOKER +NOT DETERMINISTIC +READS SQL DATA +BEGIN + INSERT INTO ulog2 (oob_time_sec, oob_time_usec, oob_prefix, oob_mark, oob_in, oob_out, + ip_saddr, ip_daddr, ip_protocol) VALUES + (_oob_time_sec, _oob_time_usec, _oob_prefix, _oob_mark, _oob_in, _oob_out, + _ip_saddr, _ip_daddr, _ip_protocol); + RETURN LAST_INSERT_ID(); +END +$$ + +delimiter $$ +DROP FUNCTION IF EXISTS INSERT_IP_PACKET_FULL; +CREATE FUNCTION INSERT_IP_PACKET_FULL( + _oob_time_sec int(10) unsigned, + _oob_time_usec int(10) unsigned, + _oob_prefix varchar(32), + _oob_mark int(10) unsigned, + _oob_in varchar(32), + _oob_out varchar(32), + _ip_saddr int(16), + _ip_daddr int(16), + _ip_protocol tinyint(3) unsigned, + _ip_tos tinyint(3) unsigned, + _ip_ttl tinyint(3) unsigned, + _ip_totlen smallint(5) unsigned, + _ip_ihl tinyint(3) unsigned, + _ip_csum smallint(5) unsigned, + _ip_id smallint(5) unsigned, + _ip_fragoff smallint(5) unsigned + ) RETURNS int(10) unsigned +SQL SECURITY INVOKER +NOT DETERMINISTIC +READS SQL DATA +BEGIN + INSERT INTO ulog2 (oob_time_sec, oob_time_usec, oob_prefix, oob_mark, oob_in, oob_out, + ip_saddr, ip_daddr, ip_protocol, ip_tos, ip_ttl, ip_totlen, ip_ihl, + ip_csum, ip_id, ip_fragoff ) VALUES + (_oob_time_sec, _oob_time_usec, _oob_prefix, _oob_mark, _oob_in, _oob_out, + _ip_saddr, _ip_daddr, _ip_protocol, _ip_tos, _ip_ttl, _ip_totlen, _ip_ihl, + _ip_csum, _ip_id, _ip_fragoff); + RETURN LAST_INSERT_ID(); +END +$$ + + +delimiter $$ +DROP PROCEDURE IF EXISTS PACKET_ADD_TCP_FULL; +CREATE PROCEDURE PACKET_ADD_TCP_FULL( + IN `id` int(10) unsigned, + IN `_sport` smallint(5) unsigned, + IN `_dport` smallint(5) unsigned, + IN `_seq` int(10) unsigned, + IN `_ackseq` int(10) unsigned, + IN `_window` smallint(5) unsigned, + IN `_urg` tinyint(4), + IN `_urgp` smallint(5) unsigned, + IN `_ack` tinyint(4), + IN `_psh` tinyint(4), + IN `_rst` tinyint(4), + IN `_syn` tinyint(4), + IN `_fin` tinyint(4) + ) +BEGIN + INSERT INTO tcp (_tcp_id, tcp_sport, tcp_dport, tcp_seq, tcp_ackseq, tcp_window, tcp_urg, tcp_urgp, tcp_ack, tcp_psh, tcp_rst, tcp_syn, tcp_fin) VALUES + (id, _sport, _dport, _seq, _ackseq, _window, _urg, _urgp, _ack, _psh, _rst, _syn, _fin); +END +$$ + +delimiter $$ +DROP PROCEDURE IF EXISTS PACKET_ADD_TCP; +CREATE PROCEDURE PACKET_ADD_TCP( + IN `id` int(10) unsigned, + IN `_sport` smallint(5) unsigned, + IN `_dport` smallint(5) unsigned + ) +BEGIN + INSERT INTO tcp (_tcp_id, tcp_sport, tcp_dport) VALUES (id, _sport, _dport); +END +$$ + +delimiter $$ +DROP PROCEDURE IF EXISTS PACKET_ADD_UDP; +CREATE PROCEDURE PACKET_ADD_UDP( + IN `id` int(10) unsigned, + IN `_sport` smallint(5) unsigned, + IN `_dport` smallint(5) unsigned, + IN `_len` smallint(5) unsigned + ) +BEGIN + INSERT INTO udp (_udp_id, udp_sport, udp_dport, udp_len) VALUES + (id, _sport, _dport, _len); +END +$$ + +delimiter $$ +DROP PROCEDURE IF EXISTS PACKET_ADD_ICMP; +CREATE PROCEDURE PACKET_ADD_ICMP( + IN `id` int(10) unsigned, + IN `_icmp_type` tinyint(3) unsigned, + IN `_icmp_code` tinyint(3) unsigned, + IN `_icmp_echoid` smallint(5) unsigned, + IN `_icmp_echoseq` smallint(5) unsigned, + IN `_icmp_gateway` int(10) unsigned, + IN `_icmp_fragmtu` smallint(5) unsigned + ) +BEGIN + INSERT INTO icmp (_icmp_id, icmp_type, icmp_code, icmp_echoid, icmp_echoseq, + icmp_gateway, icmp_fragmtu) VALUES + (id, _icmp_type, _icmp_code, _icmp_echoid, _icmp_echoseq, + _icmp_gateway, _icmp_fragmtu); + +END +$$ + + +delimiter $$ +DROP PROCEDURE IF EXISTS PACKET_ADD_MAC; +CREATE PROCEDURE PACKET_ADD_MAC( + IN `id` int(10) unsigned, + IN `_saddr` binary(12), + IN `_daddr` binary(12), + IN `_protocol` smallint(5) + ) +BEGIN + INSERT INTO mac (_mac_id, mac_saddr, mac_daddr, mac_protocol) VALUES + (id, _saddr, _daddr, _protocol); +END +$$ + +delimiter $$ +DROP PROCEDURE IF EXISTS INSERT_PACKET_FULL; +CREATE PROCEDURE INSERT_PACKET_FULL( + IN `_oob_time_sec` int(10) unsigned, + IN `_oob_time_usec` int(10) unsigned, + IN `_oob_prefix` varchar(32), + IN `_oob_mark` int(10) unsigned, + IN `_oob_in` varchar(32), + IN `_oob_out` varchar(32), + IN `_ip_saddr` int(16), + IN `_ip_daddr` int(16), + IN `_ip_protocol` tinyint(3) unsigned, + IN `_ip_tos` tinyint(3) unsigned, + IN `_ip_ttl` tinyint(3) unsigned, + IN `_ip_totlen` smallint(5) unsigned, + IN `_ip_ihl` tinyint(3) unsigned, + IN `_ip_csum` smallint(5) unsigned, + IN `_ip_id` smallint(5) unsigned, + IN `_ip_fragoff` smallint(5) unsigned, + IN `tcp_sport` smallint(5) unsigned, + IN `tcp_dport` smallint(5) unsigned, + IN `tcp_seq` int(10) unsigned, + IN `tcp_ackseq` int(10) unsigned, + IN `tcp_window` smallint(5) unsigned, + IN `tcp_urg` tinyint(4), + IN `tcp_urgp` smallint(5) unsigned, + IN `tcp_ack` tinyint(4), + IN `tcp_psh` tinyint(4), + IN `tcp_rst` tinyint(4), + IN `tcp_syn` tinyint(4), + IN `tcp_fin` tinyint(4), + IN `udp_sport` smallint(5) unsigned, + IN `udp_dport` smallint(5) unsigned, + IN `udp_len` smallint(5) unsigned, + IN `icmp_type` tinyint(3) unsigned, + IN `icmp_code` tinyint(3) unsigned, + IN `icmp_echoid` smallint(5) unsigned, + IN `icmp_echoseq` smallint(5) unsigned, + IN `icmp_gateway` int(10) unsigned, + IN `icmp_fragmtu` smallint(5) unsigned +-- IN `mac_saddr` binary(12), +-- IN `mac_daddr` binary(12), +-- IN `mac_protocol` smallint(5) + ) +BEGIN + SET @lastid = INSERT_IP_PACKET_FULL(_oob_time_sec, _oob_time_usec, _oob_prefix, + _oob_mark, _oob_in, _oob_out, _ip_saddr, + _ip_daddr, _ip_protocol, _ip_tos, _ip_ttl, + _ip_totlen, _ip_ihl, _ip_csum, _ip_id, + _ip_fragoff); + IF _ip_protocol = 6 THEN + CALL PACKET_ADD_TCP_FULL(@lastid, tcp_sport, tcp_dport, tcp_seq, tcp_ackseq, + tcp_window, tcp_urg, tcp_urgp, tcp_ack, tcp_psh, + tcp_rst, tcp_syn, tcp_fin); + ELSEIF _ip_protocol = 17 THEN + CALL PACKET_ADD_UDP(@lastid, udp_sport, udp_dport, udp_len); + ELSEIF _ip_protocol = 1 THEN + CALL PACKET_ADD_ICMP(@lastid, icmp_type, icmp_code, icmp_echoid, icmp_echoseq, + icmp_gateway, icmp_fragmtu); + END IF; +-- IF mac_protocol IS NOT NULL THEN +-- CALL PACKET_ADD_MAC(@lastid, mac_saddr, mac_daddr, mac_protocol); +-- END IF; +END +$$ + + +delimiter $$ +DROP PROCEDURE IF EXISTS PACKET_ADD_NUFW; +CREATE PROCEDURE PACKET_ADD_NUFW( + IN `id` int(10) unsigned, + IN `username` varchar(30), + IN `user_id` int(10) unsigned, + IN `client_os` varchar(100), + IN `client_app` varchar(256), + IN `socket` smallint(5) + ) +BEGIN + INSERT INTO nufw (_nufw_id, username, user_id, client_os, client_app, socket) VALUES + (id, username, user_id, client_os, client_app, socket); +END +$$ + +delimiter $$ +DROP PROCEDURE IF EXISTS INSERT_CT; +CREATE PROCEDURE INSERT_CT( + IN `_orig_ip_saddr` binary(16), + IN `_orig_ip_daddr` binary(16), + IN `_orig_ip_protocol` tinyint(3) unsigned, + IN `_orig_l4_sport` int(5), + IN `_orig_l4_dport` int(5), + IN `_orig_bytes` bigint, + IN `_orig_packets` bigint, + IN `_reply_ip_saddr` binary(16), + IN `_reply_ip_daddr` binary(16), + IN `_reply_ip_protocol` tinyint(3) unsigned, + IN `_reply_l4_sport` int(5), + IN `_reply_l4_dport` int(5), + IN `_reply_bytes` bigint, + IN `_reply_packets` bigint, + IN `_icmp_code` tinyint(3), + IN `_icmp_type` tinyint(3), + IN `_ct_mark` bigint, + IN `_flow_start_sec` int(10), + IN `_flow_start_usec` int(10), + IN `_flow_end_sec` int(10), + IN `_flow_end_usec` int(10) + ) +BEGIN + INSERT INTO ulog2_ct (orig_ip_saddr, orig_ip_daddr, orig_ip_protocol, + orig_l4_sport, orig_l4_dport, orig_bytes, orig_packets, + reply_ip_saddr, reply_ip_daddr, reply_ip_protocol, + reply_l4_sport, reply_l4_dport, reply_bytes, reply_packets, + icmp_code, icmp_type, ct_mark, + flow_start_sec, flow_start_usec, + flow_end_sec, flow_end_usec) + VALUES (_orig_ip_saddr, _orig_ip_daddr, _orig_ip_protocol, + _orig_l4_sport, _orig_l4_dport, _orig_bytes, _orig_packets, + _reply_ip_saddr, _reply_ip_daddr, _reply_ip_protocol, + _reply_l4_sport, _reply_l4_dport, _reply_bytes, _reply_packets, + _icmp_code, _icmp_type, _ct_mark, + _flow_start_sec, _flow_start_usec, + _flow_end_sec, _flow_end_usec); + +END +$$ + +delimiter ; + +-- suppressing packets +-- better use trigger ? +-- -> a trigger needs super-user access +-- -> triggers on delete does not affect drop tables +DROP PROCEDURE IF EXISTS DELETE_PACKET; +delimiter $$ +CREATE PROCEDURE DELETE_PACKET( + IN _packet_id bigint unsigned + ) +SQL SECURITY INVOKER +COMMENT 'Delete a packet (from ulog tables only)' +BEGIN + -- remember : table with most constraints first + DELETE FROM icmp WHERE icmp._icmp_id = _packet_id; + DELETE FROM tcp WHERE tcp._tcp_id = _packet_id; + DELETE FROM udp WHERE udp._udp_id = _packet_id; + DELETE FROM ulog2 WHERE ulog2._id = _packet_id; +END +$$ +delimiter ; + +DROP PROCEDURE IF EXISTS DELETE_CUSTOM_ONE; +delimiter $$ +-- XXX be careful with SQL injections !! +CREATE PROCEDURE DELETE_CUSTOM_ONE( + IN tname varchar(64), + IN tjoin varchar(64), + IN _id bigint + ) +SQL SECURITY INVOKER +COMMENT 'Delete packet in a custom table (specified at runtime) using a prepared query' +BEGIN + SET @l_sql = CONCAT('DELETE FROM ',@tname,' WHERE ',@tname,'.',@tfield,' = ',_id); + PREPARE delete_stmt FROM @l_sql; + EXECUTE delete_stmt; + DEALLOCATE PREPARE delete_stmt; +END +$$ +delimiter ; + +DROP PROCEDURE IF EXISTS DELETE_PACKET_FULL; +delimiter $$ +CREATE PROCEDURE DELETE_PACKET_FULL( + IN _packet_id bigint unsigned + ) +SQL SECURITY INVOKER +COMMENT 'Delete packet in all tables (including extensions)' +BEGIN + DECLARE tname varchar(64); + DECLARE tjoin varchar(64); + DECLARE l_last INT DEFAULT 0; + + DECLARE ext_csr CURSOR FOR + SELECT table_name,join_name FROM _extensions; + + DECLARE CONTINUE HANDLER FOR NOT FOUND SET l_last=1; + + OPEN ext_csr; + ext_loop:LOOP + FETCH ext_csr INTO tname,tjoin; + IF l_last THEN + LEAVE ext_loop; + END IF; + CALL DELETE_CUSTOM_ONE(tname,tjoin,_packet_id); + END LOOP ext_loop; + CLOSE ext_csr; + + CALL DELETE_PACKET(_packet_id); +END +$$ +delimiter ; + +-- suppressing tuples +DROP PROCEDURE IF EXISTS DELETE_CT_TUPLE; +delimiter $$ +CREATE PROCEDURE DELETE_CT_TUPLE( + IN _packet_id bigint unsigned + ) +SQL SECURITY INVOKER +COMMENT 'Delete a tuple from conntrack' +BEGIN + -- remember : table with most constraints first + DELETE FROM ct_icmp WHERE ct_icmp._icmp_id = _packet_id; + DELETE FROM ct_l4 WHERE ct_l4._l4_id = _packet_id; + DELETE FROM ct_tuple WHERE ct_tuple._tuple_id = _packet_id; +END +$$ + +delimiter ; + + +DROP PROCEDURE IF EXISTS DELETE_CT_FLOW; +delimiter $$ +CREATE PROCEDURE DELETE_CT_FLOW( + IN _ct_packet_id bigint unsigned + ) +SQL SECURITY INVOKER +COMMENT 'Delete a packet from the conntrack tables' +BEGIN + DELETE FROM ulog2_ct WHERE ulog2_ct._ct_id = _ct_packet_id; +END +$$ +delimiter ; + +-- DROP TRIGGER IF EXISTS _trigger_delete; +-- delimiter $$ +-- CREATE TRIGGER _trigger_delete BEFORE DELETE ON ulog2 +-- FOR EACH ROW +-- BEGIN +-- DELETE FROM icmp WHERE icmp._icmp_id = _packet_id; +-- DELETE FROM tcp WHERE tcp._tcp_id = _packet_id; +-- DELETE FROM udp WHERE udp._udp_id = _packet_id; +-- END +-- $$ +-- delimiter ; + + +-- Tables compression + +DROP PROCEDURE IF EXISTS COMPRESS_TABLES; +delimiter $$ +CREATE PROCEDURE COMPRESS_TABLES( + ) +SQL SECURITY INVOKER +COMMENT 'Try to remove dead entries and call OPTIMIZE for each table' +BEGIN + -- look for packets in table _tcp and not in table ulog2 + DELETE FROM tcp WHERE _tcp_id NOT IN (SELECT _id FROM ulog2); + -- XXX note: could be rewritten (need to see what is more efficient) as: + -- DELETE FROM tcp WHERE _tcp_id IN (SELECT tcp._tcp_id FROM tcp LEFT OUTER JOIN ulog2 ON (tcp._tcp_id = ulog2._id) WHERE ulog2._id IS NULL); + DELETE FROM mac WHERE _mac_id NOT IN (SELECT _id FROM ulog2); + DELETE FROM udp WHERE _udp_id NOT IN (SELECT _id FROM ulog2); + DELETE FROM icmp WHERE _icmp_id NOT IN (SELECT _id FROM ulog2); + -- look for packets in table ulog2 with proto tcp (or ipv6 ?) and not in table tcp + DELETE FROM ulog2 WHERE ulog2.ip_protocol = '6' AND _id NOT IN (SELECT _tcp_id FROM tcp); + DELETE FROM ulog2 WHERE ulog2.ip_protocol = '17' AND _id NOT IN (SELECT _udp_id FROM udp); + DELETE FROM ulog2 WHERE ulog2.ip_protocol = '2' AND _id NOT IN (SELECT _icmp_id FROM icmp); + -- finally, call optimize to reclaim unused space and defragment the data file + OPTIMIZE TABLE ulog2, mac, tcp, udp, icmp, ulog2_ct; +END +$$ +delimiter ; + +DROP PROCEDURE IF EXISTS ANALYZE_TABLES; +delimiter $$ +CREATE PROCEDURE ANALYZE_TABLES( + ) +SQL SECURITY INVOKER +COMMENT 'ANALYZE all ulog2 tables' +BEGIN + ANALYZE TABLE ulog2, mac, tcp, udp, icmp, ulog2_ct; +END +$$ +delimiter ; + +-- Add foreign keys to tables +CALL ULOG2_ADD_FOREIGN_KEYS(); +
doc/pgsql-ulogd2.sql new file +357 -0
357 lines not on this page · raw message
include/ulogd/db.h +7 -1
@@ -41,13 +41,19 @@ struct db_instance { { \ .key = "connect_timeout", \ .type = CONFIG_TYPE_INT, \ + }, \ + { \ + .key = "procedure", \ + .type = CONFIG_TYPE_STRING, \ + .options = CONFIG_OPT_MANDATORY, \ } -#define DB_CE_NUM 4 +#define DB_CE_NUM 5#define table_ce (x) (x->ces[0])#define reconnect_ce (x) (x->ces[1])#define asstring_ce (x) (x->ces[2])#define timeout_ce (x) (x->ces[3]) +#define procedure_ce (x) (x->ces[4])void ulogd_db_signal (struct ulogd_pluginstance *upi,int signal);int ulogd_db_start (struct ulogd_pluginstance *upi);
ulogd.conf.in +7 -0
@@ -70,3 +70,10 @@ sync=1 file="/var/log/ulogd_oprint.log" sync=1 +[mysql1] +db="nulog" +host="localhost" +user="nupik" +table="ulog" +pass="changeme" +procedure="INSERT_PACKET_FULL"
util/db.c +4 -19
@@ -7,6 +7,7 @@* Portions (C) 2001 Alex Janssen <[email protected]>, * (C) 2005 Sven Schuster <[email protected]>, * (C) 2005 Jozsef Kadlecsik <[email protected]> +* (C) 2008 Eric Leblond <[email protected]> * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2
@@ -65,6 +66,7 @@ static int sql_createstmt(struct ulogd_pluginstance *upi)char *underscore;int i;char *table =table_ce (upi->config_kset).u.string; +char *procedure =procedure_ce (upi->config_kset).u.string;if (mi->stmt)free (mi->stmt);
@@ -88,25 +90,8 @@ static int sql_createstmt(struct ulogd_pluginstance *upi)return -ENOMEM ; } -if (mi->schema) -sprintf (mi->stmt,"insert into %s.%s (" , mi->schema, table); -else -sprintf (mi->stmt,"insert into %s (" , table); - mi->stmt_val = mi->stmt +strlen (mi->stmt); - -for (i =0 ; i < upi->input.num_keys; i++) { -if (upi->input.keys[i].flags &ULOGD_KEYF_INACTIVE ) -continue ; - -strncpy (buf, upi->input.keys[i].name,ULOGD_MAX_KEYLEN ); -while ((underscore =strchr (buf,'.' ))) - *underscore ='_' ; -sprintf (mi->stmt_val,"%s," , buf); - mi->stmt_val = mi->stmt +strlen (mi->stmt); - } - *(mi->stmt_val -1 ) =')' ; +sprintf (mi->stmt,"CALL %s(" , procedure); -sprintf (mi->stmt_val," values (" ); mi->stmt_val = mi->stmt +strlen (mi->stmt);ulogd_log (ULOGD_DEBUG ,"stmt='%s'\n" , mi->stmt);
@@ -285,7 +270,7 @@ static int __interp_db(struct ulogd_pluginstance *upi) tmpstr =inet_ntoa (addr); di->driver->escape_string (upi, di->stmt_ins, tmpstr,strlen (tmpstr)); - di->stmt_ins = di->stmt +strlen (di->stmt); + di->stmt_ins = di->stmt +strlen (di->stmt);sprintf (di->stmt_ins,"'," );break ; }
Signature · 1 line
1.5.2.5
cover [ULOGD RFC PATCH 0/34]
Hello,
This patchset contains patches for ulogd2 from Pierre Chifflier and I.
As discussed during Netfilter workshop, the goal of this patchset is to provide a new and modern SQL logging schema. Some colateral patchs are present in the patchset due to the state of Ulogd2. As stated by Holger, people using ulogd2 now are early adopters and we tried to improve usability of ulogd2. For example, we've added a --info switch to ulogd2 to be able to display option of a plugin.
But, the main work is on SQL logging. Ulogd 1.x schema was really bad. It lacks index and the way data are stored (one big line per entry full of NULL fields) is not efficient for databases.
Thus, we propose new schemas for MySQL and PGsql which use advanced database feature without complication on developper side. In fact, the SQL related C code did not change very much. The main change is the use of a call to a SQL function instead of using a SQL query. The advantage of doing this is to hide the complexity of the database to developpers and let people knowing databases work on their side without bothering us.
I will finished this mail by a description of the avantages of the new schema. It uses a set of small dedicated tables (a TCP tables for example). From an SQL point of view this is more efficient as we limit the number of NULL fields (storage of empty datas has a cost). The schema has some SQL views (virtual table) and some of them provides an near complete backward compatility with the existing one.
One other advantage of the new schema is that extension (like nufw one) can used without changing anything for non-aware system.
This patchset should not conflict with Holger patchset (if NFCT related work is omitted). I can do the merge work if some is needed, just let me know.
BR,
--
Eric Leblond <[email protected]>
INL: http://www.inl.fr/
PATCH 28/34 Update SQL schema
In reply to Eric LeblondFrom: Pierre Chifflier <[email protected]>
This patch adds oob_family to the schema. Thus it is now possible to easily select IPv4 or IPv6 entries in the database. This patch also explicitly selects fields to create view.
---
:100644 100644 ccbb8e8... a146d87... M doc/mysql-ulogd2.sql
:100644 100644 d6e6efd... a488c11... M doc/pgsql-ulogd2.sql
doc/mysql-ulogd2.sql | 179 ++++++++++++++++++++++++++++++++------------------ doc/pgsql-ulogd2.sql | 66 ++++++++++++++++--- 2 files changed, 170 insertions(+), 75 deletions(-)
doc/mysql-ulogd2.sql +114 -65
@@ -14,7 +14,7 @@ CREATE TABLE `_format` ( `version` int(4) NOT NULL ) ENGINE=INNODB; -INSERT INTO _format (version) VALUES (3); +INSERT INTO _format (version) VALUES (4); -- this table could be used to know which user-defined tables are linked -- to ulog
@@ -33,9 +33,6 @@ DROP TABLE IF EXISTS `udp`; DROP TABLE IF EXISTS `icmp`; DROP TABLE IF EXISTS `nufw`; DROP TABLE IF EXISTS `ulog2_ct`; -DROP TABLE IF EXISTS `ct_tuple`; -DROP TABLE IF EXISTS `ct_l4`; -DROP TABLE IF EXISTS `ct_icmp`; DROP TABLE IF EXISTS `ulog2`; CREATE TABLE `ulog2` (
@@ -46,6 +43,7 @@ CREATE TABLE `ulog2` ( `oob_mark` int(10) unsigned default NULL, `oob_in` varchar(32) default NULL, `oob_out` varchar(32) default NULL, + `oob_family` tinyint(3) unsigned default NULL, `ip_saddr` binary(16) default NULL, `ip_daddr` binary(16) default NULL, `ip_protocol` tinyint(3) unsigned default NULL,
@@ -61,9 +59,10 @@ CREATE TABLE `ulog2` ( ) ENGINE=INNODB COMMENT='Table for IP packets'; ALTER TABLE ulog2 ADD KEY `index_id` (`_id`); -ALTER TABLE ulog2 ADD KEY `timestamp` (`timestamp`); +ALTER TABLE ulog2 ADD KEY `oob_family` (`oob_family`); ALTER TABLE ulog2 ADD KEY `ip_saddr` (`ip_saddr`); ALTER TABLE ulog2 ADD KEY `ip_daddr` (`ip_daddr`); +ALTER TABLE ulog2 ADD KEY `timestamp` (`timestamp`); -- This index does not seem very useful: -- ALTER TABLE ulog2 ADD KEY `oob_time_sec` (`oob_time_sec`);
@@ -146,9 +145,51 @@ CREATE SQL SECURITY INVOKER VIEW `view_icmp` AS -- ulog view DROP VIEW IF EXISTS `ulog`; +-- CREATE SQL SECURITY INVOKER VIEW `ulog` AS +-- SELECT * FROM ulog2 INNER JOIN tcp ON ulog2._id = tcp._tcp_id INNER JOIN udp ON ulog2._id = udp._udp_id +-- INNER JOIN icmp ON ulog2._id = icmp._icmp_id INNER JOIN mac ON ulog2._id = mac._mac_id; CREATE SQL SECURITY INVOKER VIEW `ulog` AS - SELECT * FROM ulog2 INNER JOIN tcp ON ulog2._id = tcp._tcp_id INNER JOIN udp ON ulog2._id = udp._udp_id - INNER JOIN icmp ON ulog2._id = icmp._icmp_id INNER JOIN mac ON ulog2._id = mac._mac_id; + SELECT _id, + oob_time_sec, + oob_time_usec, + oob_prefix, + oob_mark, + oob_in, + oob_out, + oob_family, + ip_saddr as ip_saddr_bin, + ip_daddr as ip_daddr_bin, + ip_protocol, + ip_tos, + ip_ttl, + ip_totlen, + ip_ihl, + ip_csum, + ip_id, + ip_fragoff, + tcp_sport, + tcp_dport, + tcp_seq, + tcp_ackseq, + tcp_window, + tcp_urg, + tcp_urgp, + tcp_ack, + tcp_psh, + tcp_rst, + tcp_syn, + tcp_fin, + udp_sport, + udp_dport, + udp_len, + icmp_type, + icmp_code, + icmp_echoid, + icmp_echoseq, + icmp_gateway, + icmp_fragmtu + FROM ulog2 INNER JOIN tcp ON ulog2._id = tcp._tcp_id INNER JOIN udp ON ulog2._id = udp._udp_id + INNER JOIN icmp ON ulog2._id = icmp._icmp_id INNER JOIN mac ON ulog2._id = mac._mac_id; -- shortcuts DROP VIEW IF EXISTS `view_tcp_quad`;
@@ -165,6 +206,7 @@ CREATE SQL SECURITY INVOKER VIEW `view_udp_quad` AS CREATE TABLE `ulog2_ct` ( `_ct_id` bigint unsigned NOT NULL auto_increment, + `oob_family` tinyint(3) unsigned default NULL, `orig_ip_saddr` binary(16) default NULL, `orig_ip_daddr` binary(16) default NULL, `orig_ip_protocol` tinyint(3) unsigned default NULL,
@@ -192,6 +234,7 @@ CREATE TABLE `ulog2_ct` ( ) ENGINE=INNODB; ALTER TABLE ulog2_ct ADD KEY `index_ct_id` (`_ct_id`); +ALTER TABLE ulog2_ct ADD KEY `oob_family` (`oob_family`); ALTER TABLE ulog2_ct ADD KEY `orig_ip_saddr` (`orig_ip_saddr`); ALTER TABLE ulog2_ct ADD KEY `orig_ip_daddr` (`orig_ip_daddr`); ALTER TABLE ulog2_ct ADD KEY `orig_ip_protocol` (`orig_ip_protocol`);
@@ -211,15 +254,16 @@ ALTER TABLE ulog2_ct ADD KEY `reply_tuple` (`reply_ip_saddr`, `reply_ip_daddr`, DROP VIEW IF EXISTS `conntrack`; CREATE SQL SECURITY INVOKER VIEW `conntrack` AS SELECT _ct_id, - orig_ip_saddr, - orig_ip_daddr, + oob_family, + orig_ip_saddr AS orig_ip_saddr_raw, + orig_ip_daddr AS orig_ip_daddr_raw, orig_ip_protocol, orig_l4_sport, orig_l4_dport, orig_bytes AS orig_raw_pktlen, orig_packets AS orig_raw_pktcount, - reply_ip_saddr, - reply_ip_daddr, + reply_ip_saddr AS reply_ip_saddr_bin, + reply_ip_daddr AS reply_ip_daddr_bin, reply_ip_protocol, reply_l4_sport, reply_l4_dport,
@@ -321,17 +365,18 @@ CREATE FUNCTION INSERT_IP_PACKET( _oob_mark int(10) unsigned, _oob_in varchar(32), _oob_out varchar(32), - _ip_saddr int(16), - _ip_daddr int(16), + _oob_family tinyint(3) unsigned, + _ip_saddr binary(16), + _ip_daddr binary(16), _ip_protocol tinyint(3) unsigned ) RETURNS bigint unsigned SQL SECURITY INVOKER NOT DETERMINISTIC READS SQL DATA BEGIN - INSERT INTO ulog2 (oob_time_sec, oob_time_usec, oob_prefix, oob_mark, oob_in, oob_out, + INSERT INTO ulog2 (oob_time_sec, oob_time_usec, oob_prefix, oob_mark, oob_in, oob_out, oob_family, ip_saddr, ip_daddr, ip_protocol) VALUES - (_oob_time_sec, _oob_time_usec, _oob_prefix, _oob_mark, _oob_in, _oob_out, + (_oob_time_sec, _oob_time_usec, _oob_prefix, _oob_mark, _oob_in, _oob_out, oob_family, _ip_saddr, _ip_daddr, _ip_protocol); RETURN LAST_INSERT_ID(); END
@@ -346,8 +391,9 @@ CREATE FUNCTION INSERT_IP_PACKET_FULL( _oob_mark int(10) unsigned, _oob_in varchar(32), _oob_out varchar(32), - _ip_saddr int(16), - _ip_daddr int(16), + _oob_family tinyint(3) unsigned, + _ip_saddr binary(16), + _ip_daddr binary(16), _ip_protocol tinyint(3) unsigned, _ip_tos tinyint(3) unsigned, _ip_ttl tinyint(3) unsigned,
@@ -361,10 +407,10 @@ SQL SECURITY INVOKER NOT DETERMINISTIC READS SQL DATA BEGIN - INSERT INTO ulog2 (oob_time_sec, oob_time_usec, oob_prefix, oob_mark, oob_in, oob_out, + INSERT INTO ulog2 (oob_time_sec, oob_time_usec, oob_prefix, oob_mark, oob_in, oob_out, oob_family, ip_saddr, ip_daddr, ip_protocol, ip_tos, ip_ttl, ip_totlen, ip_ihl, ip_csum, ip_id, ip_fragoff ) VALUES - (_oob_time_sec, _oob_time_usec, _oob_prefix, _oob_mark, _oob_in, _oob_out, + (_oob_time_sec, _oob_time_usec, _oob_prefix, _oob_mark, _oob_in, _oob_out, oob_family, _ip_saddr, _ip_daddr, _ip_protocol, _ip_tos, _ip_ttl, _ip_totlen, _ip_ihl, _ip_csum, _ip_id, _ip_fragoff); RETURN LAST_INSERT_ID();
@@ -457,54 +503,56 @@ END $$ delimiter $$ -DROP PROCEDURE IF EXISTS INSERT_PACKET_FULL; -CREATE PROCEDURE INSERT_PACKET_FULL( - IN `_oob_time_sec` int(10) unsigned, - IN `_oob_time_usec` int(10) unsigned, - IN `_oob_prefix` varchar(32), - IN `_oob_mark` int(10) unsigned, - IN `_oob_in` varchar(32), - IN `_oob_out` varchar(32), - IN `_ip_saddr` int(16), - IN `_ip_daddr` int(16), - IN `_ip_protocol` tinyint(3) unsigned, - IN `_ip_tos` tinyint(3) unsigned, - IN `_ip_ttl` tinyint(3) unsigned, - IN `_ip_totlen` smallint(5) unsigned, - IN `_ip_ihl` tinyint(3) unsigned, - IN `_ip_csum` smallint(5) unsigned, - IN `_ip_id` smallint(5) unsigned, - IN `_ip_fragoff` smallint(5) unsigned, - IN `tcp_sport` smallint(5) unsigned, - IN `tcp_dport` smallint(5) unsigned, - IN `tcp_seq` int(10) unsigned, - IN `tcp_ackseq` int(10) unsigned, - IN `tcp_window` smallint(5) unsigned, - IN `tcp_urg` tinyint(4), - IN `tcp_urgp` smallint(5) unsigned, - IN `tcp_ack` tinyint(4), - IN `tcp_psh` tinyint(4), - IN `tcp_rst` tinyint(4), - IN `tcp_syn` tinyint(4), - IN `tcp_fin` tinyint(4), - IN `udp_sport` smallint(5) unsigned, - IN `udp_dport` smallint(5) unsigned, - IN `udp_len` smallint(5) unsigned, - IN `icmp_type` tinyint(3) unsigned, - IN `icmp_code` tinyint(3) unsigned, - IN `icmp_echoid` smallint(5) unsigned, - IN `icmp_echoseq` smallint(5) unsigned, - IN `icmp_gateway` int(10) unsigned, - IN `icmp_fragmtu` smallint(5) unsigned --- IN `mac_saddr` binary(12), --- IN `mac_daddr` binary(12), --- IN `mac_protocol` smallint(5) - ) +DROP FUNCTION IF EXISTS INSERT_PACKET_FULL; +CREATE FUNCTION INSERT_PACKET_FULL( + _oob_time_sec int(10) unsigned, + _oob_time_usec int(10) unsigned, + _oob_prefix varchar(32), + _oob_mark int(10) unsigned, + _oob_in varchar(32), + _oob_out varchar(32), + _oob_family tinyint(3) unsigned, + _ip_saddr binary(16), + _ip_daddr binary(16), + _ip_protocol tinyint(3) unsigned, + _ip_tos tinyint(3) unsigned, + _ip_ttl tinyint(3) unsigned, + _ip_totlen smallint(5) unsigned, + _ip_ihl tinyint(3) unsigned, + _ip_csum smallint(5) unsigned, + _ip_id smallint(5) unsigned, + _ip_fragoff smallint(5) unsigned, + tcp_sport smallint(5) unsigned, + tcp_dport smallint(5) unsigned, + tcp_seq int(10) unsigned, + tcp_ackseq int(10) unsigned, + tcp_window smallint(5) unsigned, + tcp_urg tinyint(4), + tcp_urgp smallint(5) unsigned, + tcp_ack tinyint(4), + tcp_psh tinyint(4), + tcp_rst tinyint(4), + tcp_syn tinyint(4), + tcp_fin tinyint(4), + udp_sport smallint(5) unsigned, + udp_dport smallint(5) unsigned, + udp_len smallint(5) unsigned, + icmp_type tinyint(3) unsigned, + icmp_code tinyint(3) unsigned, + icmp_echoid smallint(5) unsigned, + icmp_echoseq smallint(5) unsigned, + icmp_gateway int(10) unsigned, + icmp_fragmtu smallint(5) unsigned +-- mac_saddr binary(12), +-- mac_daddr binary(12), +-- mac_protocol smallint(5) + ) RETURNS bigint unsigned +READS SQL DATA BEGIN SET @lastid = INSERT_IP_PACKET_FULL(_oob_time_sec, _oob_time_usec, _oob_prefix, - _oob_mark, _oob_in, _oob_out, _ip_saddr, - _ip_daddr, _ip_protocol, _ip_tos, _ip_ttl, - _ip_totlen, _ip_ihl, _ip_csum, _ip_id, + _oob_mark, _oob_in, _oob_out, _oob_family, + _ip_saddr, _ip_daddr, _ip_protocol, _ip_tos, + _ip_ttl, _ip_totlen, _ip_ihl, _ip_csum, _ip_id, _ip_fragoff); IF _ip_protocol = 6 THEN CALL PACKET_ADD_TCP_FULL(@lastid, tcp_sport, tcp_dport, tcp_seq, tcp_ackseq,
@@ -519,6 +567,7 @@ BEGIN -- IF mac_protocol IS NOT NULL THEN -- CALL PACKET_ADD_MAC(@lastid, mac_saddr, mac_daddr, mac_protocol); -- END IF; + RETURN @lastid; END $$
doc/pgsql-ulogd2.sql +56 -10
138 lines not on this page · raw message
Signature · 1 line
1.5.2.5
PATCH 12/34 Changed to show pcap file name when open failed.
In reply to Eric LeblondPort of Marius Tomaschewski work on ulogd.
---
:100644 100644 90f7e0a... 59b5d2c... M output/pcap/ulogd_output_PCAP.c
output/pcap/ulogd_output_PCAP.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-)
output/pcap/ulogd_output_PCAP.c +2 -1
@@ -204,7 +204,8 @@ static int append_create_outfile(struct ulogd_pluginstance *upi)if (!exist) { pi->of =fopen (filename,"w" );if (!pi->of) { -ulogd_log (ULOGD_ERROR ,"can't open pcap file: %s\n" , +ulogd_log (ULOGD_ERROR ,"can't open pcap file %s: %s\n" , + filename,strerror (errno));return -EPERM ; }
Signature · 1 line
1.5.2.5
PATCH 27/34 Free insertion function result (mysql)
In reply to Eric LeblondFrom: Pierre Chifflier <[email protected]>
Change from procedure to function in mysql schema adds the need to free MySQL result after request.
---
:100644 100644 800d79d... fd650bf... M output/mysql/ulogd_output_MYSQL.c
output/mysql/ulogd_output_MYSQL.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-)
output/mysql/ulogd_output_MYSQL.c +5 -0
@@ -231,6 +231,7 @@ static int execute_mysql(struct ulogd_pluginstance *upi, {struct mysql_instance *mi = (struct mysql_instance *) upi->private;int ret; +MYSQL_RES * result; ret =mysql_real_query (mi->dbh, stmt, len);if (ret) {
@@ -238,6 +239,10 @@ static int execute_mysql(struct ulogd_pluginstance *upi, mysql_error(mi->dbh));return -1 ; } + result =mysql_use_result (mi->dbh); +if (result) { +mysql_free_result (result); + }return 0 ; }
Signature · 1 line
1.5.2.5
PATCH 10/34 Add some missing line break.
In reply to Eric LeblondQuoted 1 line
From Marius Tomaschewski <[email protected]>
---
:100644 100644 563fc3b... 6990f8c... M output/ulogd_output_OPRINT.c
output/ulogd_output_OPRINT.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)
output/ulogd_output_OPRINT.c +2 -2
@@ -86,9 +86,9 @@ static int oprint_interp(struct ulogd_pluginstance *upi) HIPQUAD(ret->u.value.ui32));break ;case ULOGD_RET_NONE : -fprintf (opi->of,"<none>" ); +fprintf (opi->of,"<none>\n" );break ; -default :fprintf (opi->of,"default" ); +default :fprintf (opi->of,"default\n" ); } }if (upi->config_kset->ces[1 ].u.value !=0 )
Signature · 1 line
1.5.2.5
PATCH 9/34 For OPRINT, changed sighup_handler_print to fallback to continue using old descriptor on new file opening failure.
In reply to Eric LeblondBased on Marius Tomaschewski work.
---
:100644 100644 07364b6... 563fc3b... M output/ulogd_output_OPRINT.c
output/ulogd_output_OPRINT.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-)
output/ulogd_output_OPRINT.c +4 -1
@@ -118,15 +118,18 @@ static struct config_keyset oprint_kset = {static void sighup_handler_print (struct ulogd_pluginstance *upi,int signal) {struct oprint_priv *oi = (struct oprint_priv *) &upi->private; +FILE *old = oi->of;switch (signal) {case SIGHUP :ulogd_log (ULOGD_NOTICE ,"OPRINT: reopening logfile\n" ); -fclose (oi->of); oi->of =fopen (upi->config_kset->ces[0 ].u.string,"a" );if (!oi->of) {ulogd_log (ULOGD_ERROR ,"can't open PKTLOG: %s\n" ,strerror (errno)); + oi->of = old; + }else { +fclose (old); }break ;default :
Signature · 1 line
1.5.2.5
PATCH 8/34 Added explicit null termination of the hostname buffer
In reply to Eric LeblondBased on From Marius Tomaschewski <[email protected]> work on ulogd.
---
:100644 100644 41ebe5b... bf400d7... M filter/raw2packet/ulogd_raw2packet_LOCAL.c
filter/raw2packet/ulogd_raw2packet_LOCAL.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-)
filter/raw2packet/ulogd_raw2packet_LOCAL.c +1 -0
@@ -92,6 +92,7 @@ void _init(void) strerror(errno));exit (2 ); } + hostname[sizeof (hostname)-1 ] ='\0' ;/* strip off everything after first '.' */ if ((tmp =strchr (hostname,'.' ))) *tmp ='\0' ;
Signature · 1 line
1.5.2.5
PATCH 25/34 Set oob.family as VALID key.
In reply to Eric LeblondFrom: Eric leblond <[email protected]>
OOB_FAMILY output was not set by NFLOG because the key was not set as valid.
---
:100644 100644 6e5c830... be46fa2... M input/packet/ulogd_inppkt_NFLOG.c
input/packet/ulogd_inppkt_NFLOG.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-)
input/packet/ulogd_inppkt_NFLOG.c +1 -0
@@ -259,6 +259,7 @@ interp_packet(struct ulogd_pluginstance *upi, struct nflog_data *ldata)u_int32_t seq; ret[NFLOG_KEY_OOB_FAMILY ].u.value.ui8 =af_ce (upi->config_kset).u.value; + ret[NFLOG_KEY_OOB_FAMILY ].flags |=ULOGD_RETF_VALID ;if (ph) {/* FIXME */
Signature · 1 line
1.5.2.5
PATCH 26/34 Modify IPv6 parser to fill oob_family.
From: Eric leblond <[email protected]>
With this patch, BASE filter module is able fill oob_family when parsing IPv6 address.
---
:100644 100644 e837c62... 48f2993... M filter/raw2packet/ulogd_raw2packet_BASE.c
filter/raw2packet/ulogd_raw2packet_BASE.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-)
filter/raw2packet/ulogd_raw2packet_BASE.c +4 -0
@@ -795,6 +795,10 @@ static int _interp_ipv6hdr(struct ulogd_pluginstance *pi, u_int32_t len)if (fragment) goto out; + + ret[KEY_IP_PROTOCOL ].u.value.ui8 = curhdr; + ret[KEY_IP_PROTOCOL ].flags |=ULOGD_RETF_VALID ; +switch (curhdr) {case IPPROTO_TCP :_interp_tcp (pi, (void *)ipv6h + ptr, len);
Signature · 1 line
1.5.2.5
PATCH 11/34 Put O at the real end of the string.
In reply to Eric LeblondQuoted 1 line
From Marius Tomaschewski <[email protected]>
---
:100644 100644 7e417da... 2efc07d... M filter/ulogd_filter_PWSNIFF.c
filter/ulogd_filter_PWSNIFF.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)
filter/ulogd_filter_PWSNIFF.c +2 -2
@@ -123,7 +123,7 @@ static int interp_pwsniff(struct ulogd_pluginstance *pi)return 0 ; }strncpy ((char *) ret[0 ].u.value.ptr, (char *)begp, len); - *((char *)ret[0 ].u.value.ptr + len +1 ) ='\0' ; + *((char *)ret[0 ].u.value.ptr + len) ='\0' ; }if (pw_len) { ret[1 ].u.value.ptr = (char *)malloc (pw_len+1 );
@@ -133,7 +133,7 @@ static int interp_pwsniff(struct ulogd_pluginstance *pi)return 0 ; }strncpy ((char *)ret[1 ].u.value.ptr, (char *)pw_begp, pw_len); - *((char *)ret[1 ].u.value.ptr + pw_len +1 ) ='\0' ; + *((char *)ret[1 ].u.value.ptr + pw_len) ='\0' ; }return 0 ;
Signature · 1 line
1.5.2.5
PATCH 13/34 Display filename in the other error case.
In reply to Eric LeblondPort of Marius Tomaschewski work on ulogd.
---
:100644 100644 59b5d2c... 69656b1... M output/pcap/ulogd_output_PCAP.c
output/pcap/ulogd_output_PCAP.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-)
output/pcap/ulogd_output_PCAP.c +2 -1
@@ -217,7 +217,8 @@ static int append_create_outfile(struct ulogd_pluginstance *upi) }else { pi->of =fopen (filename,"a" );if (!pi->of) { -ulogd_log (ULOGD_ERROR ,"can't open pcap file: %s\n" , +ulogd_log (ULOGD_ERROR ,"can't open pcap file %s: %s\n" , + filename,strerror (errno));return -EPERM ; }
Signature · 1 line
1.5.2.5
PATCH 14/34 Request at least autoconf 2.50 (needed for large file support macro).
---
:100644 100644 b09c43b... 086e4cb... M configure.in
configure.in | 1 + 1 files changed, 1 insertions(+), 0 deletions(-)
configure.in +1 -0
@@ -1,4 +1,5 @@ dnl Process this file with autoconf to produce a configure script. +AC_PREQ(2.50) AC_INIT AM_INIT_AUTOMAKE(ulogd, 2.0.0beta2)
Signature · 1 line
1.5.2.5
PATCH 29/34 Fix some place were oob_family was used instead of _oob_family.
In reply to Eric LeblondFrom: Eric leblond <[email protected]>
This patch fixes some small typo in MySQL schema.
---
:100644 100644 a146d87... 6dc865a... M doc/mysql-ulogd2.sql
doc/mysql-ulogd2.sql | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)
doc/mysql-ulogd2.sql +2 -2
@@ -376,7 +376,7 @@ READS SQL DATA BEGIN INSERT INTO ulog2 (oob_time_sec, oob_time_usec, oob_prefix, oob_mark, oob_in, oob_out, oob_family, ip_saddr, ip_daddr, ip_protocol) VALUES - (_oob_time_sec, _oob_time_usec, _oob_prefix, _oob_mark, _oob_in, _oob_out, oob_family, + (_oob_time_sec, _oob_time_usec, _oob_prefix, _oob_mark, _oob_in, _oob_out, _oob_family, _ip_saddr, _ip_daddr, _ip_protocol); RETURN LAST_INSERT_ID(); END
@@ -410,7 +410,7 @@ BEGIN INSERT INTO ulog2 (oob_time_sec, oob_time_usec, oob_prefix, oob_mark, oob_in, oob_out, oob_family, ip_saddr, ip_daddr, ip_protocol, ip_tos, ip_ttl, ip_totlen, ip_ihl, ip_csum, ip_id, ip_fragoff ) VALUES - (_oob_time_sec, _oob_time_usec, _oob_prefix, _oob_mark, _oob_in, _oob_out, oob_family, + (_oob_time_sec, _oob_time_usec, _oob_prefix, _oob_mark, _oob_in, _oob_out, _oob_family, _ip_saddr, _ip_daddr, _ip_protocol, _ip_tos, _ip_ttl, _ip_totlen, _ip_ihl, _ip_csum, _ip_id, _ip_fragoff); RETURN LAST_INSERT_ID();
Signature · 1 line
1.5.2.5
PATCH 30/34 Convert SQL procedure to function call.
From: Pierre Chifflier <[email protected]>
SQL standard says a function has to be called with SELECT and not CALL. This patch modify code accordingly.
---
:100644 100644 4c4298c... d57ab6a... M util/db.c
util/db.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-)
util/db.c +1 -1
@@ -90,7 +90,7 @@ static int sql_createstmt(struct ulogd_pluginstance *upi)return -ENOMEM ; } -sprintf (mi->stmt,"CALL %s(" , procedure); +sprintf (mi->stmt,"SELECT %s(" , procedure); mi->stmt_val = mi->stmt +strlen (mi->stmt);
Signature · 1 line
1.5.2.5
PATCH RESEND 2/34 Do not propagate one conntrack event via 2 messages
In reply to Eric LeblondFrom: Eric leblond <[email protected]>
Ulogd2 was propagating through a stack 2 message for one single conntrack event. This patch provides a fall back to on message per event. It also uses an enum to improve code readability instead of direct access to array via numerical index.
---
:100644 100644 d3cd20c... bf6587d... M input/flow/ulogd_inpflow_NFCT.c
input/flow/ulogd_inpflow_NFCT.c | 236 ++++++++++++++++++++++++++++----------- 1 files changed, 168 insertions(+), 68 deletions(-)
input/flow/ulogd_inpflow_NFCT.c +168 -68
@@ -106,11 +106,101 @@ static struct config_keyset nfct_kset = {#define buckets_ce (x) (x->ces[3])#define maxentries_ce (x) (x->ces[4]) +enum nfct_keys { +NFCT_ORIG_IP_SADDR =0 , +NFCT_ORIG_IP_DADDR , +NFCT_ORIG_IP_PROTOCOL , +NFCT_ORIG_L4_SPORT , +NFCT_ORIG_L4_DPORT , +NFCT_ORIG_RAW_PKTLEN , +NFCT_ORIG_RAW_PKTCOUNT , +NFCT_REPLY_IP_SADDR , +NFCT_REPLY_IP_DADDR , +NFCT_REPLY_IP_PROTOCOL , +NFCT_REPLY_L4_SPORT , +NFCT_REPLY_L4_DPORT , +NFCT_REPLY_RAW_PKTLEN , +NFCT_REPLY_RAW_PKTCOUNT , +NFCT_ICMP_CODE , +NFCT_ICMP_TYPE , +NFCT_CT_MARK , +NFCT_CT_ID , +NFCT_FLOW_START_SEC , +NFCT_FLOW_START_USEC , +NFCT_FLOW_END_SEC , +NFCT_FLOW_END_USEC , +}; +static struct ulogd_key nfct_okeys[] = { { .type =ULOGD_RET_IPADDR , .flags =ULOGD_RETF_NONE , - .name ="ip.saddr" , + .name ="orig.ip.saddr" , + .ipfix = { + .vendor =IPFIX_VENDOR_IETF , + .field_id = IPFIX_sourceIPv4Address, + }, + }, + { + .type =ULOGD_RET_IPADDR , + .flags =ULOGD_RETF_NONE , + .name ="orig.ip.daddr" , + .ipfix = { + .vendor =IPFIX_VENDOR_IETF , + .field_id = IPFIX_destinationIPv4Address, + }, + }, + { + .type =ULOGD_RET_UINT8 , + .flags =ULOGD_RETF_NONE , + .name ="orig.ip.protocol" , + .ipfix = { + .vendor =IPFIX_VENDOR_IETF , + .field_id = IPFIX_protocolIdentifier, + }, + }, + { + .type =ULOGD_RET_UINT16 , + .flags =ULOGD_RETF_NONE , + .name ="orig.l4.sport" , + .ipfix = { + .vendor =IPFIX_VENDOR_IETF , + .field_id = IPFIX_sourceTransportPort, + }, + }, + { + .type =ULOGD_RET_UINT16 , + .flags =ULOGD_RETF_NONE , + .name ="orig.l4.dport" , + .ipfix = { + .vendor =IPFIX_VENDOR_IETF , + .field_id = IPFIX_destinationTransportPort, + }, + }, + { + .type =ULOGD_RET_UINT32 , + .flags =ULOGD_RETF_NONE , + .name ="orig.raw.pktlen" , + .ipfix = { + .vendor =IPFIX_VENDOR_IETF , + .field_id = IPFIX_octetTotalCount, +/* FIXME: this could also be octetDeltaCount */ + }, + }, + { + .type =ULOGD_RET_UINT32 , + .flags =ULOGD_RETF_NONE , + .name ="orig.raw.pktcount" , + .ipfix = { + .vendor =IPFIX_VENDOR_IETF , + .field_id = IPFIX_packetTotalCount, +/* FIXME: this could also be packetDeltaCount */ + }, + }, + { + .type =ULOGD_RET_IPADDR , + .flags =ULOGD_RETF_NONE , + .name ="reply.ip.saddr" , .ipfix = { .vendor =IPFIX_VENDOR_IETF , .field_id = IPFIX_sourceIPv4Address,
@@ -119,7 +209,7 @@ static struct ulogd_key nfct_okeys[] = { { .type = ULOGD_RET_IPADDR, .flags = ULOGD_RETF_NONE, - .name = "ip.daddr", + .name = "reply.ip.daddr", .ipfix = { .vendor = IPFIX_VENDOR_IETF, .field_id = IPFIX_destinationIPv4Address,
@@ -128,7 +218,7 @@ static struct ulogd_key nfct_okeys[] = { { .type = ULOGD_RET_UINT8, .flags = ULOGD_RETF_NONE, - .name = "ip.protocol", + .name = "reply.ip.protocol", .ipfix = { .vendor = IPFIX_VENDOR_IETF, .field_id = IPFIX_protocolIdentifier,
@@ -137,7 +227,7 @@ static struct ulogd_key nfct_okeys[] = { { .type = ULOGD_RET_UINT16, .flags = ULOGD_RETF_NONE, - .name = "l4.sport", + .name = "reply.l4.sport", .ipfix = { .vendor = IPFIX_VENDOR_IETF, .field_id = IPFIX_sourceTransportPort,
@@ -146,7 +236,7 @@ static struct ulogd_key nfct_okeys[] = { { .type = ULOGD_RET_UINT16, .flags = ULOGD_RETF_NONE, - .name = "l4.dport", + .name = "reply.l4.dport", .ipfix = { .vendor = IPFIX_VENDOR_IETF, .field_id = IPFIX_destinationTransportPort,
@@ -155,7 +245,7 @@ static struct ulogd_key nfct_okeys[] = { { .type = ULOGD_RET_UINT32, .flags = ULOGD_RETF_NONE, - .name = "raw.pktlen", + .name = "reply.raw.pktlen", .ipfix = { .vendor = IPFIX_VENDOR_IETF, .field_id = IPFIX_octetTotalCount,
@@ -165,7 +255,7 @@ static struct ulogd_key nfct_okeys[] = { { .type = ULOGD_RET_UINT32, .flags = ULOGD_RETF_NONE, - .name = "raw.pktcount", + .name = "reply.raw.pktcount", .ipfix = { .vendor = IPFIX_VENDOR_IETF, .field_id = IPFIX_packetTotalCount,
@@ -244,11 +334,6 @@ static struct ulogd_key nfct_okeys[] = { .field_id = IPFIX_flowEndSeconds, }, }, - { - .type = ULOGD_RET_BOOL, - .flags = ULOGD_RETF_NONE, - .name = "dir", - }, };static struct ct_htable *htable_alloc (int htable_size,int prealloc)
@@ -364,93 +449,108 @@ static struct ct_timestamp *ct_hash_get(struct ct_htable *htable, uint32_t id)return ct; } -static int propagate_ct_flow (struct ulogd_pluginstance *upi, -struct nfct_conntrack *ct, -unsigned int flags, -int dir, -struct ct_timestamp *ts) +static int propagate_ct (struct ulogd_pluginstance *upi, +struct nfct_conntrack *ct, +unsigned int flags, +struct ct_timestamp *ts) {struct ulogd_key *ret = upi->output.keys; +int dir; + + dir =NFCT_DIR_ORIGINAL ; + ret[NFCT_ORIG_IP_SADDR ].u.value.ui32 =htonl (ct->tuple[dir].src.v4); + ret[NFCT_ORIG_IP_SADDR ].flags |=ULOGD_RETF_VALID ; - ret[0 ].u.value.ui32 =htonl (ct->tuple[dir].src.v4); - ret[0 ].flags |=ULOGD_RETF_VALID ; - - ret[1 ].u.value.ui32 =htonl (ct->tuple[dir].dst.v4); - ret[1 ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_ORIG_IP_DADDR ].u.value.ui32 =htonl (ct->tuple[dir].dst.v4); + ret[NFCT_ORIG_IP_DADDR ].flags |=ULOGD_RETF_VALID ; - ret[2 ].u.value.ui8 = ct->tuple[dir].protonum; - ret[2 ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_ORIG_IP_PROTOCOL ].u.value.ui8 = ct->tuple[dir].protonum; + ret[NFCT_ORIG_IP_PROTOCOL ].flags |=ULOGD_RETF_VALID ; -switch (ct->tuple[1 ].protonum) { +switch (ct->tuple[dir].protonum) {case IPPROTO_TCP :case IPPROTO_UDP :case IPPROTO_SCTP :/* FIXME: DCCP */ - ret[3 ].u.value.ui16 =htons (ct->tuple[dir].l4src.tcp.port); - ret[3 ].flags |=ULOGD_RETF_VALID ; - ret[4 ].u.value.ui16 =htons (ct->tuple[dir].l4dst.tcp.port); - ret[4 ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_ORIG_L4_SPORT ].u.value.ui16 =htons (ct->tuple[dir].l4src.tcp.port); + ret[NFCT_ORIG_L4_SPORT ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_ORIG_L4_DPORT ].u.value.ui16 =htons (ct->tuple[dir].l4dst.tcp.port); + ret[NFCT_ORIG_L4_DPORT ].flags |=ULOGD_RETF_VALID ;break ;case IPPROTO_ICMP : - ret[7 ].u.value.ui8 = ct->tuple[dir].l4src.icmp.code; - ret[7 ].flags |=ULOGD_RETF_VALID ; - ret[8 ].u.value.ui8 = ct->tuple[dir].l4src.icmp.type; - ret[8 ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_ICMP_CODE ].u.value.ui8 = ct->tuple[dir].l4src.icmp.code; + ret[NFCT_ICMP_CODE ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_ICMP_TYPE ].u.value.ui8 = ct->tuple[dir].l4src.icmp.type; + ret[NFCT_ICMP_TYPE ].flags |=ULOGD_RETF_VALID ;break ; } -if ((dir ==NFCT_DIR_ORIGINAL && flags &NFCT_COUNTERS_ORIG ) || - (dir ==NFCT_DIR_REPLY && flags &NFCT_COUNTERS_RPLY )) { - ret[5 ].u.value.ui64 = ct->counters[dir].bytes; - ret[5 ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_ORIG_RAW_PKTLEN ].u.value.ui64 = ct->counters[dir].bytes; + ret[NFCT_ORIG_RAW_PKTLEN ].flags |=ULOGD_RETF_VALID ; + + ret[NFCT_ORIG_RAW_PKTCOUNT ].u.value.ui64 = ct->counters[dir].packets; + ret[NFCT_ORIG_RAW_PKTCOUNT ].flags |=ULOGD_RETF_VALID ; + + dir =NFCT_DIR_REPLY ; + ret[NFCT_REPLY_IP_SADDR ].u.value.ui32 =htonl (ct->tuple[dir].src.v4); + ret[NFCT_REPLY_IP_SADDR ].flags |=ULOGD_RETF_VALID ; - ret[6 ].u.value.ui64 = ct->counters[dir].packets; - ret[6 ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_REPLY_IP_DADDR ].u.value.ui32 =htonl (ct->tuple[dir].dst.v4); + ret[NFCT_REPLY_IP_DADDR ].flags |=ULOGD_RETF_VALID ; + + ret[NFCT_REPLY_IP_PROTOCOL ].u.value.ui8 = ct->tuple[dir].protonum; + ret[NFCT_REPLY_IP_PROTOCOL ].flags |=ULOGD_RETF_VALID ; + +switch (ct->tuple[dir].protonum) { +case IPPROTO_TCP : +case IPPROTO_UDP : +case IPPROTO_SCTP : +/* FIXME: DCCP */ + ret[NFCT_REPLY_L4_SPORT ].u.value.ui16 =htons (ct->tuple[dir].l4src.tcp.port); + ret[NFCT_REPLY_L4_SPORT ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_REPLY_L4_DPORT ].u.value.ui16 =htons (ct->tuple[dir].l4dst.tcp.port); + ret[NFCT_REPLY_L4_DPORT ].flags |=ULOGD_RETF_VALID ; +break ; +case IPPROTO_ICMP : + ret[NFCT_ICMP_CODE ].u.value.ui8 = ct->tuple[dir].l4src.icmp.code; + ret[NFCT_ICMP_CODE ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_ICMP_TYPE ].u.value.ui8 = ct->tuple[dir].l4src.icmp.type; + ret[NFCT_ICMP_TYPE ].flags |=ULOGD_RETF_VALID ; +break ; } + ret[NFCT_REPLY_RAW_PKTLEN ].u.value.ui64 = ct->counters[dir].bytes; + ret[NFCT_REPLY_RAW_PKTLEN ].flags |=ULOGD_RETF_VALID ; + + ret[NFCT_REPLY_RAW_PKTCOUNT ].u.value.ui64 = ct->counters[dir].packets; + ret[NFCT_REPLY_RAW_PKTCOUNT ].flags |=ULOGD_RETF_VALID ; +if (flags &NFCT_MARK ) { - ret[9 ].u.value.ui32 = ct->mark; - ret[9 ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_CT_MARK ].u.value.ui32 = ct->mark; + ret[NFCT_CT_MARK ].flags |=ULOGD_RETF_VALID ; }if (flags &NFCT_ID ) { - ret[10 ].u.value.ui32 = ct->id; - ret[10 ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_CT_ID ].u.value.ui32 = ct->id; + ret[NFCT_CT_ID ].flags |=ULOGD_RETF_VALID ; }if (ts) { - ret[11 ].u.value.ui32 = ts->time[START ].tv_sec; - ret[11 ].flags |=ULOGD_RETF_VALID ; - ret[12 ].u.value.ui32 = ts->time[START ].tv_usec; - ret[12 ].flags |=ULOGD_RETF_VALID ; - ret[13 ].u.value.ui32 = ts->time[STOP ].tv_sec; - ret[13 ].flags |=ULOGD_RETF_VALID ; - ret[14 ].u.value.ui32 = ts->time[STOP ].tv_usec; - ret[14 ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_FLOW_START_SEC ].u.value.ui32 = ts->time[START ].tv_sec; + ret[NFCT_FLOW_START_SEC ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_FLOW_START_USEC ].u.value.ui32 = ts->time[START ].tv_usec; + ret[NFCT_FLOW_START_USEC ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_FLOW_END_SEC ].u.value.ui32 = ts->time[STOP ].tv_sec; + ret[NFCT_FLOW_END_SEC ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_FLOW_END_USEC ].u.value.ui32 = ts->time[STOP ].tv_usec; + ret[NFCT_FLOW_END_USEC ].flags |=ULOGD_RETF_VALID ; } - ret[15 ].u.value.b = (dir ==NFCT_DIR_ORIGINAL ) ?0 :1 ; - ret[15 ].flags |=ULOGD_RETF_VALID ; -ulogd_propagate_results (upi);return 0 ; } -static int propagate_ct (struct ulogd_pluginstance *upi, -struct nfct_conntrack *ct, -unsigned int flags, -struct ct_timestamp *ctstamp) -{ -int rc; - - rc =propagate_ct_flow (upi, ct, flags,NFCT_DIR_ORIGINAL , ctstamp); -if (rc <0 ) -return rc; - -return propagate_ct_flow (upi, ct, flags,NFCT_DIR_REPLY , ctstamp); -} -static int event_handler (void *arg,unsigned int flags,int type,void *data) {
Signature · 1 line
1.5.2.5
PATCH 31/34 Switch from INNER JOIN to LEFT JOIN in ulog view.
In reply to Eric LeblondFrom: Eric leblond <[email protected]>
This patch fixes an error in MySQL schema which causes MySQL view to be always empty.
---
:100644 100644 6dc865a... eec6e69... M doc/mysql-ulogd2.sql
doc/mysql-ulogd2.sql | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-)
doc/mysql-ulogd2.sql +4 -4
@@ -157,8 +157,8 @@ CREATE SQL SECURITY INVOKER VIEW `ulog` AS oob_in, oob_out, oob_family, - ip_saddr as ip_saddr_bin, - ip_daddr as ip_daddr_bin, + ip_saddr AS ip_saddr_bin, + ip_daddr AS ip_daddr_bin, ip_protocol, ip_tos, ip_ttl,
@@ -188,8 +188,8 @@ CREATE SQL SECURITY INVOKER VIEW `ulog` AS icmp_echoseq, icmp_gateway, icmp_fragmtu - FROM ulog2 INNER JOIN tcp ON ulog2._id = tcp._tcp_id INNER JOIN udp ON ulog2._id = udp._udp_id - INNER JOIN icmp ON ulog2._id = icmp._icmp_id INNER JOIN mac ON ulog2._id = mac._mac_id; + FROM ulog2 LEFT JOIN tcp ON ulog2._id = tcp._tcp_id LEFT JOIN udp ON ulog2._id = udp._udp_id + LEFT JOIN icmp ON ulog2._id = icmp._icmp_id LEFT JOIN mac ON ulog2._id = mac._mac_id; -- shortcuts DROP VIEW IF EXISTS `view_tcp_quad`;
Signature · 1 line
1.5.2.5
PATCH 15/34 MySQL client library does not reconnect automatically since 5.0.
In reply to Eric LeblondFrom: Eric leblond <[email protected]>
This patch restores the reconnection functionnality for the mysql output plugin.
---
:100644 100644 1826c03... 800d79d... M output/mysql/ulogd_output_MYSQL.c
output/mysql/ulogd_output_MYSQL.c | 14 ++++++++++++++ 1 files changed, 14 insertions(+), 0 deletions(-)
output/mysql/ulogd_output_MYSQL.c +14 -0
@@ -180,6 +180,9 @@ static int open_db_mysql(struct ulogd_pluginstance *upi)char *user =user_ce (upi->config_kset).u.string;char *pass =pass_ce (upi->config_kset).u.string;char *db =db_ce (upi->config_kset).u.string; +#ifdef MYSQL_OPT_RECONNECT +my_bool trueval =1 ; +#endif mi->dbh =mysql_init (NULL );if (!mi->dbh) {
@@ -190,6 +193,11 @@ static int open_db_mysql(struct ulogd_pluginstance *upi)if (connect_timeout)mysql_options (mi->dbh,MYSQL_OPT_CONNECT_TIMEOUT , (const char *) &connect_timeout); +#ifdef MYSQL_OPT_RECONNECT +# if defined(MYSQL_VERSION_ID ) && (MYSQL_VERSION_ID >=50019 ) +mysql_options (mi->dbh,MYSQL_OPT_RECONNECT , &trueval); +# endif +#endif if (!mysql_real_connect (mi->dbh, server, user, pass, db, port,NULL ,0 )) { ulogd_log(ULOGD_ERROR, "can't connect to db: %s\n",
@@ -197,6 +205,12 @@ static int open_db_mysql(struct ulogd_pluginstance *upi)return -1 ; } +#ifdef MYSQL_OPT_RECONNECT +# if defined(MYSQL_VERSION_ID ) && (MYSQL_VERSION_ID <50019 ) +mysql_options (mi->dbh,MYSQL_OPT_RECONNECT , &trueval); +# endif +#endif +return 0 ; }
Signature · 1 line
1.5.2.5
PATCH 17/34 Suppress key relative to IPv6 address.
In reply to Eric LeblondFrom: Eric leblond <[email protected]>
This patch suppress key relative to IPv6 address because IPv4 and IPv6 can be stored in the same key.
---
:100644 100644 4420507... e837c62... M filter/raw2packet/ulogd_raw2packet_BASE.c
:100644 100644 3db2862... ae56946... M include/ulogd/printpkt.h
:100644 100644 d43f1fd... 465b224... M ulogd.conf.in
:100644 100644 7719cae... 6606824... M util/printpkt.c
filter/raw2packet/ulogd_raw2packet_BASE.c | 28 ++++------------------------ include/ulogd/printpkt.h | 2 -- ulogd.conf.in | 10 +++++++++- util/printpkt.c | 26 ++++++++------------------ 4 files changed, 21 insertions(+), 45 deletions(-)
filter/raw2packet/ulogd_raw2packet_BASE.c +4 -24
@@ -54,8 +54,6 @@ enum output_keys {KEY_IP_CSUM ,KEY_IP_ID ,KEY_IP_FRAGOFF , -KEY_IP6_SADDR , -KEY_IP6_DADDR ,KEY_IP6_PAYLOAD_LEN ,KEY_IP6_PRIORITY , KEY_IP6_FLOWLABEL,
@@ -186,24 +184,6 @@ static struct ulogd_key iphdr_rets[] = { .field_id = IPFIX_fragmentOffsetIPv4, }, }, - [KEY_IP6_SADDR ] = { - .type =ULOGD_RET_RAW , - .flags =ULOGD_RETF_NONE , - .name ="ip6.saddr" , - .ipfix = { - .vendor =IPFIX_VENDOR_IETF , - .field_id = IPFIX_sourceIPv6Address, - }, - }, - [KEY_IP6_DADDR ] = { - .type =ULOGD_RET_RAW , - .flags =ULOGD_RETF_NONE , - .name ="ip6.daddr" , - .ipfix = { - .vendor =IPFIX_VENDOR_IETF , - .field_id = IPFIX_destinationIPv6Address, - }, - }, [KEY_IP6_PAYLOAD_LEN ] = { .type =ULOGD_RET_UINT16 , .flags =ULOGD_RETF_NONE ,
@@ -730,10 +710,10 @@ static int _interp_ipv6hdr(struct ulogd_pluginstance *pi, u_int32_t len)if (len <sizeof (struct ip6_hdr ))return 0 ; - ret[KEY_IP6_SADDR ].u.value.ptr = &ipv6h->ip6_src; - ret[KEY_IP6_SADDR ].flags |=ULOGD_RETF_VALID ; - ret[KEY_IP6_DADDR ].u.value.ptr = &ipv6h->ip6_dst; - ret[KEY_IP6_DADDR ].flags |=ULOGD_RETF_VALID ; + ret[KEY_IP_SADDR ].u.value.ptr = &ipv6h->ip6_src; + ret[KEY_IP_SADDR ].flags |=ULOGD_RETF_VALID ; + ret[KEY_IP_DADDR ].u.value.ptr = &ipv6h->ip6_dst; + ret[KEY_IP_DADDR ].flags |=ULOGD_RETF_VALID ; ret[KEY_IP6_PAYLOAD_LEN ].u.value.ui16 =ntohs (ipv6h->ip6_plen); ret[KEY_IP6_PAYLOAD_LEN ].flags |=ULOGD_RETF_VALID ; ret[KEY_IP6_PRIORITY ].u.value.ui8 =ntohl (ipv6h->ip6_flow &0x0ff00000 ) >>20 ;
include/ulogd/printpkt.h +0 -2
@@ -17,8 +17,6 @@ enum pkt_keys {KEY_IP_ID ,KEY_IP_FRAGOFF ,KEY_IP_PROTOCOL , -KEY_IP6_SADDR , -KEY_IP6_DADDR ,KEY_IP6_PAYLOAD_LEN ,KEY_IP6_PRIORITY , KEY_IP6_HOPLIMIT,
ulogd.conf.in +9 -1
@@ -35,6 +35,7 @@ bufsize=150000 plugin="@libdir@/ulogd/ulogd_inppkt_NFLOG.so" plugin="@libdir@/ulogd/ulogd_inpflow_NFCT.so" plugin="@libdir@/ulogd/ulogd_filter_IFINDEX.so" +plugin="@libdir@/ulogd/ulogd_filter_IP2STR.so" plugin="@libdir@/ulogd/ulogd_filter_PRINTPKT.so" plugin="@libdir@/ulogd/ulogd_filter_PRINTFLOW.so" plugin="@libdir@/ulogd/ulogd_output_LOGEMU.so"
@@ -42,7 +43,10 @@ plugin="@libdir@/ulogd/ulogd_output_OPRINT.so" plugin="@libdir@/ulogd/ulogd_raw2packet_BASE.so" # this is a stack for packet-based logging via LOGEMU -#stack=log1:NFLOG,base1:BASE,ifi1:IFINDEX,print1:PRINTPKT,emu1:LOGEMU +#stack=log1:NFLOG,base1:BASE,ifi1:IFINDEX,ip2str1:IP2STR,print1:PRINTPKT,emu1:LOGEMU + +# this is a stack for IPv6 packet-based logging via LOGEMU +#stack=log2:NFLOG,base1:BASE,ifi1:IFINDEX,ip2str1:IP2STR,print1:PRINTPKT,emu1:LOGEMU # this is a stack for ULOG packet-based logging via LOGEMU #stack=ulog1:ULOG,base1:BASE,print1:PRINTPKT,emu1:LOGEMU
@@ -59,6 +63,10 @@ plugin="@libdir@/ulogd/ulogd_raw2packet_BASE.so" # netlink multicast group (the same as the iptables --ulog-nlgroup param) group=0 +[log2] +group=1 +addressfamily=10 + [ulog1] nlgroup=1
util/printpkt.c +8 -18
@@ -45,16 +45,14 @@ struct ulogd_key printpkt_keys[] = { [KEY_OOB_OUT] = { .name = "oob.out", }, [KEY_RAW_MAC] = { .name = "raw.mac", }, [KEY_RAW_MACLEN] = { .name = "raw.mac_len", }, - [KEY_IP_SADDR] = { .name = "ip.saddr", }, - [KEY_IP_DADDR] = { .name = "ip.daddr", }, + [KEY_IP_SADDR] = { .name = "ip.saddr.str", }, + [KEY_IP_DADDR] = { .name = "ip.daddr.str", }, [KEY_IP_TOTLEN] = { .name = "ip.totlen", }, [KEY_IP_TOS] = { .name = "ip.tos", }, [KEY_IP_TTL] = { .name = "ip.ttl", }, [KEY_IP_ID] = { .name = "ip.id", }, [KEY_IP_FRAGOFF] = { .name = "ip.fragoff", }, [KEY_IP_PROTOCOL] = { .name = "ip.protocol", }, - [KEY_IP6_SADDR] = { .name = "ip6.saddr", }, - [KEY_IP6_DADDR] = { .name = "ip6.daddr", }, [KEY_IP6_PAYLOAD_LEN] = { .name = "ip6.payload_len" }, [KEY_IP6_PRIORITY] = { .name = "ip6.priority" }, [KEY_IP6_HOPLIMIT] = { .name = "ip6.hoplimit" },
@@ -182,15 +180,11 @@ static int printpkt_ipv4(struct ulogd_key *res, char *buf)if (pp_is_valid (res,KEY_IP_SADDR )) buf_cur +=sprintf (buf_cur,"SRC=%s " , -inet_ntop (AF_INET , - &GET_VALUE (res,KEY_IP_SADDR ).ui32, - tmp,sizeof (tmp))); +GET_VALUE (res,KEY_IP_SADDR ).ptr);if (pp_is_valid (res,KEY_IP_DADDR )) buf_cur +=sprintf (buf_cur,"DST=%s " , -inet_ntop (AF_INET , - &GET_VALUE (res,KEY_IP_DADDR ).ui32, - tmp,sizeof (tmp))); +GET_VALUE (res,KEY_IP_DADDR ).ptr);/* FIXME: add pp_is_valid calls to remainder of file */ buf_cur += sprintf(buf_cur,"LEN=%u TOS=%02X PREC=0x%02X TTL=%u ID=%u ",
@@ -271,17 +265,13 @@ static int printpkt_ipv6(struct ulogd_key *res, char *buf)char *buf_cur = buf;char tmp[INET6_ADDRSTRLEN ]; -if (pp_is_valid (res,KEY_IP6_SADDR )) +if (pp_is_valid (res,KEY_IP_SADDR )) buf_cur +=sprintf (buf_cur,"SRC=%s " , -inet_ntop (AF_INET6 , -GET_VALUE (res,KEY_IP6_SADDR ).ptr, - tmp,sizeof (tmp))); +GET_VALUE (res,KEY_IP_SADDR ).ptr); -if (pp_is_valid (res,KEY_IP6_DADDR )) +if (pp_is_valid (res,KEY_IP_DADDR )) buf_cur +=sprintf (buf_cur,"DST=%s " , -inet_ntop (AF_INET6 , -GET_VALUE (res,KEY_IP6_DADDR ).ptr, - tmp,sizeof (tmp))); +GET_VALUE (res,KEY_IP_DADDR ).ptr);if (pp_is_valid (res,KEY_IP6_PAYLOAD_LEN )) buf_cur += sprintf(buf_cur, "LEN=%Zu ",
Signature · 1 line
1.5.2.5
PATCH 19/34 Fix options for pgsql module
In reply to Eric LeblondFrom: Pierre Chifflier <[email protected]>
Options where wrongly set for PGsql module.
---
:100644 100644 4737d28... 6697b64... M output/pgsql/ulogd_output_PGSQL.c
output/pgsql/ulogd_output_PGSQL.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)
output/pgsql/ulogd_output_PGSQL.c +2 -2
@@ -78,8 +78,8 @@ static struct config_keyset pgsql_kset = {#define host_ce (x) (x->ces[DB_CE_NUM+1])#define user_ce (x) (x->ces[DB_CE_NUM+2])#define pass_ce (x) (x->ces[DB_CE_NUM+3]) -#define port_ce (x) (x->ces[DB_CE_NUM+5]) -#define schema_ce (x) (x->ces[DB_CE_NUM+6]) +#define port_ce (x) (x->ces[DB_CE_NUM+4]) +#define schema_ce (x) (x->ces[DB_CE_NUM+5])#define PGSQL_HAVE_NAMESPACE_TEMPLATE \ "SELECT nspname FROM pg_namespace n WHERE n.nspname='%s'"
Signature · 1 line
1.5.2.5
PATCH 3/34 Use an enum to clarify code.
In reply to Eric LeblondFrom: Eric leblond <[email protected]>
This patch clarifies code which will be modified in next patch.
---
:100644 100644 aa354d4... 1d0b9e9... M util/printflow.c
util/printflow.c | 43 ++++++++++++++++++++++++++++--------------- 1 files changed, 28 insertions(+), 15 deletions(-)
util/printflow.c +28 -15
@@ -28,6 +28,19 @@#include <ulogd/ulogd.h> #include <ulogd/printflow.h> +enum printflow_fields { +PRINTFLOW_IP_SADDR =0 , +PRINTFLOW_IP_DADDR , +PRINTFLOW_IP_PROTOCOL , +PRINTFLOW_L4_SPORT , +PRINTFLOW_L4_DPORT , +PRINTFLOW_RAW_PKTLEN , +PRINTFLOW_RAW_PKTCOUNT , +PRINTFLOW_ICMP_CODE , +PRINTFLOW_ICMP_TYPE , +PRINTFLOW_DIR , +}; +struct ulogd_key printflow_keys[] = { { .type =ULOGD_RET_IPADDR ,
@@ -94,38 +107,38 @@ int printflow_print(struct ulogd_key *res, char *buf) {char *buf_cur = buf; -if (pp_is_valid (res,9 )) +if (pp_is_valid (res,PRINTFLOW_DIR )) buf_cur +=sprintf (buf_cur,"DIR=%s " , -GET_VALUE (res,9 ).b ?"REPLY" :"ORIG " ); +GET_VALUE (res,PRINTFLOW_DIR ).b ?"REPLY" :"ORIG " ); -if (pp_is_valid (res,0 )) +if (pp_is_valid (res,PRINTFLOW_IP_SADDR )) buf_cur +=sprintf (buf_cur,"SRC=%s " ,inet_ntoa ( (struct in_addr ) {htonl (GET_VALUE (res,0 ).ui32)})); -if (pp_is_valid (res,1 )) +if (pp_is_valid (res,PRINTFLOW_IP_DADDR )) buf_cur +=sprintf (buf_cur,"DST=%s " ,inet_ntoa ( (struct in_addr ) {htonl (GET_VALUE (res,1 ).ui32)})); -if (!pp_is_valid (res,2 )) +if (!pp_is_valid (res,PRINTFLOW_IP_PROTOCOL )) goto out; -switch (GET_VALUE (res,2 ).ui8) { +switch (GET_VALUE (res,PRINTFLOW_IP_PROTOCOL ).ui8) {case IPPROTO_TCP : buf_cur +=sprintf (buf_cur,"PROTO=TCP " ); -pp_print (buf_cur,"SPT" , res,3 , ui16); -pp_print (buf_cur,"DPT" , res,4 , ui16); +pp_print (buf_cur,"SPT" , res,PRINTFLOW_L4_SPORT , ui16); +pp_print (buf_cur,"DPT" , res,PRINTFLOW_L4_DPORT , ui16);break ;case IPPROTO_UDP : buf_cur +=sprintf (buf_cur,"PROTO=UDP " ); -pp_print (buf_cur,"SPT" , res,3 , ui16); -pp_print (buf_cur,"DPT" , res,4 , ui16); +pp_print (buf_cur,"SPT" , res,PRINTFLOW_L4_SPORT , ui16); +pp_print (buf_cur,"DPT" , res,PRINTFLOW_L4_DPORT , ui16);break ;case IPPROTO_ICMP : buf_cur +=sprintf (buf_cur,"PROTO=ICMP " ); -pp_print (buf_cur,"TYPE" , res,7 , ui8); -pp_print (buf_cur,"CODE" , res,8 , ui8); +pp_print (buf_cur,"TYPE" , res,PRINTFLOW_ICMP_CODE , ui8); +pp_print (buf_cur,"CODE" , res,PRINTFLOW_ICMP_TYPE , ui8);break ;case IPPROTO_ESP :
@@ -137,13 +150,13 @@ int printflow_print(struct ulogd_key *res, char *buf)break ;default : -pp_print (buf_cur,"PROTO" , res,2 , ui8); +pp_print (buf_cur,"PROTO" , res,PRINTFLOW_IP_PROTOCOL , ui8);break ; } out: -pp_print (buf_cur,"PKTS" , res,6 , ui32); -pp_print (buf_cur,"BYTES" , res,5 , ui32); +pp_print (buf_cur,"PKTS" , res,PRINTFLOW_RAW_PKTCOUNT , ui32); +pp_print (buf_cur,"BYTES" , res,PRINTFLOW_RAW_PKTLEN , ui32);strcat (buf_cur,"\n" );return 0 ;
Signature · 1 line
1.5.2.5
PATCH 4/34 Adapt printflow for one conntrack entry per line format.
From: Eric leblond <[email protected]>
This patch update the printflow output module to be able to print a whole conntrack entry on a single line.
---
:100644 100644 7343a23... 979f673... M include/ulogd/printflow.h
:100644 100644 1d0b9e9... d803633... M util/printflow.c
include/ulogd/printflow.h | 2 +- util/printflow.c | 161 ++++++++++++++++++++++++++++++++++----------- 2 files changed, 123 insertions(+), 40 deletions(-)
include/ulogd/printflow.h +1 -1
@@ -1,7 +1,7 @@#ifndef _PRINTFLOW_H#define _PRINTFLOW_H -#define FLOW_IDS 10 +#define FLOW_IDS 16extern struct ulogd_key printflow_keys[FLOW_IDS ];int printflow_print (struct ulogd_key *res,char *buf);
util/printflow.c +122 -39
236 lines not on this page · raw message
Signature · 1 line
1.5.2.5
PATCH 5/34 Add --info option which displays information about plugin.
It is difficult to find how to configure a plugin. This patch adds an info option which can be used to display:
* Name
* Configuration variables
* Input keys
* Output keys
Output exemple:/opt/ulogd2/sbin/ulogd --info /opt/ulogd2/lib/ulogd/ulogd_filter_IFINDEX.so
Name: IFINDEX
Input keys:
Key: oob.ifindex_in (unsigned int 32)
Key: oob.ifindex_out (unsigned int 32)
Output keys:
Key: oob.in (string)
Key: oob.out (string)
---
:100644 100644 ca5fb06... e1eb951... M src/ulogd.c
src/ulogd.c | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 files changed, 133 insertions(+), 8 deletions(-)
src/ulogd.c +133 -8
@@ -78,6 +78,8 @@ static char *ulogd_configfile = ULOGD_CONFIGFILE;static char *ulogd_logfile =ULOGD_LOGFILE_DEFAULT ;static FILE syslog_dummy; +static int info_mode =0 ; +/* linked list for all registered plugins */ static LLIST_HEAD (ulogd_plugins);static LLIST_HEAD (ulogd_pi_stacks);
@@ -231,21 +233,137 @@ static struct ulogd_plugin *find_plugin(const char *name)return NULL ; } +char *type_to_string (int type) +{ +switch (type) { +case ULOGD_RET_INT8 : +return strdup ("int 8" ); +break ; +case ULOGD_RET_INT16 : +return strdup ("int 16" ); +break ; +case ULOGD_RET_INT32 : +return strdup ("int 32" ); +break ; +case ULOGD_RET_INT64 : +return strdup ("int 64" ); +break ; +case ULOGD_RET_UINT8 : +return strdup ("unsigned int 8" ); +break ; +case ULOGD_RET_UINT16 : +return strdup ("unsigned int 16" ); +break ; +case ULOGD_RET_UINT32 : +return strdup ("unsigned int 32" ); +break ; +case ULOGD_RET_UINT64 : +return strdup ("unsigned int 64" ); +break ; +case ULOGD_RET_BOOL : +return strdup ("boolean" ); +break ; +case ULOGD_RET_IPADDR : +return strdup ("IPv4 addr" ); +break ; +case ULOGD_RET_IP6ADDR : +return strdup ("IPv6 addr" ); +break ; +case ULOGD_RET_STRING : +return strdup ("string" ); +break ; +case ULOGD_RET_RAW : +return strdup ("raw data" ); +break ; +default : +return strdup ("Unknown type" ); + } +} + + +void get_plugin_infos (struct ulogd_plugin *me) +{ +int i; +printf ("Name: %s\n" , me->name); +if (me->config_kset) { +printf ("Config options:\n" ); +for (i =0 ; i < me->config_kset->num_ces; i++) { +printf ("\tVar: %s (" , me->config_kset->ces[i].key); +switch (me->config_kset->ces[i].type) { +case CONFIG_TYPE_STRING : +printf ("String" ); +printf (", Default: %s" , me->config_kset->ces[i].u.value); +break ; +case CONFIG_TYPE_INT : +printf ("Integer" ); +printf (", Default: %d" , me->config_kset->ces[i].u.value); +break ; +case CONFIG_TYPE_CALLBACK : +printf ("Callback" ); +break ; +default : +printf ("Unknown" ); +break ; + } +if (me->config_kset->ces[i].options ==CONFIG_OPT_MANDATORY ) { +printf (", Mandatory" ); + } +printf (")\n" ); + } + } +printf ("Input keys:\n" ); +if (me->input.type !=ULOGD_DTYPE_SOURCE ) { +if (me->input.num_keys ==0 ) { +printf ("\tNo statically defined keys\n" ); + }else { +for (i =0 ; i < me->input.num_keys; i++) { +char *tstring =type_to_string (me->input.keys[i].type); +printf ("\tKey: %s (%s)\n" , + me->input.keys[i].name, + tstring); +free (tstring); + } + } + }else { +printf ("\tInput plugin, No keys\n" ); + } +printf ("Output keys:\n" ); +if (me->output.type !=ULOGD_DTYPE_SINK ) { +if (me->output.num_keys ==0 ) { +printf ("\tNo statically defined keys\n" ); + }else { +for (i =0 ; i < me->output.num_keys; i++) { +char *tstring =type_to_string (me->output.keys[i].type); +printf ("\tKey: %s (%s)\n" , + me->output.keys[i].name, + tstring); +free (tstring); + } + } + }else { +printf ("\tOutput plugin, No keys\n" ); + } +} +/* the function called by all plugins for registering themselves */ void ulogd_register_plugin (struct ulogd_plugin *me) {if (strcmp (me->version,ULOGD_VERSION )) {ulogd_log (ULOGD_NOTICE ,"plugin `%s' has incompatible version %s\n" , - me->version); + me->version);return ; } -if (find_plugin (me->name)) { -ulogd_log (ULOGD_NOTICE ,"plugin `%s' already registered\n" , - me->name); -exit (EXIT_FAILURE ); +if (info_mode ==0 ) { +if (find_plugin (me->name)) { +ulogd_log (ULOGD_NOTICE ,"plugin `%s' already registered\n" , + me->name); +exit (EXIT_FAILURE ); + } +ulogd_log (ULOGD_NOTICE ,"registering plugin `%s'\n" , me->name); +llist_add (&me->list, &ulogd_plugins); + }else { +get_plugin_infos (me); } -ulogd_log (ULOGD_NOTICE ,"registering plugin `%s'\n" , me->name); -llist_add (&me->list, &ulogd_plugins); }/***********************************************************************
@@ -853,6 +971,7 @@ static void print_usage(void)printf ("\t-d --daemon\tDaemonize (fork into background)\n" );printf ("\t-c --configfile\tUse alternative Configfile\n" );printf ("\t-u --uid\tChange UID/GID\n" ); +printf ("\t-i --info\tDisplay infos about plugin\n" ); }static struct option opts[] = {
@@ -861,6 +980,7 @@ static struct option opts[] = { { "help", 0, NULL, 'h' }, { "configfile", 1, NULL, 'c'}, { "uid", 1, NULL, 'u' }, + { "info", 1, NULL, 'i'}, { 0 } };
@@ -875,7 +995,7 @@ int main(int argc, char* argv[])gid_t gid =0 ; -while ((argch =getopt_long (argc, argv,"c:dh::Vu:" , opts,NULL )) !=-1 ) { +while ((argch =getopt_long (argc, argv,"c:dh::Vu:i:" , opts,NULL )) !=-1 ) {switch (argch) {default :case '?' :
@@ -916,6 +1036,11 @@ int main(int argc, char* argv[]) uid = pw->pw_uid; gid = pw->pw_gid;break ; +case 'i' : + info_mode =1 ; +load_plugin (optarg); +exit (0 ); +break ; } }
Signature · 1 line
1.5.2.5
PATCH 7/34 Changed sighup_handler_print to fallback to continue using old descriptor on new file opening failure.
In reply to Eric LeblondThis patch is a backport of Marius Tomaschewski <[email protected]> work on ulogd.
---
:100644 100644 c18aad7... b8d3903... M output/ulogd_output_LOGEMU.c
output/ulogd_output_LOGEMU.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-)
output/ulogd_output_LOGEMU.c +4 -1
@@ -114,15 +114,18 @@ static int _output_logemu(struct ulogd_pluginstance *upi)static void signal_handler_logemu (struct ulogd_pluginstance *pi,int signal) {struct logemu_instance *li = (struct logemu_instance *) &pi->private; +FILE *old = li->of;switch (signal) {case SIGHUP :ulogd_log (ULOGD_NOTICE ,"syslogemu: reopening logfile\n" ); -fclose (li->of); li->of =fopen (pi->config_kset->ces[0 ].u.string,"a" );if (!li->of) {ulogd_log (ULOGD_ERROR ,"can't reopen syslogemu: %s\n" ,strerror (errno)); + li->of = old; + }else { +fclose (old); }break ;default :
Signature · 1 line
1.5.2.5
PATCH 33/34 ULOGD_RET_IPADDR is for IPv4 or IPv6 address: fix description.
In reply to Eric LeblondFrom: Eric leblond <[email protected]>
Description of ULOGD_RET_IPADDR was incorrect in information display mode.
---
:100644 100644 e1eb951... 48529ca... M src/ulogd.c
src/ulogd.c | 5 +---- 1 files changed, 1 insertions(+), 4 deletions(-)
src/ulogd.c +1 -4
@@ -264,10 +264,7 @@ char *type_to_string(int type)return strdup ("boolean" );break ;case ULOGD_RET_IPADDR : -return strdup ("IPv4 addr" ); -break ; -case ULOGD_RET_IP6ADDR : -return strdup ("IPv6 addr" ); +return strdup ("IP addr" );break ;case ULOGD_RET_STRING :return strdup ("string" );
Signature · 1 line
1.5.2.5
PATCH 23/34 Print RAW as raw string.
In reply to Eric LeblondFrom: Eric leblond <[email protected]>
MySQL need no to be able to print RAW datas to be able to display IP addresses.
---
:100644 100644 1702acc... 4c4298c... M util/db.c
util/db.c | 4 +--- 1 files changed, 1 insertions(+), 3 deletions(-)
util/db.c +1 -3
@@ -295,9 +295,7 @@ static int __interp_db(struct ulogd_pluginstance *upi)sprintf (di->stmt_ins,"'," );break ;case ULOGD_RET_RAW : -ulogd_log (ULOGD_NOTICE , -"%s: type RAW not supported by MySQL\n" , - upi->input.keys[i].name); +sprintf (di->stmt_ins,"%s," , res->u.value.ptr);break ;default : ulogd_log(ULOGD_NOTICE,
Signature · 1 line
1.5.2.5
PATCH 34/34 Modify insert functions to accept standard integers to avoid casts.
In reply to Eric LeblondFrom: Pierre Chifflier <[email protected]>
This patch fixes the type of some fields in the SQL schema to sync with datatype of the corresponding ulogd2 keys.
---
:100644 100644 a488c11... 740a7ef... M doc/pgsql-ulogd2.sql
:100644 100644 0882357... 7ece626... M output/pgsql/ulogd_output_PGSQL.c
doc/pgsql-ulogd2.sql | 150 +++++++++++++++++-------------------- output/pgsql/ulogd_output_PGSQL.c | 6 +- 2 files changed, 72 insertions(+), 84 deletions(-)
doc/pgsql-ulogd2.sql +69 -81
@@ -52,8 +52,8 @@ CREATE TABLE ulog2 ( ip_ttl smallint default NULL, ip_totlen smallint default NULL, ip_ihl smallint default NULL, - ip_csum smallint default NULL, - ip_id smallint default NULL, + ip_csum integer default NULL, + ip_id integer default NULL, ip_fragoff smallint default NULL, timestamp timestamp NOT NULL default 'now' ) WITH (OIDS=FALSE);
@@ -77,16 +77,16 @@ CREATE TABLE tcp ( _tcp_id bigint PRIMARY KEY UNIQUE NOT NULL, tcp_sport integer default NULL, tcp_dport integer default NULL, - tcp_seq integer default NULL, + tcp_seq bigint default NULL, tcp_ackseq integer default NULL, - tcp_window smallint default NULL, - tcp_urg smallint default NULL, - tcp_urgp smallint default NULL, - tcp_ack smallint default NULL, - tcp_psh smallint default NULL, - tcp_rst smallint default NULL, - tcp_syn smallint default NULL, - tcp_fin smallint default NULL + tcp_window integer default NULL, + tcp_urg boolean default NULL, + tcp_urgp integer default NULL, + tcp_ack boolean default NULL, + tcp_psh boolean default NULL, + tcp_rst boolean default NULL, + tcp_syn boolean default NULL, + tcp_fin boolean default NULL ) WITH (OIDS=FALSE); CREATE INDEX tcp_sport ON tcp(tcp_sport);
@@ -172,8 +172,8 @@ CREATE OR REPLACE VIEW ulog AS icmp_echoseq, icmp_gateway, icmp_fragmtu - FROM ulog2 INNER JOIN tcp ON ulog2._id = tcp._tcp_id INNER JOIN udp ON ulog2._id = udp._udp_id - INNER JOIN icmp ON ulog2._id = icmp._icmp_id INNER JOIN mac ON ulog2._id = mac._mac_id; + FROM ulog2 LEFT JOIN tcp ON ulog2._id = tcp._tcp_id LEFT JOIN udp ON ulog2._id = udp._udp_id + LEFT JOIN icmp ON ulog2._id = icmp._icmp_id LEFT JOIN mac ON ulog2._id = mac._mac_id; -- shortcuts CREATE OR REPLACE VIEW view_tcp_quad AS
@@ -306,10 +306,10 @@ CREATE OR REPLACE FUNCTION INSERT_IP_PACKET( IN oob_mark integer, IN oob_in varchar(32), IN oob_out varchar(32), - IN oob_family smallint, + IN oob_family integer, IN ip_saddr_str inet, IN ip_daddr_str inet, - IN ip_protocol smallint + IN ip_protocol integer ) RETURNS bigint AS $$ INSERT INTO ulog2 (oob_time_sec,oob_time_usec,oob_prefix,oob_mark,
@@ -326,17 +326,17 @@ CREATE OR REPLACE FUNCTION INSERT_IP_PACKET_FULL( IN oob_mark integer, IN oob_in varchar(32), IN oob_out varchar(32), - IN oob_family smallint, + IN oob_family integer, IN ip_saddr_str inet, IN ip_daddr_str inet, - IN ip_protocol smallint, - IN ip_tos smallint, - IN ip_ttl smallint, - IN ip_totlen smallint, - IN ip_ihl smallint, - IN ip_csum smallint, - IN ip_id smallint, - IN ip_fragoff smallint + IN ip_protocol integer, + IN ip_tos integer, + IN ip_ttl integer, + IN ip_totlen integer, + IN ip_ihl integer, + IN ip_csum integer, + IN ip_id integer, + IN ip_fragoff integer ) RETURNS bigint AS $$ INSERT INTO ulog2 (oob_time_sec,oob_time_usec,oob_prefix,oob_mark,
@@ -350,16 +350,16 @@ CREATE OR REPLACE FUNCTION INSERT_TCP_FULL( IN tcp_id bigint, IN tcp_sport integer, IN tcp_dport integer, - IN tcp_seq integer, + IN tcp_seq bigint, IN tcp_ackseq integer, - IN tcp_window smallint, - IN tcp_urg smallint, - IN tcp_urgp smallint , - IN tcp_ack smallint, - IN tcp_psh smallint, - IN tcp_rst smallint, - IN tcp_syn smallint, - IN tcp_fin smallint + IN tcp_window integer, + IN tcp_urg boolean, + IN tcp_urgp integer , + IN tcp_ack boolean, + IN tcp_psh boolean, + IN tcp_rst boolean, + IN tcp_syn boolean, + IN tcp_fin boolean ) RETURNS bigint AS $$ INSERT INTO tcp (_tcp_id,tcp_sport,tcp_dport,tcp_seq,tcp_ackseq,tcp_window,tcp_urg,
@@ -369,10 +369,10 @@ RETURNS bigint AS $$ $$ LANGUAGE SQL SECURITY INVOKER; CREATE OR REPLACE FUNCTION INSERT_UDP( - IN tcp_id bigint, - IN tcp_sport integer, - IN tcp_dport integer, - IN tcp_len smallint + IN udp_id bigint, + IN udp_sport integer, + IN udp_dport integer, + IN udp_len integer ) RETURNS bigint AS $$ INSERT INTO udp (_udp_id,udp_sport,udp_dport,udp_len)
@@ -382,12 +382,12 @@ $$ LANGUAGE SQL SECURITY INVOKER; CREATE OR REPLACE FUNCTION INSERT_ICMP( IN icmp_id bigint, - IN icmp_type smallint, - IN icmp_code smallint, - IN icmp_echoid smallint, - IN icmp_echoseq smallint, + IN icmp_type integer, + IN icmp_code integer, + IN icmp_echoid integer, + IN icmp_echoseq integer, IN icmp_gateway integer, - IN icmp_fragmtu smallint + IN icmp_fragmtu integer ) RETURNS bigint AS $$ INSERT INTO icmp (_icmp_id,icmp_type,icmp_code,icmp_echoid,icmp_echoseq,icmp_gateway,icmp_fragmtu)
@@ -395,18 +395,6 @@ RETURNS bigint AS $$ SELECT currval('ulog2__id_seq'); $$ LANGUAGE SQL SECURITY INVOKER; -CREATE OR REPLACE FUNCTION INSERT_MAC( - IN tcp_id bigint, - IN udp_sport integer, - IN udp_dport integer, - IN udp_len smallint - ) -RETURNS bigint AS $$ - INSERT INTO udp (_udp_id,udp_sport,udp_dport,udp_len) - VALUES ($1,$2,$3,$4); - SELECT currval('ulog2__id_seq'); -$$ LANGUAGE SQL SECURITY INVOKER; - -- this function requires plpgsql -- su -c "createlang plpgsql ulog2" postgres CREATE OR REPLACE FUNCTION INSERT_PACKET_FULL(
@@ -416,38 +404,38 @@ CREATE OR REPLACE FUNCTION INSERT_PACKET_FULL( IN oob_mark integer, IN oob_in varchar(32), IN oob_out varchar(32), - IN oob_family smallint, + IN oob_family integer, IN ip_saddr_str inet, IN ip_daddr_str inet, - IN ip_protocol smallint, - IN ip_tos smallint, - IN ip_ttl smallint, - IN ip_totlen smallint, - IN ip_ihl smallint, - IN ip_csum smallint, - IN ip_id smallint, - IN ip_fragoff smallint, + IN ip_protocol integer, + IN ip_tos integer, + IN ip_ttl integer, + IN ip_totlen integer, + IN ip_ihl integer, + IN ip_csum integer, + IN ip_id integer, + IN ip_fragoff integer, IN tcp_sport integer, IN tcp_dport integer, - IN tcp_seq integer, + IN tcp_seq bigint, IN tcp_ackseq integer, - IN tcp_window smallint, - IN tcp_urg smallint, - IN tcp_urgp smallint , - IN tcp_ack smallint, - IN tcp_psh smallint, - IN tcp_rst smallint, - IN tcp_syn smallint, - IN tcp_fin smallint, + IN tcp_window integer, + IN tcp_urg boolean, + IN tcp_urgp integer , + IN tcp_ack boolean, + IN tcp_psh boolean, + IN tcp_rst boolean, + IN tcp_syn boolean, + IN tcp_fin boolean, IN udp_sport integer, IN udp_dport integer, - IN udp_len smallint, - IN icmp_type smallint, - IN icmp_code smallint, - IN icmp_echoid smallint, - IN icmp_echoseq smallint, + IN udp_len integer, + IN icmp_type integer, + IN icmp_code integer, + IN icmp_echoid integer, + IN icmp_echoseq integer, IN icmp_gateway integer, - IN icmp_fragmtu smallint + IN icmp_fragmtu integer ) RETURNS bigint AS $$ DECLARE
@@ -455,11 +443,11 @@ DECLARE BEGIN _id := INSERT_IP_PACKET_FULL($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17) ; IF (ip_protocol = 6) THEN - SELECT INSERT_TCP_FULL(_id,$18,$19,$20,$21,$22,$23,$24,$25,$26,$27,$28,$29); + PERFORM INSERT_TCP_FULL(_id,$18,$19,$20,$21,$22,$23,$24,$25,$26,$27,$28,$29); ELSIF (ip_protocol = 17) THEN - SELECT INSERT_UDP(_id,$30,$31,$32,$33); + PERFORM INSERT_UDP(_id,$30,$31,$32,$33); ELSIF (ip_protocol = 1) THEN - SELECT INSERT_ICMP(_id,$34,$35,$36,$37,$38,$39); + PERFORM INSERT_ICMP(_id,$34,$35,$36,$37,$38,$39); END IF; RETURN _id; END
output/pgsql/ulogd_output_PGSQL.c +3 -3
@@ -280,8 +280,7 @@ static int open_db_pgsql(struct ulogd_pluginstance *upi)static int escape_string_pgsql (struct ulogd_pluginstance *upi,char *dst,const char *src,unsigned int len) { -PQescapeString (dst, src,strlen (src)); -return 0 ; +return PQescapeString (dst, src,strlen (src)); } static int execute_pgsql(struct ulogd_pluginstance *upi,
@@ -290,7 +289,8 @@ static int execute_pgsql(struct ulogd_pluginstance *upi,struct pgsql_instance *pi = (struct pgsql_instance *) upi->private; pi->pgres =PQexec (pi->dbh, stmt); -if (!pi->pgres ||PQresultStatus (pi->pgres) !=PGRES_COMMAND_OK ) { +if (!(pi->pgres && (PQresultStatus (pi->pgres) ==PGRES_COMMAND_OK ) + || (PQresultStatus (pi->pgres) ==PGRES_TUPLES_OK ))) {ulogd_log (ULOGD_ERROR ,"execute failed (%s)\n" ,PQerrorMessage (pi->dbh));return -1 ;
Signature · 1 line
1.5.2.5
Pablo Neira Ayuso
maintainerQuoted 26 lines · Eric Leblond
This patchset contains patches for ulogd2 from Pierre Chifflier and I.
As discussed during Netfilter workshop, the goal of this patchset is to provide a new and modern SQL logging schema. Some colateral patchs are present in the patchset due to the state of Ulogd2. As stated by Holger, people using ulogd2 now are early adopters and we tried to improve usability of ulogd2. For example, we've added a --info switch to ulogd2 to be able to display option of a plugin.
But, the main work is on SQL logging. Ulogd 1.x schema was really bad. It lacks index and the way data are stored (one big line per entry full of NULL fields) is not efficient for databases.
Thus, we propose new schemas for MySQL and PGsql which use advanced database feature without complication on developper side. In fact, the SQL related C code did not change very much. The main change is the use of a call to a SQL function instead of using a SQL query. The advantage of doing this is to hide the complexity of the database to developpers and let people knowing databases work on their side without bothering us.
I will finished this mail by a description of the avantages of the new schema. It uses a set of small dedicated tables (a TCP tables for example). From an SQL point of view this is more efficient as we limit the number of NULL fields (storage of empty datas has a cost). The schema has some SQL views (virtual table) and some of them provides an near complete backward compatility with the existing one.
Quoted 5 lines
One other advantage of the new schema is that extension (like nufw one) can used without changing anything for non-aware system.
This patchset should not conflict with Holger patchset (if NFCT related work is omitted). I can do the merge work if some is needed, just let me know.
At first glance this looks really great. I think that this doesn't clash with Holger's efforts either. I'll apply these patches tomorrow. Thanks Eric.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 02/34] [Resend] Do not propagate one conntrack event via 2 messages
In reply to Eric LeblondQuoted 3 lines · Eric Leblond
Ulogd2 was propagating through a stack 2 message for one single conntrack event. This patch provides a fall back to on message per event. It also uses an enum to improve code readability instead of direct access to array via numerical index.
Hi Eric,
this one would clash with the changes I did. I'll check what you do did there. In the meantime can you please check
[ULOGD 15/15] NFCT: rework and let it scale
from my last patchset? I'll happily change that code to keep the output key compatibility.
Thanks.
/holger
Re: [PATCH 01/34] Introduce new SQL schema.
In reply to Eric LeblondApplied. Thanks. BTW, one question:
Eric Leblond wrote
Quoted 17 lines
util/db.c +0
-13@@ -88,25 +90,8 @@ static int sql_createstmt(struct ulogd_pluginstance *upi)return -ENOMEM ; } -if (mi->schema) -sprintf (mi->stmt,"insert into %s.%s (" , mi->schema, table); -else -sprintf (mi->stmt,"insert into %s (" , table); - mi->stmt_val = mi->stmt +strlen (mi->stmt); - -for (i =0 ; i < upi->input.num_keys; i++) { -if (upi->input.keys[i].flags &ULOGD_KEYF_INACTIVE ) -continue ; - -strncpy (buf, upi->input.keys[i].name,ULOGD_MAX_KEYLEN ); -while ((underscore =strchr (buf,'.' ))) - *underscore ='_' ;
util/db.c static int sql_createstmt(struct ulogd_pluginstance *upi) · line 97
Quoted 10 lines
util/db.c +1
-5-sprintf (mi->stmt_val,"%s," , buf); - mi->stmt_val = mi->stmt +strlen (mi->stmt); - } - *(mi->stmt_val -1 ) =')' ; +sprintf (mi->stmt,"CALL %s(" , procedure); -sprintf (mi->stmt_val," values (" ); mi->stmt_val = mi->stmt +strlen (mi->stmt);ulogd_log (ULOGD_DEBUG ,"stmt='%s'\n" , mi->stmt);
Since now we use user-defined procedures and call them, we have to remove the old mysql and pgsql definitions from doc/ as well which does not use it, and what about sqlite3?
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 03/34] Use an enum to clarify code.
In reply to Eric LeblondQuoted 3 lines · Eric Leblond
From: Eric leblond <[email protected]>
This patch clarifies code which will be modified in next patch.
Applied. Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 04/34] Adapt printflow for one conntrack entry per line format.
In reply to Eric LeblondQuoted 4 lines · Eric Leblond
From: Eric leblond <[email protected]>
This patch update the printflow output module to be able to print a whole conntrack entry on a single line.
Applied. Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 05/34] Add --info option which displays information about plugin.
In reply to Eric LeblondQuoted 7 lines · Eric Leblond
It is difficult to find how to configure a plugin. This patch adds an info option which can be used to display:
* Name
* Configuration variables
* Input keys
* Output keys
Quoted 9 lines
Output exemple:
/opt/ulogd2/sbin/ulogd--info/opt/ulogd2/lib/ulogd/ulogd_filter_IFINDEX.so
Name: IFINDEX
Input keys:
Key: oob.ifindex_in (unsigned int 32)
Key: oob.ifindex_out (unsigned int 32)
Output keys:
Key: oob.in (string)
Key: oob.out (string)
Applied. Thanks. BTW, please break lines at 80 chars. I have fix this myself.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 06/34] New version of SQL schema.
In reply to Eric LeblondQuoted 3 lines · Eric Leblond
From: Pierre Chifflier <[email protected]>
Add insert functions for the PostgreSQL version (read instructions).
Applied. Thanks. BTW, please, don't send patchesets which stack several local modifications, just send it in one. I have noticed that this also happens with other Pierre's patches.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Pablo Neira Ayuso
maintainerRe: [PATCH 07/34] Changed sighup_handler_print to fallback to continue using old descriptor on new file opening failure.
In reply to Eric LeblondQuoted 1 line · Eric Leblond
This patch is a backport of Marius Tomaschewski <[email protected]> work on ulogd.
I like these backports. Applied. Thanks Eric.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 08/34] Added explicit null termination of the hostname buffer
In reply to Eric LeblondQuoted 1 line · Eric Leblond
Based on From Marius Tomaschewski <[email protected]> work on ulogd.
Applied. Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 09/34] For OPRINT, changed sighup_handler_print to fallback to continue using old descriptor on new file opening failure.
In reply to Eric LeblondQuoted 1 line · Eric Leblond
Based on Marius Tomaschewski work.
Applied. Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Pablo Neira Ayuso
maintainerRe: [PATCH 10/34] Add some missing line break.
In reply to Eric LeblondQuoted 1 line · Eric Leblond
From Marius Tomaschewski <[email protected]>
Also applied. Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 11/34] Put O at the real end of the string.
In reply to Eric LeblondQuoted 1 line · Eric Leblond
From Marius Tomaschewski <[email protected]>
Applied. Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 12/34] Changed to show pcap file name when open failed.
In reply to Eric LeblondQuoted 1 line · Eric Leblond
Port of Marius Tomaschewski work on ulogd.
Applied. Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 13/34] Display filename in the other error case.
In reply to Eric LeblondQuoted 1 line · Eric Leblond
Port of Marius Tomaschewski work on ulogd.
Applied. Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 14/34] Request at least autoconf 2.50 (needed for large file support macro).
In reply to Eric LeblondQuoted 1 line · Eric Leblond
Signed-off-by Eric leblond <[email protected]>
Applied. Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 15/34] MySQL client library does not reconnect automatically since 5.0.
In reply to Eric LeblondQuoted 3 lines · Eric Leblond
From: Eric leblond <[email protected]>
This patch restores the reconnection functionnality for the mysql output plugin.
Applied. Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 01/34] Introduce new SQL schema.
In reply to Pablo Neira AyusoHi,
Quoted 9 lines · Pablo Neira Ayuso
Applied. Thanks. BTW, one question:
util/db.c static int sql_createstmt(struct ulogd_pluginstance *upi) · line 97
Quoted 3 lines
util/db.c +0
-0mi->stmt_val = mi->stmt +strlen (mi->stmt);ulogd_log (ULOGD_DEBUG ,"stmt='%s'\n" , mi->stmt);Since now we use user-defined procedures and call them, we have to remove the old mysql and pgsql definitions from doc/ as well which does not use it,
You're right?
Quoted 1 line
and what about sqlite3?
I don't think sqlite3 will be able to support SQL things used in this schema. Holger, do you confirm that ?
BR,
Signature · 2 lines
Eric Leblond INL: http://www.inl.fr/
Re: [PATCH 02/34] [Resend] Do not propagate one conntrack event via 2 messages
In reply to Holger EitzenbergerHello,
Quoted 9 lines · Holger Eitzenberger
Quoted 3 lines · Eric Leblond
Ulogd2 was propagating through a stack 2 message for one single conntrack event. This patch provides a fall back to on message per event. It also uses an enum to improve code readability instead of direct access to array via numerical index.
Hi Eric,
this one would clash with the changes I did.
Yes that's the only point where our respective work clash.
Quoted 7 lines
I'll check what you do
did there. In the meantime can you please check[ULOGD 15/15] NFCT: rework and let it scale
from my last patchset? I'll happily change that code to keep the output key compatibility.
As long as you maintain output key compatibility, I'm fine with it.
BR,
Signature · 2 lines
Eric Leblond INL: http://www.inl.fr/
Pablo Neira Ayuso
maintainerRe: [PATCH 16/34] Introduce IP2STR module which convert IP to string.
In reply to Eric LeblondQuoted 5 lines · Eric Leblond
From: Eric leblond <[email protected]>
This module is a generic module which is used to convert an IP from internal representation to string representation. This is a task needed by several modules like printpkt or pgsql. This module factorizes the code.
Applied with minor glitches. Thanks.
filter/ulogd_filter_IP2STR.c line 172
Quoted 9 lines
filter/ulogd_filter_IP2STR.c new file +9
-0+static int ip2str_start (struct ulogd_pluginstance *upi) +{ +return 0 ; +} + +static int ip2str_fini (struct ulogd_pluginstance *upi) +{ +return 0 ; +}
Removed this since they are not required.
Quoted 1 line
+static struct
ulogd_pluginifindex_plugin= {
^^^
Mind the copy and paste ;)
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 01/34] Introduce new SQL schema.
In reply to Eric LeblondHi,
Quoted 7 lines · Eric Leblond
Hi,
Quoted 3 lines
Since now we use user-defined procedures and call them, we have to remove the old mysql and pgsql definitions from doc/ as well which does not use it,
You're right?
Please forget about the ? this is a dot. Old schemas should be removed.
BR,
Signature · 2 lines
Eric Leblond INL: http://www.inl.fr/
Re: [PATCH 17/34] Suppress key relative to IPv6 address.
In reply to Eric LeblondQuoted 4 lines · Eric Leblond
From: Eric leblond <[email protected]>
This patch suppress key relative to IPv6 address because IPv4 and IPv6 can be stored in the same key.
Applied. Thanks. I have also added a minor comment to the changelog about the missing IP2STR from ulogd.conf.in line that it also include in this patch.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 18/34] Update schema for PostgreSQL.
In reply to Eric LeblondQuoted 3 lines · Eric Leblond
From: Pierre Chifflier <[email protected]>
This patch add
_strsuffix to inet types (needed after IP2STR introduction)
Applied. Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 19/34] Fix options for pgsql module
In reply to Eric LeblondQuoted 3 lines · Eric Leblond
From: Pierre Chifflier <[email protected]>
Options where wrongly set for PGsql module.
Applied. Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 20/34] Mark ID as inactive (sequence in pg schema)
In reply to Eric LeblondQuoted 1 line · Eric Leblond
From: Pierre Chifflier <[email protected]>
Applied. Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Pablo Neira Ayuso
maintainerRe: [PATCH 21/34] Add IP2BIN module: convert IP address to binary string.
In reply to Eric LeblondQuoted 4 lines · Eric Leblond
From: Eric leblond <[email protected]>
This module convert IP from internal notation to a string in binary notation which is used by the MySQL output plugin.
Applied with similar changes in IP2STR. Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Pablo Neira Ayuso
maintainerRe: [PATCH 22/34] Fix description and indenting.
In reply to Eric LeblondQuoted 1 line · Eric Leblond
From: Eric leblond <[email protected]>
Applied. Thanks. As said, please don't send stacked local changes. Send them in one patch, ie. in 16/32.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 23/34] Print RAW as raw string.
In reply to Eric LeblondQuoted 4 lines · Eric Leblond
From: Eric leblond <[email protected]>
MySQL need no to be able to print RAW datas to be able to display IP addresses.
Applied. Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Pablo Neira Ayuso
maintainerRe: [PATCH 24/34] Fix IPv4 output.
In reply to Eric LeblondQuoted 3 lines · Eric Leblond
From: Eric leblond <[email protected]>
Fix a bug in IPv4 output of IP2BIN module.
Fixing code previously applied is nasty. Applied.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Pablo Neira Ayuso
maintainerRe: [PATCH 01/34] Introduce new SQL schema.
In reply to Eric LeblondQuoted 7 lines · Eric Leblond
Quoted 4 lines · Eric Leblond
Quoted 3 lines
Since now we use user-defined procedures and call them, we have to remove the old mysql and pgsql definitions from doc/ as well which does not use it,
You're right?
Please forget about the ? this is a dot. Old schemas should be removed.
I'd appreciate a patch for that. BTW, I noticed that ulogd maintains an internal copy of iftable.c and rtnl.c which is used by IFINDEX, we should use libnfnetlink instead. Would you cook a patch for that? Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 25/34] Set oob.family as VALID key.
In reply to Eric LeblondQuoted 3 lines · Eric Leblond
From: Eric leblond <[email protected]>
OOB_FAMILYoutput was not set by NFLOG because the key was not set as valid.
Applied. Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Pablo Neira Ayuso
maintainerRe: [PATCH 01/34] Introduce new SQL schema.
In reply to Eric LeblondQuoted 10 lines · Eric Leblond
Quoted 3 lines
Since now we use user-defined procedures and call them, we have to remove the old mysql and pgsql definitions from doc/ as well which does not use it,
You're right?
Quoted 1 line
and what about sqlite3?
I don't think sqlite3 will be able to support SQL things used in this schema. Holger, do you confirm that ?
Then, my guess is that we have to introduce some kind of compatibility if we decide to support sqlite3. Holger?
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 26/34] Modify IPv6 parser to fill oob_family.
In reply to Eric LeblondQuoted 4 lines · Eric Leblond
From: Eric leblond <[email protected]>
With this patch, BASE filter module is able fill
oob_familywhen parsing IPv6 address.
Applied. Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 27/34] Free insertion function result (mysql)
In reply to Eric LeblondQuoted 4 lines · Eric Leblond
From: Pierre Chifflier <[email protected]>
Change from procedure to function in mysql schema adds the need to free MySQL result after request.
Applied. Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 28/34] Update SQL schema
In reply to Eric LeblondQuoted 4 lines · Eric Leblond
From: Pierre Chifflier <[email protected]>
This patch adds
oob_familyto the schema. Thus it is now possible to easily select IPv4 or IPv6 entries in the database. This patch also explicitly selects fields to create view.
Applied. Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 29/34] Fix some place were oob_family was used instead of _oob_family.
In reply to Eric LeblondQuoted 3 lines · Eric Leblond
From: Eric leblond <[email protected]>
This patch fixes some small typo in MySQL schema.
Applied. Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 30/34] Convert SQL procedure to function call.
In reply to Eric LeblondQuoted 4 lines · Eric Leblond
From: Pierre Chifflier <[email protected]>
SQL standard says a function has to be called with SELECT and not CALL. This patch modify code accordingly.
Applied. Thanks
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 31/34] Switch from INNER JOIN to LEFT JOIN in ulog view.
In reply to Eric LeblondQuoted 4 lines · Eric Leblond
From: Eric leblond <[email protected]>
This patch fixes an error in MySQL schema which causes MySQL view to be always empty.
Applied. Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 32/34] Add state extension.
In reply to Eric LeblondQuoted 4 lines · Eric Leblond
From: Eric leblond <[email protected]>
This patch adds an state extension to SQL schema. This can be used to store the information about the packet being dropped or accepted.
Applied. Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 34/34] Modify insert functions to accept standard integers to avoid casts.
In reply to Eric LeblondQuoted 4 lines · Eric Leblond
From: Pierre Chifflier <[email protected]>
This patch fixes the type of some fields in the SQL schema to sync with datatype of the corresponding ulogd2 keys.
Applied. Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 33/34] ULOGD_RET_IPADDR is for IPv4 or IPv6 address: fix description.
In reply to Eric LeblondQuoted 3 lines · Eric Leblond
From: Eric leblond <[email protected]>
Description of
ULOGD_RET_IPADDRwas incorrect in information display mode.
Applied. Thanks.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 01/34] Introduce new SQL schema.
In reply to Eric LeblondQuoted 4 lines · Eric Leblond
Quoted 1 line
and what about sqlite3?
I don't think sqlite3 will be able to support SQL things used in this schema. Holger, do you confirm that ?
AFAIK sqlite3 does not have stored procedures as other databases have, instead you can use triggers with somewhat limited functionality compared to stored procedures. Alternatively you can write extensions in plain C, but those may not be what you need.
Does your change optimize DB memory consumption or is it targeted at improving access times?
/holger
Re: [PATCH 01/34] Introduce new SQL schema.
Quoted 7 lines · Holger Eitzenberger
Quoted 4 lines · Eric Leblond
Quoted 1 line
and what about sqlite3?
I don't think sqlite3 will be able to support SQL things used in this schema. Holger, do you confirm that ?
Quoted 7 lines
AFAIK sqlite3 does not have stored procedures as other databases have, instead you can use triggers with somewhat limited functionality compared to stored procedures. Alternatively you can write extensions in plain C, but those may not be what you need.
Does your change optimize DB memory consumption or is it targeted at improving access times?
Hi,
The new SQL schema has the following advantages:
- storage: split packets into several tables (instead of one big table). This can help a lot on storage efficiency, depending on the DB
- indexes: many indexes were missing
- SQL schema independence: stored procedures and views are used to ensure the C code does not have to know how SQL data are stored. This abstraction layer allows to change the SQL layout without changing the C code
- extensibility: a unique ID is used as a relation between tables. This way, a third-party application can create tables to add information, without changing the default SQL schema.
Foreign keys are used to ensure consistency.
The main idea was to introduce this abstraction layer to "hide" insertions into multiple tables. Unfortunately, I'm not sure how to handle this with sqlite ..
Using stored procedures may improve access times by preparing the query once (sort of compilation) and executing it faster, though this is not the first objective.
Using several tables will mostly help for read access.
Regards,
Pierre
Re: [PATCH 24/34] Fix IPv4 output.
In reply to Pablo Neira AyusoHello,
Quoted 6 lines · Pablo Neira Ayuso
Quoted 3 lines · Eric Leblond
From: Eric leblond <[email protected]>
Fix a bug in IPv4 output of IP2BIN module.
Fixing code previously applied is nasty.
Sorry, I send a reworked version of our Ulogd2 git tree but I need to improve my git king-fu a little bit and I miss this bugfix.
Anyway, I will be more cautious next time.
BR,
Signature · 2 lines
Eric Leblond INL: http://www.inl.fr/
Pablo Neira Ayuso
maintainerRe: [PATCH 01/34] Introduce new SQL schema.
In reply to Pierre ChifflierQuoted 3 lines · Pierre Chifflier
The main idea was to introduce this abstraction layer to "hide" insertions into multiple tables. Unfortunately, I'm not sure how to handle this with sqlite ..
Since sqlite doesn't seem to have stored procedures, I think that we can cook a modified version of sql_createstmt() for sqlite that recovers the code that Eric's patch [1/34] has removed from util/db.c. Patches welcome.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
PATCH 1/1 Use index2name capabilities of libnfnetlink in IFINDEX filter.
In reply to Pablo Neira AyusoThis patch modify ulogd_filter_IFINDEX to use libnfnetlink for index to interface name mapping instead of using local version. This requires at least libnfnetlink 0.0.30. This dependancy is checked in configure (thanks to Sebastien Tricaud for his patch).
---
:100644 100644 086e4cb... a70848c... M configure.in
:100644 100644 51bcc3f... 52b639c... M filter/Makefile.am
:100644 000000 02a5f7e... 0000000... D filter/iftable.c
:100644 000000 6725d67... 0000000... D filter/iftable.h
:100644 000000 5365af6... 0000000... D filter/rtnl.c
:100644 000000 18f0963... 0000000... D filter/rtnl.h
:100644 100644 b386f4a... 468a4c4... M filter/ulogd_filter_IFINDEX.c
configure.in | 5 + filter/Makefile.am | 6 +- filter/iftable.c | 259 ----------------------------------------- filter/iftable.h | 11 -- filter/rtnl.c | 246 -------------------------------------- filter/rtnl.h | 29 ----- filter/ulogd_filter_IFINDEX.c | 84 +++++++------- 7 files changed, 51 insertions(+), 589 deletions(-)
configure.in +5 -0
@@ -30,6 +30,11 @@ dnl Checks for library functions. AC_FUNC_VPRINTF AC_CHECK_FUNCS(socket strerror) +dnl Check for the right nfnetlink version +LIBNFNETLINK_REQUIRED=0.0.30 +PKG_CHECK_MODULES(LIBNFNETLINK, libnfnetlink >= $LIBNFNETLINK_REQUIRED,, + AC_MSG_ERROR(Cannot find libnfnetlink >= $LIBNFNETLINK_REQUIRED)) + AC_CHECK_HEADER([libnetfilter_log/linux_nfnetlink_log.h], [AC_MSG_RESULT([found])], [AC_MSG_ERROR([libnetfilter_log Version 0.0.11 or later needed])])
filter/Makefile.am +2 -4
@@ -2,14 +2,12 @@ SUBDIRS = raw2packet packet2flow INCLUDES = $(all_includes) -I$(top_srcdir)/include -noinst_HEADERS = rtnl.h iftable.h - pkglib_LTLIBRARIES = ulogd_filter_IFINDEX.la ulogd_filter_PWSNIFF.la \ ulogd_filter_PRINTPKT.la ulogd_filter_PRINTFLOW.la \ ulogd_filter_IP2STR.la ulogd_filter_IP2BIN.la -ulogd_filter_IFINDEX_la_SOURCES = ulogd_filter_IFINDEX.c rtnl.c iftable.c -ulogd_filter_IFINDEX_la_LDFLAGS = -module +ulogd_filter_IFINDEX_la_SOURCES = ulogd_filter_IFINDEX.c +ulogd_filter_IFINDEX_la_LDFLAGS = -module -lnfnetlink ulogd_filter_PWSNIFF_la_SOURCES = ulogd_filter_PWSNIFF.c ulogd_filter_PWSNIFF_la_LDFLAGS = -module
filter/iftable.c deleted +0 -259
259 lines not on this page · raw message
filter/iftable.h deleted +0 -11
@@ -1,11 +0,0 @@ -#ifndef _IFTABLE_H -#define _IFTABLE_H - -extern char *ifindex_2name (unsigned int index); -extern int iftable_up (unsigned int index); - -extern int iftable_init (void ); -extern void iftable_fini (void ); - -extern int iftable_dump (FILE *outfd); -#endif
filter/rtnl.c deleted +0 -246
246 lines not on this page · raw message
filter/rtnl.h deleted +0 -29
@@ -1,29 +0,0 @@ -#ifndef _RTNL_H -#define _RTNL_H - -#include <sys/socket.h> -#include <linux/types.h> -#include <linux/rtnetlink.h> - -struct rtnl_handler { -struct rtnl_handler *next; - -u_int16_t nlmsg_type; -int (*handlefn)(struct nlmsghdr *h,void *arg); -void *arg; -}; - - -/* api for handler plugins */ -int rtnl_handler_register (struct rtnl_handler *hdlr); -int rtnl_handler_unregister (struct rtnl_handler *hdlr); -int rtnl_parse_rtattr (struct rtattr *tb[],int max,struct rtattr *rta,int len); -int rtnl_dump_type (unsigned int type); - -/* api for core program */ -int rtnl_init (void ); -void rtnl_fini (void ); -int rtnl_receive (); - - -#endif
filter/ulogd_filter_IFINDEX.c +44 -40
138 lines not on this page · raw message
Signature · 1 line
1.5.2.5
Pablo Neira Ayuso
maintainerRe: [PATCH] Use index2name capabilities of libnfnetlink in IFINDEX filter.
Quoted 4 lines · Eric Leblond
This patch modify
ulogd_filter_IFINDEXto use libnfnetlink for index to interface name mapping instead of using local version. This requires at least libnfnetlink 0.0.30. This dependancy is checked in configure (thanks to Sebastien Tricaud for his patch).
Great. Applied. Thanks Eric.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
Re: [PATCH 02/34] [Resend] Do not propagate one conntrack event via 2 messages
In reply to Eric LeblondHello,
Quoted 9 lines · Eric Leblond
Hello,
Quoted 6 lines · Holger Eitzenberger
Quoted 3 lines · Eric Leblond
Ulogd2 was propagating through a stack 2 message for one single conntrack event. This patch provides a fall back to on message per event. It also uses an enum to improve code readability instead of direct access to array via numerical index.
Quoted 6 lines
Quoted 4 lines
[ULOGD 15/15] NFCT: rework and let it scale
from my last patchset? I'll happily change that code to keep the output key compatibility.
As long as you maintain output key compatibility, I'm fine with it.
We're still without decision on this problem and we thus don't have a working ulogd2 NFCT input plugin.
On one side we've got Holger work which is quiet important but adds some kernel dependancies and in the other side my small work which will be at term compatible
with Holger's work. Maybe someone form the team could apply the patch and
let code be updated when Holger's kernel side code reach mainstream ?
By the way, it could be interesting to plan an ulogd2 beta release in a not too far future.
BR,
Signature · 3 lines
Eric Leblond INL: http://www.inl.fr/ NuFW: http://www.nufw.org/
Pablo Neira Ayuso
maintainerRe: [PATCH 02/34] [Resend] Do not propagate one conntrack event via 2 messages
Quoted 4 lines · Eric Leblond
On one side we've got Holger work which is quiet important but adds some kernel dependancies and in the other side my small work which will be at term compatible
with Holger's work. Maybe someone form the team could apply the patch and
let code be updated when Holger's kernel side code reach mainstream ?
Please, send me a patch with your fix against current SVN snapshot.
Quoted 2 lines
By the way, it could be interesting to plan an ulogd2 beta release in a not too far future.
Sure, but let me have some time to give some final spins to ulogd2 before.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers
PATCH 1/1 Sends one message for each connection event instead of two.
Hi,
This is the resent of my patch on NFCT input plugin.
--- input/flow/ulogd_inpflow_NFCT.c | 236 ++++++++++++++++++++++++++++----------- 1 files changed, 168 insertions(+), 68 deletions(-)
input/flow/ulogd_inpflow_NFCT.c +168 -68
@@ -106,11 +106,101 @@ static struct config_keyset nfct_kset = {#define buckets_ce (x) (x->ces[3])#define maxentries_ce (x) (x->ces[4]) +enum nfct_keys { +NFCT_ORIG_IP_SADDR =0 , +NFCT_ORIG_IP_DADDR , +NFCT_ORIG_IP_PROTOCOL , +NFCT_ORIG_L4_SPORT , +NFCT_ORIG_L4_DPORT , +NFCT_ORIG_RAW_PKTLEN , +NFCT_ORIG_RAW_PKTCOUNT , +NFCT_REPLY_IP_SADDR , +NFCT_REPLY_IP_DADDR , +NFCT_REPLY_IP_PROTOCOL , +NFCT_REPLY_L4_SPORT , +NFCT_REPLY_L4_DPORT , +NFCT_REPLY_RAW_PKTLEN , +NFCT_REPLY_RAW_PKTCOUNT , +NFCT_ICMP_CODE , +NFCT_ICMP_TYPE , +NFCT_CT_MARK , +NFCT_CT_ID , +NFCT_FLOW_START_SEC , +NFCT_FLOW_START_USEC , +NFCT_FLOW_END_SEC , +NFCT_FLOW_END_USEC , +}; +static struct ulogd_key nfct_okeys[] = { { .type =ULOGD_RET_IPADDR , .flags =ULOGD_RETF_NONE , - .name ="ip.saddr" , + .name ="orig.ip.saddr" , + .ipfix = { + .vendor =IPFIX_VENDOR_IETF , + .field_id = IPFIX_sourceIPv4Address, + }, + }, + { + .type =ULOGD_RET_IPADDR , + .flags =ULOGD_RETF_NONE , + .name ="orig.ip.daddr" , + .ipfix = { + .vendor =IPFIX_VENDOR_IETF , + .field_id = IPFIX_destinationIPv4Address, + }, + }, + { + .type =ULOGD_RET_UINT8 , + .flags =ULOGD_RETF_NONE , + .name ="orig.ip.protocol" , + .ipfix = { + .vendor =IPFIX_VENDOR_IETF , + .field_id = IPFIX_protocolIdentifier, + }, + }, + { + .type =ULOGD_RET_UINT16 , + .flags =ULOGD_RETF_NONE , + .name ="orig.l4.sport" , + .ipfix = { + .vendor =IPFIX_VENDOR_IETF , + .field_id = IPFIX_sourceTransportPort, + }, + }, + { + .type =ULOGD_RET_UINT16 , + .flags =ULOGD_RETF_NONE , + .name ="orig.l4.dport" , + .ipfix = { + .vendor =IPFIX_VENDOR_IETF , + .field_id = IPFIX_destinationTransportPort, + }, + }, + { + .type =ULOGD_RET_UINT32 , + .flags =ULOGD_RETF_NONE , + .name ="orig.raw.pktlen" , + .ipfix = { + .vendor =IPFIX_VENDOR_IETF , + .field_id = IPFIX_octetTotalCount, +/* FIXME: this could also be octetDeltaCount */ + }, + }, + { + .type =ULOGD_RET_UINT32 , + .flags =ULOGD_RETF_NONE , + .name ="orig.raw.pktcount" , + .ipfix = { + .vendor =IPFIX_VENDOR_IETF , + .field_id = IPFIX_packetTotalCount, +/* FIXME: this could also be packetDeltaCount */ + }, + }, + { + .type =ULOGD_RET_IPADDR , + .flags =ULOGD_RETF_NONE , + .name ="reply.ip.saddr" , .ipfix = { .vendor =IPFIX_VENDOR_IETF , .field_id = IPFIX_sourceIPv4Address,
@@ -119,7 +209,7 @@ static struct ulogd_key nfct_okeys[] = { { .type = ULOGD_RET_IPADDR, .flags = ULOGD_RETF_NONE, - .name = "ip.daddr", + .name = "reply.ip.daddr", .ipfix = { .vendor = IPFIX_VENDOR_IETF, .field_id = IPFIX_destinationIPv4Address,
@@ -128,7 +218,7 @@ static struct ulogd_key nfct_okeys[] = { { .type = ULOGD_RET_UINT8, .flags = ULOGD_RETF_NONE, - .name = "ip.protocol", + .name = "reply.ip.protocol", .ipfix = { .vendor = IPFIX_VENDOR_IETF, .field_id = IPFIX_protocolIdentifier,
@@ -137,7 +227,7 @@ static struct ulogd_key nfct_okeys[] = { { .type = ULOGD_RET_UINT16, .flags = ULOGD_RETF_NONE, - .name = "l4.sport", + .name = "reply.l4.sport", .ipfix = { .vendor = IPFIX_VENDOR_IETF, .field_id = IPFIX_sourceTransportPort,
@@ -146,7 +236,7 @@ static struct ulogd_key nfct_okeys[] = { { .type = ULOGD_RET_UINT16, .flags = ULOGD_RETF_NONE, - .name = "l4.dport", + .name = "reply.l4.dport", .ipfix = { .vendor = IPFIX_VENDOR_IETF, .field_id = IPFIX_destinationTransportPort,
@@ -155,7 +245,7 @@ static struct ulogd_key nfct_okeys[] = { { .type = ULOGD_RET_UINT32, .flags = ULOGD_RETF_NONE, - .name = "raw.pktlen", + .name = "reply.raw.pktlen", .ipfix = { .vendor = IPFIX_VENDOR_IETF, .field_id = IPFIX_octetTotalCount,
@@ -165,7 +255,7 @@ static struct ulogd_key nfct_okeys[] = { { .type = ULOGD_RET_UINT32, .flags = ULOGD_RETF_NONE, - .name = "raw.pktcount", + .name = "reply.raw.pktcount", .ipfix = { .vendor = IPFIX_VENDOR_IETF, .field_id = IPFIX_packetTotalCount,
@@ -244,11 +334,6 @@ static struct ulogd_key nfct_okeys[] = { .field_id = IPFIX_flowEndSeconds, }, }, - { - .type = ULOGD_RET_BOOL, - .flags = ULOGD_RETF_NONE, - .name = "dir", - }, };static struct ct_htable *htable_alloc (int htable_size,int prealloc)
@@ -364,93 +449,108 @@ static struct ct_timestamp *ct_hash_get(struct ct_htable *htable, uint32_t id)return ct; } -static int propagate_ct_flow (struct ulogd_pluginstance *upi, -struct nfct_conntrack *ct, -unsigned int flags, -int dir, -struct ct_timestamp *ts) +static int propagate_ct (struct ulogd_pluginstance *upi, +struct nfct_conntrack *ct, +unsigned int flags, +struct ct_timestamp *ts) {struct ulogd_key *ret = upi->output.keys; +int dir; + + dir =NFCT_DIR_ORIGINAL ; + ret[NFCT_ORIG_IP_SADDR ].u.value.ui32 =htonl (ct->tuple[dir].src.v4); + ret[NFCT_ORIG_IP_SADDR ].flags |=ULOGD_RETF_VALID ; - ret[0 ].u.value.ui32 =htonl (ct->tuple[dir].src.v4); - ret[0 ].flags |=ULOGD_RETF_VALID ; - - ret[1 ].u.value.ui32 =htonl (ct->tuple[dir].dst.v4); - ret[1 ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_ORIG_IP_DADDR ].u.value.ui32 =htonl (ct->tuple[dir].dst.v4); + ret[NFCT_ORIG_IP_DADDR ].flags |=ULOGD_RETF_VALID ; - ret[2 ].u.value.ui8 = ct->tuple[dir].protonum; - ret[2 ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_ORIG_IP_PROTOCOL ].u.value.ui8 = ct->tuple[dir].protonum; + ret[NFCT_ORIG_IP_PROTOCOL ].flags |=ULOGD_RETF_VALID ; -switch (ct->tuple[1 ].protonum) { +switch (ct->tuple[dir].protonum) {case IPPROTO_TCP :case IPPROTO_UDP :case IPPROTO_SCTP :/* FIXME: DCCP */ - ret[3 ].u.value.ui16 =htons (ct->tuple[dir].l4src.tcp.port); - ret[3 ].flags |=ULOGD_RETF_VALID ; - ret[4 ].u.value.ui16 =htons (ct->tuple[dir].l4dst.tcp.port); - ret[4 ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_ORIG_L4_SPORT ].u.value.ui16 =htons (ct->tuple[dir].l4src.tcp.port); + ret[NFCT_ORIG_L4_SPORT ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_ORIG_L4_DPORT ].u.value.ui16 =htons (ct->tuple[dir].l4dst.tcp.port); + ret[NFCT_ORIG_L4_DPORT ].flags |=ULOGD_RETF_VALID ;break ;case IPPROTO_ICMP : - ret[7 ].u.value.ui8 = ct->tuple[dir].l4src.icmp.code; - ret[7 ].flags |=ULOGD_RETF_VALID ; - ret[8 ].u.value.ui8 = ct->tuple[dir].l4src.icmp.type; - ret[8 ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_ICMP_CODE ].u.value.ui8 = ct->tuple[dir].l4src.icmp.code; + ret[NFCT_ICMP_CODE ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_ICMP_TYPE ].u.value.ui8 = ct->tuple[dir].l4src.icmp.type; + ret[NFCT_ICMP_TYPE ].flags |=ULOGD_RETF_VALID ;break ; } -if ((dir ==NFCT_DIR_ORIGINAL && flags &NFCT_COUNTERS_ORIG ) || - (dir ==NFCT_DIR_REPLY && flags &NFCT_COUNTERS_RPLY )) { - ret[5 ].u.value.ui64 = ct->counters[dir].bytes; - ret[5 ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_ORIG_RAW_PKTLEN ].u.value.ui64 = ct->counters[dir].bytes; + ret[NFCT_ORIG_RAW_PKTLEN ].flags |=ULOGD_RETF_VALID ; + + ret[NFCT_ORIG_RAW_PKTCOUNT ].u.value.ui64 = ct->counters[dir].packets; + ret[NFCT_ORIG_RAW_PKTCOUNT ].flags |=ULOGD_RETF_VALID ; + + dir =NFCT_DIR_REPLY ; + ret[NFCT_REPLY_IP_SADDR ].u.value.ui32 =htonl (ct->tuple[dir].src.v4); + ret[NFCT_REPLY_IP_SADDR ].flags |=ULOGD_RETF_VALID ; - ret[6 ].u.value.ui64 = ct->counters[dir].packets; - ret[6 ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_REPLY_IP_DADDR ].u.value.ui32 =htonl (ct->tuple[dir].dst.v4); + ret[NFCT_REPLY_IP_DADDR ].flags |=ULOGD_RETF_VALID ; + + ret[NFCT_REPLY_IP_PROTOCOL ].u.value.ui8 = ct->tuple[dir].protonum; + ret[NFCT_REPLY_IP_PROTOCOL ].flags |=ULOGD_RETF_VALID ; + +switch (ct->tuple[dir].protonum) { +case IPPROTO_TCP : +case IPPROTO_UDP : +case IPPROTO_SCTP : +/* FIXME: DCCP */ + ret[NFCT_REPLY_L4_SPORT ].u.value.ui16 =htons (ct->tuple[dir].l4src.tcp.port); + ret[NFCT_REPLY_L4_SPORT ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_REPLY_L4_DPORT ].u.value.ui16 =htons (ct->tuple[dir].l4dst.tcp.port); + ret[NFCT_REPLY_L4_DPORT ].flags |=ULOGD_RETF_VALID ; +break ; +case IPPROTO_ICMP : + ret[NFCT_ICMP_CODE ].u.value.ui8 = ct->tuple[dir].l4src.icmp.code; + ret[NFCT_ICMP_CODE ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_ICMP_TYPE ].u.value.ui8 = ct->tuple[dir].l4src.icmp.type; + ret[NFCT_ICMP_TYPE ].flags |=ULOGD_RETF_VALID ; +break ; } + ret[NFCT_REPLY_RAW_PKTLEN ].u.value.ui64 = ct->counters[dir].bytes; + ret[NFCT_REPLY_RAW_PKTLEN ].flags |=ULOGD_RETF_VALID ; + + ret[NFCT_REPLY_RAW_PKTCOUNT ].u.value.ui64 = ct->counters[dir].packets; + ret[NFCT_REPLY_RAW_PKTCOUNT ].flags |=ULOGD_RETF_VALID ; +if (flags &NFCT_MARK ) { - ret[9 ].u.value.ui32 = ct->mark; - ret[9 ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_CT_MARK ].u.value.ui32 = ct->mark; + ret[NFCT_CT_MARK ].flags |=ULOGD_RETF_VALID ; }if (flags &NFCT_ID ) { - ret[10 ].u.value.ui32 = ct->id; - ret[10 ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_CT_ID ].u.value.ui32 = ct->id; + ret[NFCT_CT_ID ].flags |=ULOGD_RETF_VALID ; }if (ts) { - ret[11 ].u.value.ui32 = ts->time[START ].tv_sec; - ret[11 ].flags |=ULOGD_RETF_VALID ; - ret[12 ].u.value.ui32 = ts->time[START ].tv_usec; - ret[12 ].flags |=ULOGD_RETF_VALID ; - ret[13 ].u.value.ui32 = ts->time[STOP ].tv_sec; - ret[13 ].flags |=ULOGD_RETF_VALID ; - ret[14 ].u.value.ui32 = ts->time[STOP ].tv_usec; - ret[14 ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_FLOW_START_SEC ].u.value.ui32 = ts->time[START ].tv_sec; + ret[NFCT_FLOW_START_SEC ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_FLOW_START_USEC ].u.value.ui32 = ts->time[START ].tv_usec; + ret[NFCT_FLOW_START_USEC ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_FLOW_END_SEC ].u.value.ui32 = ts->time[STOP ].tv_sec; + ret[NFCT_FLOW_END_SEC ].flags |=ULOGD_RETF_VALID ; + ret[NFCT_FLOW_END_USEC ].u.value.ui32 = ts->time[STOP ].tv_usec; + ret[NFCT_FLOW_END_USEC ].flags |=ULOGD_RETF_VALID ; } - ret[15 ].u.value.b = (dir ==NFCT_DIR_ORIGINAL ) ?0 :1 ; - ret[15 ].flags |=ULOGD_RETF_VALID ; -ulogd_propagate_results (upi);return 0 ; } -static int propagate_ct (struct ulogd_pluginstance *upi, -struct nfct_conntrack *ct, -unsigned int flags, -struct ct_timestamp *ctstamp) -{ -int rc; - - rc =propagate_ct_flow (upi, ct, flags,NFCT_DIR_ORIGINAL , ctstamp); -if (rc <0 ) -return rc; - -return propagate_ct_flow (upi, ct, flags,NFCT_DIR_REPLY , ctstamp); -} -static int event_handler (void *arg,unsigned int flags,int type,void *data) {
Signature · 1 line
1.5.2.5
Re: [ULOGD2 PATCH] Sends one message for each connection event instead of two.
Quoted 1 line · Eric Leblond
This is the resent of my patch on NFCT input plugin.
Applied. Thanks Eric.
Signature · 1 line
"Los honestos son inadaptados sociales" -- Les Luthiers