Skip to contentSkip to conversation

netfilter: add connlabel conntrack extension

RFC PATCH Superseded Florian Westphal 3 participants 8 messages Oct 18, 2012 – Oct 20, 2012

Subsystems (2)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Florian Westphal

author maintainer
#0 lore raw

RFC PATCH 1/1 netfilter: add connlabel conntrack extension

When users wish to annotate connections with extra information, (e.g. "connection came in on interface eth1"), the only way of doing so at the moment is using connmarks.

As connmark is only 32-bit wide, this can become a problem when the number of desired meta-information increases (especially true when also using connmarks to label policy routing marks, and/or using dpi crap, etc).

This patch essentially adds a kernel store of (user-defined) label names. Users can then assign these names to connections, with as many names per connection as desired (current - arbitrarily chosen - limit is 32k different names total).

Also includes match/target extensions to add/match based on connection names (or "connection labels").

Notes:
- label added to a connection is "sticky", i.e. cannot be "un-set" again
- adding a new label to kernel is slow (new labels can currently only be added via netfilter ruleset).
- matching connection based on label is fast.

Things that might make sense, once this idea is deemed sane enough:
- ability for userspace to tag connections via nfqueue
- nfnetlink extension that can be used by userspace to dump list of labels, and get notifications when new labels are added/removed
- tc match

I am not sure of connlabels should interact with ctnetlink (don't think its a good idea to also send a list of all ct-assigned labels per ct-event); perhaps via a new nfnetlink extension?

Comments?
Feature requests?

Thanks,
Florian

---
 include/linux/netfilter/xt_connlabel.h      |   17 +
 include/net/netfilter/nf_conntrack_extend.h |    4 +
 include/net/netfilter/nf_conntrack_labels.h |   80 +++++
 include/net/netns/conntrack.h               |    9 +
 net/netfilter/Kconfig                       |   16 +
 net/netfilter/Makefile                      |    2 +
 net/netfilter/nf_conntrack_core.c           |   11 +
 net/netfilter/nf_conntrack_labels.c         |  468 +++++++++++++++++++++++++++
 net/netfilter/nf_conntrack_netlink.c        |    2 +
 net/netfilter/xt_connlabel.c                |  141 ++++++++
 10 files changed, 750 insertions(+), 0 deletions(-)
 create mode 100644 include/linux/netfilter/xt_connlabel.h
 create mode 100644 include/net/netfilter/nf_conntrack_labels.h
 create mode 100644 net/netfilter/nf_conntrack_labels.c
 create mode 100644 net/netfilter/xt_connlabel.c
include/linux/netfilter/xt_connlabel.h new file +17 -0
new file mode 100644
index 0000000..850167d
@@ -0,0 +1,17 @@
+#include <linux/types.h>
+
+#define XT_CONNLABEL_MAXLEN 22
+
+struct xt_connlabel_mtinfo {
+	char labelname[XT_CONNLABEL_MAXLEN-1]; /* not null terminated */
+	bool invert;
+	/* used internally by kernel: */
+	__u16 bit;
+};
+
+struct xt_connlabel_tginfo {
+	char labelname[XT_CONNLABEL_MAXLEN];
+	/* used internally by kernel: */
+	__u16 bit;
+};
+
include/net/netfilter/nf_conntrack_extend.h +4 -0
index 8b4d1fc2..977bc8a 100644
@@ -23,6 +23,9 @@ enum nf_ct_ext_id {
 #ifdef CONFIG_NF_CONNTRACK_TIMEOUT
 	NF_CT_EXT_TIMEOUT,
 #endif
+#ifdef CONFIG_NF_CONNTRACK_LABELS
+	NF_CT_EXT_LABELS,
+#endif
 	NF_CT_EXT_NUM,
 };
 
@@ -33,6 +36,7 @@ enum nf_ct_ext_id {
 #define NF_CT_EXT_ZONE_TYPE struct nf_conntrack_zone
 #define NF_CT_EXT_TSTAMP_TYPE struct nf_conn_tstamp
 #define NF_CT_EXT_TIMEOUT_TYPE struct nf_conn_timeout
+#define NF_CT_EXT_LABELS_TYPE struct nf_conn_labels
 
 /* Extensions: optional stuff which isn't permanently in struct. */
 struct nf_ct_ext {
include/net/netfilter/nf_conntrack_labels.h new file +80 -0

80 lines not on this page · raw message

include/net/netns/conntrack.h +9 -0
index a1d83cc..226cf18 100644
@@ -50,6 +50,12 @@ struct nf_icmp_net {
 	unsigned int timeout;
 };
 
+struct nf_connlabels_net {
+	struct hlist_head *ct_htable;
+	struct hlist_head *ct_htable_bits;
+	unsigned int max;
+};
+
 struct nf_ip_net {
 	struct nf_generic_net   generic;
 	struct nf_tcp_net	tcp;
@@ -83,6 +89,9 @@ struct netns_ct {
 	int			sysctl_auto_assign_helper;
 	bool			auto_assign_helper_warned;
 	struct nf_ip_net	nf_ct_proto;
+#if defined(CONFIG_NF_CONNTRACK_LABELS)
+	struct nf_connlabels_net labels;
+#endif
 #ifdef CONFIG_NF_NAT_NEEDED
 	struct hlist_head	*nat_bysource;
 	unsigned int		nat_htable_size;
net/netfilter/Kconfig +16 -0
index fefa514..80a0fdd 100644
@@ -124,6 +124,15 @@ config NF_CONNTRACK_TIMESTAMP
 
 	  If unsure, say `N'.
 
+config NF_CONNTRACK_LABELS
+	bool  'Connection tracking labels'
+	depends on NETFILTER_ADVANCED
+	help
+	  This option enables support for assigning user-defined text strings
+	  to connection tracking entries.
+
+	  If unsure, say `N'.
+
 config NF_CT_PROTO_DCCP
 	tristate 'DCCP protocol connection tracking support (EXPERIMENTAL)'
 	depends on EXPERIMENTAL
@@ -472,6 +481,13 @@ config NETFILTER_XT_SET
 
 	  To compile it as a module, choose M here.  If unsure, say N.
 
+config NETFILTER_XT_CONNLABELS
+	tristate '"connlimit" target and match support"'
+	depends on NF_CONNTRACK_LABELS
+	---help---
+	  This match allows you to match against connlabels assigned to the
+	  connection.
+
 # alphabetically ordered list of targets
 
 comment "Xtables targets"
net/netfilter/Makefile +2 -0
index 3259697..0e7139c 100644
@@ -4,6 +4,7 @@ nf_conntrack-y	:= nf_conntrack_core.o nf_conntrack_standalone.o nf_conntrack_exp
 nf_conntrack-$(CONFIG_NF_CONNTRACK_TIMEOUT) += nf_conntrack_timeout.o
 nf_conntrack-$(CONFIG_NF_CONNTRACK_TIMESTAMP) += nf_conntrack_timestamp.o
 nf_conntrack-$(CONFIG_NF_CONNTRACK_EVENTS) += nf_conntrack_ecache.o
+nf_conntrack-$(CONFIG_NF_CONNTRACK_LABELS) += nf_conntrack_labels.o
 
 obj-$(CONFIG_NETFILTER) = netfilter.o
 
@@ -70,6 +71,7 @@ obj-$(CONFIG_NETFILTER_XTABLES) += x_tables.o xt_tcpudp.o
 obj-$(CONFIG_NETFILTER_XT_MARK) += xt_mark.o
 obj-$(CONFIG_NETFILTER_XT_CONNMARK) += xt_connmark.o
 obj-$(CONFIG_NETFILTER_XT_SET) += xt_set.o
+obj-$(CONFIG_NETFILTER_XT_CONNLABELS) += xt_connlabel.o
 obj-$(CONFIG_NF_NAT) += xt_nat.o
 
 # targets
net/netfilter/nf_conntrack_core.c +11 -0
index 0f241be..fb2ef9c 100644
@@ -45,6 +45,7 @@
 #include <net/netfilter/nf_conntrack_zones.h>
 #include <net/netfilter/nf_conntrack_timestamp.h>
 #include <net/netfilter/nf_conntrack_timeout.h>
+#include <net/netfilter/nf_conntrack_labels.h>
 #include <net/netfilter/nf_nat.h>
 #include <net/netfilter/nf_nat_core.h>
 
@@ -816,6 +817,7 @@ init_conntrack(struct net *net, struct nf_conn *tmpl,
 
 	nf_ct_acct_ext_add(ct, GFP_ATOMIC);
 	nf_ct_tstamp_ext_add(ct, GFP_ATOMIC);
+	nf_ct_labels_ext_add(ct);
 
 	ecache = tmpl ? nf_ct_ecache_find(tmpl) : NULL;
 	nf_ct_ecache_ext_add(ct, ecache ? ecache->ctmask : 0,
@@ -1359,6 +1361,7 @@ static void nf_conntrack_cleanup_net(struct net *net)
 	}
 
 	nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size);
+	nf_conntrack_labels_fini(net);
 	nf_conntrack_helper_fini(net);
 	nf_conntrack_timeout_fini(net);
 	nf_conntrack_ecache_fini(net);
@@ -1585,7 +1588,15 @@ static int nf_conntrack_init_net(struct net *net)
 	ret = nf_conntrack_helper_init(net);
 	if (ret < 0)
 		goto err_helper;
+
+	ret = nf_conntrack_labels_init(net);
+	if (ret < 0)
+		goto err_labels;
+
 	return 0;
+
+err_labels:
+	nf_conntrack_helper_fini(net);
 err_helper:
 	nf_conntrack_timeout_fini(net);
 err_timeout:
net/netfilter/nf_conntrack_labels.c new file +468 -0

468 lines not on this page · raw message

net/netfilter/nf_conntrack_netlink.c +2 -0
index 7bbfb3d..70365bf 100644
@@ -43,6 +43,7 @@
 #include <net/netfilter/nf_conntrack_acct.h>
 #include <net/netfilter/nf_conntrack_zones.h>
 #include <net/netfilter/nf_conntrack_timestamp.h>
+#include <net/netfilter/nf_conntrack_labels.h>
 #ifdef CONFIG_NF_NAT_NEEDED
 #include <net/netfilter/nf_nat_core.h>
 #include <net/netfilter/nf_nat_l4proto.h>
@@ -1488,6 +1489,7 @@ ctnetlink_create_conntrack(struct net *net, u16 zone,
 
 	nf_ct_acct_ext_add(ct, GFP_ATOMIC);
 	nf_ct_tstamp_ext_add(ct, GFP_ATOMIC);
+	nf_ct_labels_ext_add(ct);
 	nf_ct_ecache_ext_add(ct, 0, 0, GFP_ATOMIC);
 	/* we must add conntrack extensions before confirmation. */
 	ct->status |= IPS_CONFIRMED;
net/netfilter/xt_connlabel.c new file +141 -0

141 lines not on this page · raw message

Signature · 1 line
1.7.8.6
#1 lore raw
Quoted 8 lines · Florian Westphal

When users wish to annotate connections with extra information, (e.g. "connection came in on interface eth1"), the only way of doing so at the moment is using connmarks.

As connmark is only 32-bit wide, this can become a problem when the number of desired meta-information increases (especially true when also using connmarks to label policy routing marks, and/or using dpi crap, etc).

Quoted 7 lines

This patch essentially adds a kernel store of (user-defined) label names. Users can then assign these names to connections, with as many names per connection as desired (current - arbitrarily chosen - limit is 32k different names total).

Also includes match/target extensions to add/match based on connection names (or "connection labels").

I like this feature, but I'd propose that the kernel stores a specific 32-bits integer instead (not connmark, something new added as extension. In user-space we would have a file like:

$ cat /etc/iptables/connlabel.conf
1 "traffic leaving from eth0"
2 "traffic entering eth0"
3 "http traffic"
...

So we can do something like:

iptables -I INPUT -i eth0 -m connlabel --set-connlabel "traffic leaving from eth0"

[ Note I'm proposing (ab)using the match so we can use it together with targets in one single rule, we're doing that for nfacct so it should not be a problem ]

The connlabel match will open the file and will look up for that string.

Then, ctnetlink only dumps a 32-bits integer with the connlabel. We can extend conntrack to translate connlabels to strings.

The kernel will work with integer, which is good for performance and memory. User-space will work on the translation.

Adding rules will be slow via iptables, but we can add some quick path in iptables-restore to open the file once and populate some array that we can use for fast lookups. Same thing for the conntrack utility.

Does this match with your initiali idea or I'm missing anything?

Regards.

author maintainer
#2 lore raw
Quoted 4 lines · Pablo Neira Ayuso
Quoted 4 lines

When users wish to annotate connections with extra information, (e.g. "connection came in on interface eth1"), the only way of doing so at the moment is using connmarks.

[..]

Quoted 8 lines
Quoted 7 lines

This patch essentially adds a kernel store of (user-defined) label names. Users can then assign these names to connections, with as many names per connection as desired (current - arbitrarily chosen - limit is 32k different names total).

Also includes match/target extensions to add/match based on connection names (or "connection labels").

Quoted 8 lines

I like this feature, but I'd propose that the kernel stores a specific 32-bits integer instead (not connmark, something new added as extension. In user-space we would have a file like:

$ cat /etc/iptables/connlabel.conf
1 "traffic leaving from eth0"
2 "traffic entering eth0"
3 "http traffic"

Awesome. Originally I wanted to hide the exact label<->number mapping from userspace but you've got me convinced.

Quoted 7 lines

So we can do something like:

iptables -I INPUT -i eth0 -m connlabel --set-connlabel "traffic leaving from eth0"

[ Note I'm proposing (ab)using the match so we can use it together with targets in one single rule, we're doing that for nfacct so it should not be a problem ]

Yuck. But ok ;-)

Quoted 5 lines

The connlabel match will open the file and will look up for that string.

Then, ctnetlink only dumps a 32-bits integer with the connlabel. We can extend conntrack to translate connlabels to strings.

The problem i have with this is that 32 unique bits is a bit (pun intended) low. If its used as enumeration its plentiful but i specifically want to be able to attach 'lots of' different labels to a conntrack entry.

Quoted 2 lines

The kernel will work with integer, which is good for performance and memory. User-space will work on the translation.

I'll rework this; the label names will be removed. Instead the kernel will store bits only.

Quoted 5 lines

Adding rules will be slow via iptables, but we can add some quick path in iptables-restore to open the file once and populate some array that we can use for fast lookups. Same thing for the conntrack utility.

Does this match with your initiali idea or I'm missing anything?

No, your suggestions should work nicely with what i had in mind.

Thanks!

#3 lore raw

Hi Florian,

On Thu, Oct 18, 2012 at 10:38:09PM +0200, Florian Westphal wrote:
[...]

Quoted 3 lines

The problem i have with this is that 32 unique bits is a bit (pun intended) low. If its used as enumeration its plentiful but i specifically want to be able to attach 'lots of' different labels to a conntrack entry.

Ah I see. You want overlapping labels, I like that idea :-)

What if we provide overlapping and non-overlapping label types, something like:

struct nf_conn_label {
        uint64_t overlapping:56,
                 enumerated:8;
};

That provides 56 overlapping labels and 256 non-overlapping labels. There will be two configuration files to be used depending on what you want. I'm not sure what amount of labels would be fine.

Probably, this alternative semantics could be useful:

struct nf_conn_label {
        uint64_t mask:56,
                 type:8;
};

For each type (we would have 2^8), we have 56 overlapping labels.

Thus, we make sure this extension only requires 64-bits (plus the extension structure, of course).

My only concern with dynamically allocated purely bit-based labels is that users may bloat the size of each conntrack entry.

Let me know.

Regards.

author maintainer
#4 lore raw
Quoted 8 lines · Pablo Neira Ayuso

On Thu, Oct 18, 2012 at 10:38:09PM +0200, Florian Westphal wrote:
[...]

Quoted 3 lines

The problem i have with this is that 32 unique bits is a bit (pun intended) low. If its used as enumeration its plentiful but i specifically want to be able to attach 'lots of' different labels to a conntrack entry.

Ah I see. You want overlapping labels, I like that idea :-)

Quoted 7 lines

What if we provide overlapping and non-overlapping label types, something like:

struct nf_conn_label {
        uint64_t overlapping:56,
                 enumerated:8;
};

I don't like it, because i don't understand the use case. The main point of the connlabels is so users can avoid having to play "mask-bit-games" with connmarks.

To illustrate this:
We use connmarks both as enumerator and flag-store:
Upper 8 bits are "enumerated" values, lowest 12 bits are enumerated too, which makes bits in-between avaialble as boolean flags.
At the moment, those 12 bits are enough but it looks like that we'll run out, eventually).

So, in a way the kernel already provides a "enumerated label type": connmark. And i do not plan on connlabels to supersede connmark; they are different tools.

Does that make sense, or did i misunderstand what you were saying?

Quoted 3 lines

That provides 56 overlapping labels and 256 non-overlapping labels. There will be two configuration files to be used depending on what you want. I'm not sure what amount of labels would be fine.

Yes, thats something I also don't know yet. My guess is that in most cases it will be unused, or very low (< 32 labels in total).

But I don't want to limit it artificially.

Quoted 2 lines

Thus, we make sure this extension only requires 64-bits (plus the extension structure, of course).

I'm thinking about making the label store grow as-needed, so we'd start with 0 (extension off), and then grow the area once connlabel-rules are added (we know the largest bit requested, so we don't need to allocate more than needed).

Quoted 2 lines

My only concern with dynamically allocated purely bit-based labels is that users may bloat the size of each conntrack entry.

True, thats a concern. I chose 32k max label limit because it amounts to one page (I admit 32k labels is insane, even 1k (128 bytes) is
probably more than enough...). I'll reduce it.

Thanks,
Florian

#5 lore raw

Hi Florian,

Quoted 21 lines · Florian Westphal
Quoted 8 lines

What if we provide overlapping and non-overlapping label types, something like:

struct nf_conn_label {
        uint64_t overlapping:56,
                 enumerated:8;
};

I don't like it, because i don't understand the use case. The main point of the connlabels is so users can avoid having to play "mask-bit-games" with connmarks.

To illustrate this:
We use connmarks both as enumerator and flag-store:
Upper 8 bits are "enumerated" values, lowest 12 bits are enumerated too, which makes bits in-between avaialble as boolean flags.
At the moment, those 12 bits are enough but it looks like that we'll run out, eventually).

Quoted 3 lines

So, in a way the kernel already provides a "enumerated label type": connmark. And i do not plan on connlabels to supersede connmark; they are different tools.

Oh, not thinking on connlabels to supersede connmark either.

Quoted 8 lines

Does that make sense, or did i misunderstand what you were saying?

Quoted 3 lines

That provides 56 overlapping labels and 256 non-overlapping labels. There will be two configuration files to be used depending on what you want. I'm not sure what amount of labels would be fine.

Yes, thats something I also don't know yet. My guess is that in most cases it will be unused, or very low (< 32 labels in total).

I was thinking on the layer-7 pre-classification from kernel-space via nfgrep.

This connlabel (or something similar) can be very useful for it. In initial states we can match several layer 7 filters until we finally conclude which one it is. For example, many protocols are based on HTTP, so the enumerate would allow to set some type and set several protocols matching. Connmark is too generic for this and it's heavily used by others.

I just think that having some clear use case for this is important.

If you're original idea is just to attach labels to help sysadmins to understand what's going on through the gateway, then we can leave this as is and add some new specific extension for nfgrep once it comes into place.

Quoted 10 lines

But I don't want to limit it artificially.

Quoted 2 lines

Thus, we make sure this extension only requires 64-bits (plus the extension structure, of course).

I'm thinking about making the label store grow as-needed, so we'd start with 0 (extension off), and then grow the area once connlabel-rules are added (we know the largest bit requested, so we don't need to allocate more than needed).

Quoted 6 lines
Quoted 2 lines

My only concern with dynamically allocated purely bit-based labels is that users may bloat the size of each conntrack entry.

True, thats a concern. I chose 32k max label limit because it amounts to one page (I admit 32k labels is insane, even 1k (128 bytes) is
probably more than enough...). I'll reduce it.

Yes, that limit sounds more reasonable to me.

author maintainer
#6 lore raw
Quoted 6 lines · Pablo Neira Ayuso

I just think that having some clear use case for this is important.

If you're original idea is just to attach labels to help sysadmins to understand what's going on through the gateway, then we can leave this as is and add some new specific extension for nfgrep once it comes into place.

No, I intend for userspace to assign labels to connections, e.g. via NFQUEUE.

Also, labels should also be made available via ctnetlink, e.g. for logging/accounting.

Example: Conntracks are interface agnostic, so you would be able to provide "came in via interface X" information via connlabels.

My main problem is currently understanding what nfgrep needs. Since you suggested to do all labelname<->number mapping in userspace, how would the nfgrep part assign a label?

Is that also done via netfilter rules, or via some "module magic" feature? It would be nice to come up with something that fits nfgrep needs, too.

Best regards,
Florian

#7 lore raw
In reply to Pablo Neira Ayuso

Hi

Quoted 5 lines
Quoted 2 lines

Yes, thats something I also don't know yet. My guess is that in most cases it will be unused, or very low (< 32 labels in total).

I was thinking on the layer-7 pre-classification from kernel-space via nfgrep.

Quoted 8 lines

This connlabel (or something similar) can be very useful for it. In initial states we can match several layer 7 filters until we finally conclude which one it is. For example, many protocols are based on HTTP, so the enumerate would allow to set some type and set several protocols matching. Connmark is too generic for this and it's heavily used by others.

I just think that having some clear use case for this is important.

Aha, perfect. I was just about to ask a question about how to do something like this!

I'm updating the opendpi-netfilter patches to run against nDPI, which is the ntop fork of opendpi. This seems to be a very decent attempt to do something like layer 7 heuristics and in particular does a rather clever labelling of https connections by inspecting the certs themselves (so we can block/allow facebook despite it being encrypted)
https://github.com/ewildgoose/ndpi-netfilter

What I would very much desire is to be able to extent this to conntrack so I can easily get summary stats back through userspace conntrack. So I would like to simply log the output of "conntrack -E" and group by some protocol column

However, I have a second use case that isn't covered by this:
- For use with a captive portal I need to calculate bandwidth used by the WAN connection
- Some WAN connections have an overhead because they are wrapped inside some outer protocol (ATM would be an example, eg where we transmit over an ADSL connection the billed bandwidth is higher than the ethernet bandwidth). Or other connections may use PPP header compression, etc
- I desire to have conntrack userspace have visibility over the *charged* bandwidth so again I can very easily use the output as a fairly accurate billing mechanism
- Also I need to know which WAN connection the packets went out via (since billing will be different per WAN). This isn't computable from routing since we use policy routing. Also there is NAT on all outbound connections.

Given that this isn't satisfied by the proposed labelling system, how else might I implement this requirement please? Suggestions appreciated?

Cheers

Ed W