: Flexible SFQ flow classification

One good thing about ESFQ is the more flexible flow classification, but I don't like the concept of having a set of selectable hash functions very much.

These patches change SFQ to allow attaching external classifiers and add a new "flow" classifier that allows to classify flows based on an arbitary combination of pre-defined keys. Its probably not the fastest classifier when used with multiple keys, but frankly, I don't think speed is very important in most situations where the current SFQ implementation is used.

It currently does not support perturbation, I didn't want to move this into the classifier, so I need to think about a way to handle it within SFQ.

Some examples:

# behave identical to internal SFQ hash
tc filter add ... flow baseclass x:1 classes 1024 \
		       keys src,dst,proto-src,proto-dst

# the same, but based on source address/port before NAT
tc filter add ... flow baseclass x:1 classes 1024 \
		       keys nfct-src,dst,nfct-proto-src,proto-dst

# classify based on UID
tc filter add ... flow baseclass x:1 classes 1024 \
		       keys sk-uid

and so on .. check out the iproute help text for the full set of supported keys.

Comments welcome.

 include/linux/pkt_cls.h |   37 +++
 net/sched/Kconfig       |   11 
 net/sched/Makefile      |    1 
 net/sched/cls_flow.c    |  570 ++++++++++++++++++++++++++++++++++++++++++++++++
 net/sched/sch_sfq.c     |   98 +++++++-
 5 files changed, 713 insertions(+), 4 deletions(-)

Patrick McHardy (2):
[NET_SCHED]: sch_sfq: add support for external classifiers [NET_SCHED]: Add flow classifier

[NET_SCHED]: sch_sfq: add support for external classifiers

Add support for external classifiers to allow using different flow hash functions similar to ESFQ. As long as no classifier is attached the built-in hash is used as before.

---
commit 666e402224e5ca36af0b5a07d424b9c092bce91e
tree f0e020de640693ba9601cc2e4b82daba7bf03689
parent c420bc9f09a0926b708c3edb27eacba434a4f4ba
author Patrick McHardy <[email protected]> Wed, 30 May 2007 11:21:05 +0200 committer Patrick McHardy <[email protected]> Wed, 30 May 2007 11:21:05 +0200

 net/sched/sch_sfq.c |   98 +++++++++++++++++++++++++++++++++++++++++++++++++--
 1 files changed, 94 insertions(+), 4 deletions(-)

index 96dfdf7..42cd4fc 100644
@@ -108,6 +108,7 @@ struct sfq_sched_data
 	int		limit;
 
 /* Variables */
+	struct tcf_proto *filter_list;
 	struct timer_list perturb_timer;
 	int		perturbation;
 	sfq_index	tail;		/* Index of current slot in round */

@@ -172,6 +173,42 @@ static unsigned sfq_hash(struct sfq_sched_data *q, struct sk_buff *skb)
 	return sfq_fold_hash(q, h, h2);
 }
 
+static unsigned int sfq_classify(struct sk_buff *skb, struct Qdisc *sch,
+				 int *qerr)
+{
+	struct sfq_sched_data *q = qdisc_priv(sch);
+	struct tcf_result res;
+	int result;
+
+	if (TC_H_MAJ(skb->priority) == sch->handle &&
+	    TC_H_MIN(skb->priority) > 0 &&
+	    TC_H_MIN(skb->priority) <= SFQ_HASH_DIVISOR)
+		return TC_H_MIN(skb->priority);
+
+	if (!q->filter_list)
+		return sfq_hash(q, skb) + 1;
+
+	*qerr = NET_XMIT_BYPASS;
+	result = tc_classify(skb, q->filter_list, &res);
+	if (result >= 0) {
+#ifdef CONFIG_NET_CLS_ACT
+		switch (result) {
+		case TC_ACT_STOLEN:
+		case TC_ACT_QUEUED:
+			*qerr = NET_XMIT_SUCCESS;
+		case TC_ACT_SHOT:
+			return 0;
+		}
+#elif defined(CONFIG_NET_CLS_POLICE)
+		if (result == TC_POLICE_SHOT)
+			return 0;
+#endif
+		if (TC_H_MIN(res.classid) <= SFQ_HASH_DIVISOR)
+			return TC_H_MIN(res.classid);
+	}
+	return 0;
+}
+
 static inline void sfq_link(struct sfq_sched_data *q, sfq_index x)
 {
 	sfq_index p, n;

@@ -262,8 +299,18 @@ static int
 sfq_enqueue(struct sk_buff *skb, struct Qdisc* sch)
 {
 	struct sfq_sched_data *q = qdisc_priv(sch);
-	unsigned hash = sfq_hash(q, skb);
+	unsigned int hash;
 	sfq_index x;
+	int ret;
+
+	hash = sfq_classify(skb, sch, &ret);
+	if (hash == 0) {
+		if (ret == NET_XMIT_BYPASS)
+			sch->qstats.drops++;
+		kfree_skb(skb);
+		return ret;
+	}
+	hash--;
 
 	x = q->ht[hash];
 	if (x == SFQ_DEPTH) {

@@ -298,8 +345,18 @@ static int
 sfq_requeue(struct sk_buff *skb, struct Qdisc* sch)
 {
 	struct sfq_sched_data *q = qdisc_priv(sch);
-	unsigned hash = sfq_hash(q, skb);
+	unsigned int hash;
 	sfq_index x;
+	int ret;
+
+	hash = sfq_classify(skb, sch, &ret);
+	if (hash == 0) {
+		if (ret == NET_XMIT_BYPASS)
+			sch->qstats.drops++;
+		kfree_skb(skb);
+		return ret;
+	}
+	hash--;
 
 	x = q->ht[hash];
 	if (x == SFQ_DEPTH) {

@@ -456,6 +513,8 @@ static int sfq_init(struct Qdisc *sch, struct rtattr *opt)
 static void sfq_destroy(struct Qdisc *sch)
 {
 	struct sfq_sched_data *q = qdisc_priv(sch);
+
+	tcf_destroy_chain(q->filter_list);
 	del_timer(&q->perturb_timer);
 }
 

@@ -481,9 +540,40 @@ rtattr_failure:
 	return -1;
 }
 
+static int sfq_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
+			    struct rtattr **tca, unsigned long *arg)
+{
+	return -ENOSYS;
+}
+
+static unsigned long sfq_get(struct Qdisc *sch, u32 classid)
+{
+	return 0;
+}
+
+static struct tcf_proto **sfq_find_tcf(struct Qdisc *sch, unsigned long cl)
+{
+	struct sfq_sched_data *q = qdisc_priv(sch);
+
+	if (cl)
+		return NULL;
+	return &q->filter_list;
+}
+
+static void sfq_walk(struct Qdisc *sch, struct qdisc_walker *walker)
+{
+	return;
+}
+
+static struct Qdisc_class_ops sfq_class_ops = {
+	.get		=	sfq_get,
+	.change		=	sfq_change_class,
+	.tcf_chain	=	sfq_find_tcf,
+	.walk		=	sfq_walk,
+};
+
 static struct Qdisc_ops sfq_qdisc_ops = {
-	.next		=	NULL,
-	.cl_ops		=	NULL,
+	.cl_ops		=	&sfq_class_ops,
 	.id		=	"sfq",
 	.priv_size	=	sizeof(struct sfq_sched_data),
 	.enqueue	=	sfq_enqueue,

[NET_SCHED]: Add flow classifier

---
commit e98a4dd4f6cef4d46f180dee2e91e7f359a854c3
tree f56ea6da8115c5ee9fbc30421f9c4e392a8c55a7
parent 666e402224e5ca36af0b5a07d424b9c092bce91e
author Patrick McHardy <[email protected]> Wed, 30 May 2007 11:22:49 +0200 committer Patrick McHardy <[email protected]> Wed, 30 May 2007 11:22:49 +0200

 include/linux/pkt_cls.h |   37 +++
 net/sched/Kconfig       |   11 +
 net/sched/Makefile      |    1 
 net/sched/cls_flow.c    |  570 +++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 619 insertions(+), 0 deletions(-)

index c3f01b3..0137591 100644
@@ -328,6 +328,43 @@ enum
 
 #define TCA_TCINDEX_MAX     (__TCA_TCINDEX_MAX - 1)
 
+/* Flow filter */
+
+enum
+{
+	FLOW_KEY_SRC,
+	FLOW_KEY_DST,
+	FLOW_KEY_PROTO_SRC,
+	FLOW_KEY_PROTO_DST,
+	FLOW_KEY_PRIORITY,
+	FLOW_KEY_MARK,
+	FLOW_KEY_NFCT,
+	FLOW_KEY_NFCT_SRC,
+	FLOW_KEY_NFCT_DST,
+	FLOW_KEY_NFCT_PROTO_SRC,
+	FLOW_KEY_NFCT_PROTO_DST,
+	FLOW_KEY_RTIIF,
+	FLOW_KEY_RTCLASSID,
+	FLOW_KEY_SKUID,
+	FLOW_KEY_SKGID,
+	__FLOW_KEY_MAX,
+};
+
+#define FLOW_KEY_MAX	(__FLOW_KEY_MAX - 1)
+
+enum
+{
+	TCA_FLOW_UNSPEC,
+	TCA_FLOW_KEYS,
+	TCA_FLOW_BASECLASS,
+	TCA_FLOW_CLASSES,
+	TCA_FLOW_ACT,
+	TCA_FLOW_POLICE,
+	__TCA_FLOW_MAX
+};
+
+#define TCA_FLOW_MAX	(__TCA_FLOW_MAX - 1)
+
 /* Basic filter */
 
 enum

index 475df84..6f33a79 100644
@@ -302,6 +302,17 @@ config NET_CLS_RSVP6
 	  To compile this code as a module, choose M here: the
 	  module will be called cls_rsvp6.
 
+config NET_CLS_FLOW
+	tristate "Flow classifier"
+	select NET_CLS
+	---help---
+	  If you say Y here, you will be able to classify packets based on
+	  a configurable combination of packet keys. This is mostly useful
+	  in combination with SFQ.
+
+	  To compile this code as a module, choose M here: the
+	  module will be called cls_flow.
+
 config NET_EMATCH
 	bool "Extended Matches"
 	select NET_CLS

index 020767a..54ad85a 100644
@@ -35,6 +35,7 @@ obj-$(CONFIG_NET_CLS_RSVP)	+= cls_rsvp.o
 obj-$(CONFIG_NET_CLS_TCINDEX)	+= cls_tcindex.o
 obj-$(CONFIG_NET_CLS_RSVP6)	+= cls_rsvp6.o
 obj-$(CONFIG_NET_CLS_BASIC)	+= cls_basic.o
+obj-$(CONFIG_NET_CLS_FLOW)	+= cls_flow.o
 obj-$(CONFIG_NET_EMATCH)	+= ematch.o
 obj-$(CONFIG_NET_EMATCH_CMP)	+= em_cmp.o
 obj-$(CONFIG_NET_EMATCH_NBYTE)	+= em_nbyte.o

Cool, but isn't this going to show the same collision problems that the pre jhash esfq saw?

Andy.

Perturbation doesn't prevent collisions, it just distributes them (hopefully evenly). My classifier uses jhash, but that won't prevent collisions either. Anyways, I'm going to change SFQ so perturbation can also be used with external classifiers.

Patrick McHardy wrote:
My classifier uses jhash,

Ahh that's OK - I thought it still used the old sfq hash, which collided alot with a low number of consecutive addresses.

Andy.

In the spirit of SFQ it is probably ok to do that;
i.e iirc, the idea for justification behind sfq was to provide a "computationaly feasible/reasonable" way to provide fair queueing with a gazillion queues.
Classification was considered damn expensive in those days (when MC68K was sort of state of the art); it still is but maybe a much much lesser issue now for what "gazillion" was in those days. So a simple hash on a few fields with pertubation (the part that makes allows the "stochastic" part in the name) was deemed the way to go and in order to provide fairness (while avoiding packet reordering).
So if you want to keep that spirit it is ok to do what ESFQ does; I think the assumptions will still be valid if you have a gazillion queues in todays terms. A number like say 128K may make sense. (Hey Paul M is still around, just too focused on the wrong things like RCU;-> so we can ask his opinion)

The only one thing i noticed that changes the behavior is the use of skb->prio as a selector. I think if you removed that it should be fine. Another alternative is to create a brand new FQ qdisc and leave the classification to the classifiers.

It is kind of hard to put it back into the current approach because the basic assumptions of ensuring no re-ordering and a "fast" classifier are gone.

I am almost tempted to say go back and write a qdisc called FQ.

cheers,
jamal

Sure. The thing I don't like about the predefined hash functions is that its unflexible.

I don't think thats a problem, it needs to point to the correct major to have any effect, which can only happen if it is set by the user. I would prefer to keep it for consistency with other qdiscs.

I created a new classifier to leave classification to the classifiers .. Not sure exactly why I would need a new qdisc to do that :)

It doesn't affect performance in any way, but I agree that it doesn't belong in a classifier. But it should be possible to do it in SFQ.

Funny, last the this came up you suggested to do basically exactly what this classifier does, which I thought made sense :)

http://www.mail-archive.com/[email protected]/msg06801.html

agreed.

The caveat/difference with current SFQ is you have allowed the user to define which queue is selected. It is/was dynamically selected based on packet header now/before. Thats the main reason i said maybe you separate the two components totaly. i.e while FQ is useful on its own and can use other classifiers; a useful classifier IMO for SFQ is one that rips out the sfq_hash or whatever other schemes used in ESFQ into a classifier (I suppose then the user can select which hash is used). In such a classifier you can restore the pertub into it.
I am not sure i made sense.

I am almost sensing i am contradicting myself in that thread;-> It is hard to tell and i admit to being forgetful - but what i probably meant is what i said above.

cheers,
jamal

I think exposing SFQ's queues as classes is a good thing, it allows you to do whatever classification you want. In fact I'm probably going to add patch on top to also dump them to userspace. What remains for SFQ to do is serve the queues evenly.

My classifier seperates them entirely. The only thing it keeps in SFQ is the old classifier for compatibility, besides that its exactly what you say. It should be easily possible to remove it entirely and use my classifier in a compatible configuration automatically.

I'm sensing you didn't look at the patch :)

Yes, that would be useful.

And it does (yes, I looked at the patch;->).

If you removed it entirely (and had it as a separate classifier) IMO that would be a better approach. Then what you have is a pure FQ qdisc. In which case, you leave alone SFQ and have a new qdisc.
The "removed" hashing dynamic classifier would of course be better off it allowed the user to select a hash algorithm such as the other ones specified in ESFQ.

cheers,
jamal

I could do that, but I'm perfectly happy with the qdisc part of SFQ. Without the classifier SFQ is of course simply a FQ qdisc, all it cares about is serving queues equally.

Sure. You are writting the code - your call.

cheers,
jamal

Not only that, he is -really- slow about getting to some of his email sometimes. :-/ In my defense, if you CC me, I will spot it more quickly.

Yeah, the 1980s were indeed over a -long- time ago, and architectures certainly have changed. I wrote my first parallel kernel code about a year after the SFQ paper was published, and not too many years after that began my longstanding RCU habit. And not only did I inhale at the time, I am still inhaling deeply. ;-)

In any case, if you can feasibly do an exact classification, then doing so would most likely be better than using SFQ. And with today's hardware, exact classifications are much reasonable than they were back on my old handful-of-MHz 68K-based Sun workstation. If an exact classification is unworkable, but you absolutely have to maintain perfect ordering, then one approach is to let the SFQ drain before perturbing the hash function.

One way to do this is to have a pair of hash tables, and switch back and forth between them on each perturbation. Perturbation then goes as follows:

1. Select a new hash perturbation value, and associate it with the
currently-unused hash table (call it B).

2. Swing pointers so that subsequent incoming packets go to B.

3. Continue outputting from A until it is drained. If the
perturbation is lockless, you might need to ensure that a grace period passes during this drain operation. (Yep, still inhaling!!!)

4. Start outputting packets from B.

But there might be better approaches.

And this might well be a better approach. ;-)

Thanx, Paul