Skip to contentSkip to conversation

BPF updates

PATCH Superseded Daniel Borkmann 4 participants 10 messages Sep 1, 2015 – Sep 2, 2015

Subsystems (20)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Daniel Borkmann

author maintainer
#0 lore raw

cover BPF updates

This series adds a couple of improvements to bpf classifier and actions as well as mqprio. Please see individual patches for details. The series was still in my queue, I realize it's rather late, so I also don't have any problem deferring this for the next net-next cycle if it's preferred.

Thanks (also to Alexei for his reviews)!

Daniel Borkmann (3):
ebpf: migrate bpf_prog's flags to bitfield
net: {cls,act}_bpf: add helper for retrieving routing realms
net: {cls,act}_bpf: make skb->priority writable

John Fastabend (1):
net: qdisc: add op to run filters/actions before enqueue

 arch/arm/net/bpf_jit_32.c       |  2 +-
 arch/arm64/net/bpf_jit_comp.c   |  2 +-
 arch/mips/net/bpf_jit.c         |  2 +-
 arch/powerpc/net/bpf_jit_comp.c |  2 +-
 arch/s390/net/bpf_jit_comp.c    |  2 +-
 arch/sparc/net/bpf_jit_comp.c   |  2 +-
 arch/x86/net/bpf_jit_comp.c     |  2 +-
 include/linux/filter.h          |  7 +++++--
 include/net/sch_generic.h       | 16 +++++++++++-----
 include/uapi/linux/bpf.h        |  7 +++++++
 kernel/bpf/core.c               |  4 ++++
 kernel/bpf/syscall.c            |  6 ++++--
 net/core/dev.c                  | 17 +++++++++++++++++
 net/core/filter.c               | 33 ++++++++++++++++++++++++++++++---
 net/sched/Kconfig               |  5 +++++
 net/sched/cls_bpf.c             |  9 ++++++---
 net/sched/sch_generic.c         |  1 +
 net/sched/sch_mqprio.c          | 35 +++++++++++++++++++++++++++++++++++
 18 files changed, 132 insertions(+), 22 deletions(-)
Signature · 1 line
1.9.3
author maintainer
#1 lore raw

PATCH 1/4 net: qdisc: add op to run filters/actions before enqueue

From: John Fastabend <[email protected]>

Add a new ->preclassify() op to allow multiqueue queuing disciplines to call tc_classify() or perform other work before dev_pick_tx().

This helps, for example, with mqprio queueing discipline that has offload support by most popular 10G NICs, where the txq effectively picks the qdisc.

Once traffic is being directed to a specific queue then hardware TX rings may be tuned to support this traffic type. mqprio already gives the ability to do this via skb->priority where the ->preclassify() provides more control over packet steering, it can classify the skb and set the priority, for example, from an eBPF classifier (or action).

Also this allows traffic classifiers to be run without holding the qdisc lock and gives one place to attach filters when mqprio is in use. ->preclassify() could also be added to other mq qdiscs later
on: f.e. most classful qdiscs first check major/minor numbers of skb->priority before actually consulting a more complex classifier.

For mqprio case today, a filter has to be attached to each txq qdisc to have all traffic hit the filter. Since ->preclassify() is currently only used by mqprio, the __dev_queue_xmit() fast path is guarded by a generic, hidden Kconfig option (NET_CLS_PRECLASSIFY) that is only selected by mqprio, otherwise it defaults to off. Also, the Qdisc structure size will stay the same, we move __parent, used by cbq only into a write-mostly hole. If actions are enabled, __parent is written on every enqueue, and only read, rewritten in reshape_fail() phase. Therefore, this place in the read-mostly cacheline could be used by preclassify, which is written only once.

Joint work with Daniel Borkmann.

---
 include/net/sch_generic.h | 16 +++++++++++-----
 net/core/dev.c            | 17 +++++++++++++++++
 net/sched/Kconfig         |  5 +++++
 net/sched/sch_generic.c   |  1 +
 net/sched/sch_mqprio.c    | 35 +++++++++++++++++++++++++++++++++++
 5 files changed, 69 insertions(+), 5 deletions(-)
include/net/sch_generic.h +11 -5
index 444faa8..e65767d 100644
@@ -69,13 +69,11 @@ struct Qdisc {
 	u32			parent;
 	int			(*reshape_fail)(struct sk_buff *skb,
 					struct Qdisc *q);
+	int			(*preclassify)(struct sk_buff *skb,
+					       struct Qdisc *q);
 
 	void			*u32_node;
 
-	/* This field is deprecated, but it is still used by CBQ
-	 * and it will live until better solution will be invented.
-	 */
-	struct Qdisc		*__parent;
 	struct netdev_queue	*dev_queue;
 
 	struct gnet_stats_rate_est64	rate_est;
@@ -93,6 +91,14 @@ struct Qdisc {
 	unsigned int		__state;
 	struct gnet_stats_queue	qstats;
 	struct rcu_head		rcu_head;
+
+	/* This field is deprecated, but it is still used by CBQ
+	 * and it will live until better solution will be invented.
+	 * If actions are enabled, it's modified on every enqueue,
+	 * and only rw-accessed in slow-path (cbq_reshape_fail).
+	 */
+	struct Qdisc		*__parent;
+
 	int			padded;
 	atomic_t		refcnt;
 
@@ -184,6 +190,7 @@ struct Qdisc_ops {
 
 	int 			(*enqueue)(struct sk_buff *, struct Qdisc *);
 	struct sk_buff *	(*dequeue)(struct Qdisc *);
+	int			(*preclassify)(struct sk_buff *, struct Qdisc *);
 	struct sk_buff *	(*peek)(struct Qdisc *);
 	unsigned int		(*drop)(struct Qdisc *);
 
@@ -199,7 +206,6 @@ struct Qdisc_ops {
 	struct module		*owner;
 };
 
-
 struct tcf_result {
 	unsigned long	class;
 	u32		classid;
net/core/dev.c +17 -0
index 877c848..b768bca 100644
@@ -3052,6 +3052,23 @@ static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
 	rcu_read_lock_bh();
 
 	skb_update_prio(skb);
+#ifdef CONFIG_NET_CLS_PRECLASSIFY
+	q = rcu_dereference_bh(dev->qdisc);
+	if (q && q->preclassify) {
+		switch (q->preclassify(skb, q)) {
+		default:
+			break;
+#ifdef CONFIG_NET_CLS_ACT
+		case TC_ACT_SHOT:
+		case TC_ACT_STOLEN:
+		case TC_ACT_QUEUED:
+			kfree_skb(skb);
+			rc = NET_XMIT_SUCCESS;
+			goto out;
+#endif
+		}
+	}
+#endif
 
 	/* If device/qdisc don't need skb->dst, release it right now while
 	 * its hot in this cpu cache.
net/sched/Kconfig +5 -0
index daa3343..c235756 100644
@@ -219,6 +219,7 @@ config NET_SCH_DRR
 
 config NET_SCH_MQPRIO
 	tristate "Multi-queue priority scheduler (MQPRIO)"
+	select NET_CLS_PRECLASSIFY
 	help
 	  Say Y here if you want to use the Multi-queue Priority scheduler.
 	  This scheduler allows QOS to be offloaded on NICs that have support
@@ -351,6 +352,10 @@ comment "Classification"
 config NET_CLS
 	bool
 
+config NET_CLS_PRECLASSIFY
+	bool
+	default n
+
 config NET_CLS_BASIC
 	tristate "Elementary classification (BASIC)"
 	select NET_CLS
net/sched/sch_generic.c +1 -0
index cb5d4ad..ef27ea2 100644
@@ -605,6 +605,7 @@ struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
 	sch->ops = ops;
 	sch->enqueue = ops->enqueue;
 	sch->dequeue = ops->dequeue;
+	sch->preclassify = ops->preclassify;
 	sch->dev_queue = dev_queue;
 	dev_hold(dev);
 	atomic_set(&sch->refcnt, 1);
net/sched/sch_mqprio.c +35 -0
index 3811a74..7e251cc 100644
@@ -20,6 +20,7 @@
 #include <net/sch_generic.h>
 
 struct mqprio_sched {
+	struct tcf_proto __rcu *filter_list;
 	struct Qdisc		**qdiscs;
 	int hw_owned;
 };
@@ -28,9 +29,13 @@ static void mqprio_destroy(struct Qdisc *sch)
 {
 	struct net_device *dev = qdisc_dev(sch);
 	struct mqprio_sched *priv = qdisc_priv(sch);
+	struct tcf_proto *filter_list;
 	unsigned int ntx;
 
 	if (priv->qdiscs) {
+		filter_list = rtnl_dereference(priv->filter_list);
+		tcf_destroy_chain(&filter_list);
+
 		for (ntx = 0;
 		     ntx < dev->num_tx_queues && priv->qdiscs[ntx];
 		     ntx++)
@@ -391,12 +396,41 @@ static void mqprio_walk(struct Qdisc *sch, struct qdisc_walker *arg)
 	}
 }
 
+static struct tcf_proto __rcu **mqprio_find_tcf(struct Qdisc *sch,
+						unsigned long cl)
+{
+	struct mqprio_sched *q = qdisc_priv(sch);
+
+	if (cl)
+		return NULL;
+
+	return &q->filter_list;
+}
+
+static unsigned long mqprio_bind(struct Qdisc *sch, unsigned long parent,
+				 u32 classid)
+{
+	return 0;
+}
+
+static int mqprio_classify(struct sk_buff *skb, struct Qdisc *sch)
+{
+	struct mqprio_sched *q = qdisc_priv(sch);
+	struct tcf_proto *fl = rcu_dereference_bh(q->filter_list);
+	struct tcf_result res;
+
+	return tc_classify(skb, fl, &res, false);
+}
+
 static const struct Qdisc_class_ops mqprio_class_ops = {
 	.graft		= mqprio_graft,
 	.leaf		= mqprio_leaf,
 	.get		= mqprio_get,
 	.put		= mqprio_put,
 	.walk		= mqprio_walk,
+	.tcf_chain	= mqprio_find_tcf,
+	.bind_tcf	= mqprio_bind,
+	.unbind_tcf	= mqprio_put,
 	.dump		= mqprio_dump_class,
 	.dump_stats	= mqprio_dump_class_stats,
 };
@@ -407,6 +441,7 @@ static struct Qdisc_ops mqprio_qdisc_ops __read_mostly = {
 	.priv_size	= sizeof(struct mqprio_sched),
 	.init		= mqprio_init,
 	.destroy	= mqprio_destroy,
+	.preclassify	= mqprio_classify,
 	.attach		= mqprio_attach,
 	.dump		= mqprio_dump,
 	.owner		= THIS_MODULE,
Signature · 1 line
1.9.3
#2 lore raw

Re: [PATCH net-next 1/4] net: qdisc: add op to run filters/actions before enqueue

Quoted 21 lines · Daniel Borkmann

From: John Fastabend <[email protected]>

Add a new ->preclassify() op to allow multiqueue queuing disciplines to call tc_classify() or perform other work before dev_pick_tx().

This helps, for example, with mqprio queueing discipline that has offload support by most popular 10G NICs, where the txq effectively picks the qdisc.

Once traffic is being directed to a specific queue then hardware TX rings may be tuned to support this traffic type. mqprio already gives the ability to do this via skb->priority where the ->preclassify() provides more control over packet steering, it can classify the skb and set the priority, for example, from an eBPF classifier (or action).

Also this allows traffic classifiers to be run without holding the qdisc lock and gives one place to attach filters when mqprio is in use. ->preclassify() could also be added to other mq qdiscs later
on: f.e. most classful qdiscs first check major/minor numbers of skb->priority before actually consulting a more complex classifier.

Quoted 5 lines

For mqprio case today, a filter has to be attached to each txq qdisc to have all traffic hit the filter. Since ->preclassify() is currently only used by mqprio, the __dev_queue_xmit() fast path is guarded by a generic, hidden Kconfig option (NET_CLS_PRECLASSIFY) that is only selected by mqprio,

So all distros will select it, basically.

...

Quoted 16 lines
net/core/dev.c +8 -0
index 877c848..b768bca 100644
@@ -3052,6 +3052,23 @@ static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
 	rcu_read_lock_bh();
 
 	skb_update_prio(skb);
+#ifdef CONFIG_NET_CLS_PRECLASSIFY
+	q = rcu_dereference_bh(dev->qdisc);
+	if (q && q->preclassify) {
+		switch (q->preclassify(skb, q)) {
+		default:
+			break;
+#ifdef CONFIG_NET_CLS_ACT
+		case TC_ACT_SHOT:

net/core/dev.c static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv) · line 3072

Quoted 10 lines
net/core/dev.c +9 -0
+		case TC_ACT_STOLEN:
+		case TC_ACT_QUEUED:
+			kfree_skb(skb);
+			rc = NET_XMIT_SUCCESS;
+			goto out;
+#endif
+		}
+	}
+#endif
 

Since its a device attribute after all, why are you storing it in dev->qdisc->preclassify, adding a cache line miss for moderate load ?

(mqprio/mq root qdisc is normally not fetched in fast path ?)

dev->preclassify would be better IMO, close to dev->_tx

author maintainer
#3 lore raw

Re: [PATCH net-next 1/4] net: qdisc: add op to run filters/actions before enqueue

Quoted 62 lines · Eric Dumazet
Quoted 21 lines · Daniel Borkmann

From: John Fastabend <[email protected]>

Add a new ->preclassify() op to allow multiqueue queuing disciplines to call tc_classify() or perform other work before dev_pick_tx().

This helps, for example, with mqprio queueing discipline that has offload support by most popular 10G NICs, where the txq effectively picks the qdisc.

Once traffic is being directed to a specific queue then hardware TX rings may be tuned to support this traffic type. mqprio already gives the ability to do this via skb->priority where the ->preclassify() provides more control over packet steering, it can classify the skb and set the priority, for example, from an eBPF classifier (or action).

Also this allows traffic classifiers to be run without holding the qdisc lock and gives one place to attach filters when mqprio is in use. ->preclassify() could also be added to other mq qdiscs later
on: f.e. most classful qdiscs first check major/minor numbers of skb->priority before actually consulting a more complex classifier.

Quoted 5 lines

For mqprio case today, a filter has to be attached to each txq qdisc to have all traffic hit the filter. Since ->preclassify() is currently only used by mqprio, the __dev_queue_xmit() fast path is guarded by a generic, hidden Kconfig option (NET_CLS_PRECLASSIFY) that is only selected by mqprio,

So all distros will select it, basically.

...

Quoted 16 lines
net/core/dev.c +8 -0
index 877c848..b768bca 100644
@@ -3052,6 +3052,23 @@ static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
  	rcu_read_lock_bh();
 
  	skb_update_prio(skb);
+#ifdef CONFIG_NET_CLS_PRECLASSIFY
+	q = rcu_dereference_bh(dev->qdisc);
+	if (q && q->preclassify) {
+		switch (q->preclassify(skb, q)) {
+		default:
+			break;
+#ifdef CONFIG_NET_CLS_ACT
+		case TC_ACT_SHOT:

net/core/dev.c static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv) · line 3072

Quoted 10 lines
net/core/dev.c +9 -0
+		case TC_ACT_STOLEN:
+		case TC_ACT_QUEUED:
+			kfree_skb(skb);
+			rc = NET_XMIT_SUCCESS;
+			goto out;
+#endif
+		}
+	}
+#endif
 

Since its a device attribute after all, why are you storing it in dev->qdisc->preclassify, adding a cache line miss for moderate load ?

Quoted 3 lines

(mqprio/mq root qdisc is normally not fetched in fast path ?)

dev->preclassify would be better IMO, close to dev->_tx

Yes, makes sense, as this cacheline is accessed anyway few cycles later. I'll look into how well this approach integrates into tc's configuration path. I think mqprio/mq/... could just install/remove dev->preclassify() handler as soon as first filter is attached/detached.

Thanks,
Daniel

#4 lore raw

Re: [PATCH net-next 1/4] net: qdisc: add op to run filters/actions before enqueue

In reply to Daniel Borkmann

(Why not Cc'ing Jamal for net_sched pathes?)

Quoted 21 lines · Daniel Borkmann

From: John Fastabend <[email protected]>

Add a new ->preclassify() op to allow multiqueue queuing disciplines to call tc_classify() or perform other work before dev_pick_tx().

This helps, for example, with mqprio queueing discipline that has offload support by most popular 10G NICs, where the txq effectively picks the qdisc.

Once traffic is being directed to a specific queue then hardware TX rings may be tuned to support this traffic type. mqprio already gives the ability to do this via skb->priority where the ->preclassify() provides more control over packet steering, it can classify the skb and set the priority, for example, from an eBPF classifier (or action).

Also this allows traffic classifiers to be run without holding the qdisc lock and gives one place to attach filters when mqprio is in use. ->preclassify() could also be added to other mq qdiscs later
on: f.e. most classful qdiscs first check major/minor numbers of skb->priority before actually consulting a more complex classifier.

Quoted 11 lines

For mqprio case today, a filter has to be attached to each txq qdisc to have all traffic hit the filter. Since ->preclassify() is currently only used by mqprio, the __dev_queue_xmit() fast path is guarded by a generic, hidden Kconfig option (NET_CLS_PRECLASSIFY) that is only selected by mqprio, otherwise it defaults to off. Also, the Qdisc structure size will stay the same, we move __parent, used by cbq only into a write-mostly hole. If actions are enabled, __parent is written on every enqueue, and only read, rewritten in reshape_fail() phase. Therefore, this place in the read-mostly cacheline could be used by preclassify, which is written only once.

I don't like this approach. Ideally, qdisc layer should be totally on top of tx queues, which means tx queue selection should happen after dequeue. I looked at this before, the change is not trivial at all given the fact that qdisc ties too much with tx queue probably due to historical reasons, especially the tx softirq part. But that is really a long-term solution for me.

I have no big objection for this as a short-term solution, however, once we add these filters before enqueue, we can't remove them any more. We really need to think twice about it.

Jamal, do you have any better idea?

author maintainer
#5 lore raw

Re: [PATCH net-next 1/4] net: qdisc: add op to run filters/actions before enqueue

Quoted 1 line · Cong Wang

(Why not Cc'ing Jamal for net_sched pathes?)

Sorry, forgot about it, and thanks for the Cc!

maintainer
#6 lore raw

Re: [PATCH net-next 1/4] net: qdisc: add op to run filters/actions before enqueue

In reply to Cong Wang
Quoted 43 lines · Cong Wang

(Why not Cc'ing Jamal for net_sched pathes?)

Quoted 21 lines · Daniel Borkmann

From: John Fastabend <[email protected]>

Add a new ->preclassify() op to allow multiqueue queuing disciplines to call tc_classify() or perform other work before dev_pick_tx().

This helps, for example, with mqprio queueing discipline that has offload support by most popular 10G NICs, where the txq effectively picks the qdisc.

Once traffic is being directed to a specific queue then hardware TX rings may be tuned to support this traffic type. mqprio already gives the ability to do this via skb->priority where the ->preclassify() provides more control over packet steering, it can classify the skb and set the priority, for example, from an eBPF classifier (or action).

Also this allows traffic classifiers to be run without holding the qdisc lock and gives one place to attach filters when mqprio is in use. ->preclassify() could also be added to other mq qdiscs later
on: f.e. most classful qdiscs first check major/minor numbers of skb->priority before actually consulting a more complex classifier.

Quoted 11 lines

For mqprio case today, a filter has to be attached to each txq qdisc to have all traffic hit the filter. Since ->preclassify() is currently only used by mqprio, the __dev_queue_xmit() fast path is guarded by a generic, hidden Kconfig option (NET_CLS_PRECLASSIFY) that is only selected by mqprio, otherwise it defaults to off. Also, the Qdisc structure size will stay the same, we move __parent, used by cbq only into a write-mostly hole. If actions are enabled, __parent is written on every enqueue, and only read, rewritten in reshape_fail() phase. Therefore, this place in the read-mostly cacheline could be used by preclassify, which is written only once.

I don't like this approach. Ideally, qdisc layer should be totally on top of tx queues, which means tx queue selection should happen after dequeue. I looked at this before, the change is not trivial at all given the fact that qdisc ties too much with tx queue probably due to historical reasons, especially the tx softirq part. But that is really a long-term solution for me.

Quoted 6 lines

I have no big objection for this as a short-term solution, however, once we add these filters before enqueue, we can't remove them any more. We really need to think twice about it.

Jamal, do you have any better idea?

Sorry for the top quote:
Given the rcu-fication of classifiers i believe the idea will mostly work; expect user will go nuts sticking all kinds of classifiers and actions that wont work (example, I dont think connmark action would work nicely here).
Could we strive to do proper offload ala switchdev?

The comment on the patch on reshape_fail + __parent: for the record, that is an extremely useful feature (allows an inner qdisc to provide an opportunity for a classful parent qdisc to reclassify and therefore reschedule).
Yes, CBQ is the only user - but maybe if it was properly documented more schedulers could put it to good use.

cheers,
jamal

author maintainer
#7 lore raw

PATCH 2/4 ebpf: migrate bpf_prog's flags to bitfield

In reply to Daniel Borkmann

As we need to add further flags to the bpf_prog structure, lets migrate both bools to a bitfield representation. The size of the base structure (excluding insns) remains unchanged at 40 bytes.

Add also tags for the kmemchecker, so that it doesn't throw false positives. Even in case gcc would generate suboptimal code, it's not being accessed in performance critical paths.

---
 arch/arm/net/bpf_jit_32.c       | 2 +-
 arch/arm64/net/bpf_jit_comp.c   | 2 +-
 arch/mips/net/bpf_jit.c         | 2 +-
 arch/powerpc/net/bpf_jit_comp.c | 2 +-
 arch/s390/net/bpf_jit_comp.c    | 2 +-
 arch/sparc/net/bpf_jit_comp.c   | 2 +-
 arch/x86/net/bpf_jit_comp.c     | 2 +-
 include/linux/filter.h          | 6 ++++--
 kernel/bpf/core.c               | 4 ++++
 kernel/bpf/syscall.c            | 4 ++--
 net/core/filter.c               | 2 +-
 11 files changed, 18 insertions(+), 12 deletions(-)
arch/arm/net/bpf_jit_32.c +1 -1
index 876060b..0df5fd5 100644
@@ -1047,7 +1047,7 @@ void bpf_jit_compile(struct bpf_prog *fp)
 
 	set_memory_ro((unsigned long)header, header->pages);
 	fp->bpf_func = (void *)ctx.target;
-	fp->jited = true;
+	fp->jited = 1;
 out:
 	kfree(ctx.offsets);
 	return;
arch/arm64/net/bpf_jit_comp.c +1 -1
index c047598..a44e529 100644
@@ -744,7 +744,7 @@ void bpf_int_jit_compile(struct bpf_prog *prog)
 
 	set_memory_ro((unsigned long)header, header->pages);
 	prog->bpf_func = (void *)ctx.image;
-	prog->jited = true;
+	prog->jited = 1;
 out:
 	kfree(ctx.offset);
 }
arch/mips/net/bpf_jit.c +1 -1
index 0c4a133..77cb273 100644
@@ -1251,7 +1251,7 @@ void bpf_jit_compile(struct bpf_prog *fp)
 		bpf_jit_dump(fp->len, alloc_size, 2, ctx.target);
 
 	fp->bpf_func = (void *)ctx.target;
-	fp->jited = true;
+	fp->jited = 1;
 
 out:
 	kfree(ctx.offsets);
arch/powerpc/net/bpf_jit_comp.c +1 -1
index 17cea18..0478216 100644
@@ -679,7 +679,7 @@ void bpf_jit_compile(struct bpf_prog *fp)
 		((u64 *)image)[1] = local_paca->kernel_toc;
 #endif
 		fp->bpf_func = (void *)image;
-		fp->jited = true;
+		fp->jited = 1;
 	}
 out:
 	kfree(addrs);
arch/s390/net/bpf_jit_comp.c +1 -1
index eeda051..9a0c4c2 100644
@@ -1310,7 +1310,7 @@ void bpf_int_jit_compile(struct bpf_prog *fp)
 	if (jit.prg_buf) {
 		set_memory_ro((unsigned long)header, header->pages);
 		fp->bpf_func = (void *) jit.prg_buf;
-		fp->jited = true;
+		fp->jited = 1;
 	}
 free_addrs:
 	kfree(jit.addrs);
arch/sparc/net/bpf_jit_comp.c +1 -1
index f8b9f71..22564f5 100644
@@ -812,7 +812,7 @@ cond_branch:			f_offset = addrs[i + filter[i].jf];
 	if (image) {
 		bpf_flush_icache(image, image + proglen);
 		fp->bpf_func = (void *)image;
-		fp->jited = true;
+		fp->jited = 1;
 	}
 out:
 	kfree(addrs);
arch/x86/net/bpf_jit_comp.c +1 -1
index 70efcd0..7599197 100644
@@ -1109,7 +1109,7 @@ void bpf_int_jit_compile(struct bpf_prog *prog)
 		bpf_flush_icache(header, image + proglen);
 		set_memory_ro((unsigned long)header, header->pages);
 		prog->bpf_func = (void *)image;
-		prog->jited = true;
+		prog->jited = 1;
 	}
 out:
 	kfree(addrs);
include/linux/filter.h +4 -2
index fa2cab9..bad618f 100644
@@ -326,8 +326,10 @@ struct bpf_binary_header {
 
 struct bpf_prog {
 	u16			pages;		/* Number of allocated pages */
-	bool			jited;		/* Is our filter JIT'ed? */
-	bool			gpl_compatible;	/* Is our filter GPL compatible? */
+	kmemcheck_bitfield_begin(meta);
+	u16			jited:1,	/* Is our filter JIT'ed? */
+				gpl_compatible:1; /* Is filter GPL compatible? */
+	kmemcheck_bitfield_end(meta);
 	u32			len;		/* Number of filter blocks */
 	enum bpf_prog_type	type;		/* Type of BPF program */
 	struct bpf_prog_aux	*aux;		/* Auxiliary fields */
kernel/bpf/core.c +4 -0
index 67c380c..c8855c2 100644
@@ -82,6 +82,8 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags)
 	if (fp == NULL)
 		return NULL;
 
+	kmemcheck_annotate_bitfield(fp, meta);
+
 	aux = kzalloc(sizeof(*aux), GFP_KERNEL | gfp_extra_flags);
 	if (aux == NULL) {
 		vfree(fp);
@@ -110,6 +112,8 @@ struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,
 
 	fp = __vmalloc(size, gfp_flags, PAGE_KERNEL);
 	if (fp != NULL) {
+		kmemcheck_annotate_bitfield(fp, meta);
+
 		memcpy(fp, fp_old, fp_old->pages * PAGE_SIZE);
 		fp->pages = size / PAGE_SIZE;
 
kernel/bpf/syscall.c +2 -2
index dc9b464..2ba2881 100644
@@ -549,10 +549,10 @@ static int bpf_prog_load(union bpf_attr *attr)
 		goto free_prog;
 
 	prog->orig_prog = NULL;
-	prog->jited = false;
+	prog->jited = 0;
 
 	atomic_set(&prog->aux->refcnt, 1);
-	prog->gpl_compatible = is_gpl;
+	prog->gpl_compatible = is_gpl ? 1 : 0;
 
 	/* find program type: socket_filter vs tracing_filter */
 	err = find_prog_type(type, prog);
net/core/filter.c +1 -1
index 13079f0..e1c574c 100644
@@ -1001,7 +1001,7 @@ static struct bpf_prog *bpf_prepare_filter(struct bpf_prog *fp,
 	int err;
 
 	fp->bpf_func = NULL;
-	fp->jited = false;
+	fp->jited = 0;
 
 	err = bpf_check_classic(fp->insns, fp->len);
 	if (err) {
Signature · 1 line
1.9.3
author maintainer
#8 lore raw

PATCH 3/4 net: {cls,act}_bpf: add helper for retrieving routing realms

In reply to Daniel Borkmann

Using routing realms as part of the classifier is quite useful. It can be viewed as a tag for one or multiple routing entries (think of an analogy to net_cls cgroup for processes), set by user space routing daemons or via iproute2 as an indicator for traffic classifiers.

In case we use a ->preclassify() handler, we can read them out for free, on other devices we need to indicate that the dst must be kept however until skb destruction.

Unlike actions, at least the classifier can keep track of it and enable netif_keep_dst() if necessary. tc actions don't have that possibility, but in case people know exactly what they are doing, it can be used from there as well (e.g. via preclassify()).

If a realm is set, the handler returns the non-zero realm. User space can set the full 32bit realm for the dst.

---
 include/linux/filter.h   |  3 ++-
 include/uapi/linux/bpf.h |  7 +++++++
 kernel/bpf/syscall.c     |  2 ++
 net/core/filter.c        | 22 ++++++++++++++++++++++
 net/sched/cls_bpf.c      |  9 ++++++---
 5 files changed, 39 insertions(+), 4 deletions(-)
include/linux/filter.h +2 -1
index bad618f..3d5fd24 100644
@@ -328,7 +328,8 @@ struct bpf_prog {
 	u16			pages;		/* Number of allocated pages */
 	kmemcheck_bitfield_begin(meta);
 	u16			jited:1,	/* Is our filter JIT'ed? */
-				gpl_compatible:1; /* Is filter GPL compatible? */
+				gpl_compatible:1, /* Is filter GPL compatible? */
+				dst_needed:1;	/* Do we need dst entry? */
 	kmemcheck_bitfield_end(meta);
 	u32			len;		/* Number of filter blocks */
 	enum bpf_prog_type	type;		/* Type of BPF program */
include/uapi/linux/bpf.h +7 -0
index 92a48e2..5ea72bb 100644
@@ -272,6 +272,13 @@ enum bpf_func_id {
 	BPF_FUNC_skb_get_tunnel_key,
 	BPF_FUNC_skb_set_tunnel_key,
 	BPF_FUNC_perf_event_read,	/* u64 bpf_perf_event_read(&map, index) */
+
+	/**
+	 * bpf_get_route_realm(skb) - retrieve a dst's tclassid
+	 * @skb: pointer to skb
+	 * Return: realm if != 0
+	 */
+	BPF_FUNC_get_route_realm,
 	__BPF_FUNC_MAX_ID,
 };
 
kernel/bpf/syscall.c +2 -0
index 2ba2881..a12046b 100644
@@ -398,6 +398,8 @@ static void fixup_bpf_calls(struct bpf_prog *prog)
 			 */
 			BUG_ON(!prog->aux->ops->get_func_proto);
 
+			if (insn->imm == BPF_FUNC_get_route_realm)
+				prog->dst_needed = 1;
 			if (insn->imm == BPF_FUNC_tail_call) {
 				/* mark bpf_tail_call as different opcode
 				 * to avoid conditional branch in
net/core/filter.c +22 -0
index e1c574c..3765ae1 100644
@@ -49,6 +49,7 @@
 #include <net/sch_generic.h>
 #include <net/cls_cgroup.h>
 #include <net/dst_metadata.h>
+#include <net/dst.h>
 
 /**
  *	sk_filter - run a packet through a socket filter
@@ -1439,6 +1440,25 @@ static const struct bpf_func_proto bpf_get_cgroup_classid_proto = {
 	.arg1_type      = ARG_PTR_TO_CTX,
 };
 
+static u64 bpf_get_route_realm(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
+{
+#ifdef CONFIG_IP_ROUTE_CLASSID
+	const struct dst_entry *dst;
+
+	dst = skb_dst((struct sk_buff *) (unsigned long) r1);
+	if (dst)
+		return dst->tclassid;
+#endif
+	return 0;
+}
+
+static const struct bpf_func_proto bpf_get_route_realm_proto = {
+	.func           = bpf_get_route_realm,
+	.gpl_only       = false,
+	.ret_type       = RET_INTEGER,
+	.arg1_type      = ARG_PTR_TO_CTX,
+};
+
 static u64 bpf_skb_vlan_push(u64 r1, u64 r2, u64 vlan_tci, u64 r4, u64 r5)
 {
 	struct sk_buff *skb = (struct sk_buff *) (long) r1;
@@ -1607,6 +1627,8 @@ tc_cls_act_func_proto(enum bpf_func_id func_id)
 		return &bpf_skb_get_tunnel_key_proto;
 	case BPF_FUNC_skb_set_tunnel_key:
 		return bpf_get_skb_set_tunnel_key_proto();
+	case BPF_FUNC_get_route_realm:
+		return &bpf_get_route_realm_proto;
 	default:
 		return sk_filter_func_proto(func_id);
 	}
net/sched/cls_bpf.c +6 -3
index e5168f8..b6163b2 100644
@@ -237,8 +237,8 @@ static int cls_bpf_prog_from_ops(struct nlattr **tb,
 	return 0;
 }
 
-static int cls_bpf_prog_from_efd(struct nlattr **tb,
-				 struct cls_bpf_prog *prog, u32 classid)
+static int cls_bpf_prog_from_efd(struct nlattr **tb, struct cls_bpf_prog *prog,
+				 u32 classid, const struct tcf_proto *tp)
 {
 	struct bpf_prog *fp;
 	char *name = NULL;
@@ -272,6 +272,9 @@ static int cls_bpf_prog_from_efd(struct nlattr **tb,
 	prog->filter = fp;
 	prog->res.classid = classid;
 
+	if (fp->dst_needed && !tp->q->preclassify)
+		netif_keep_dst(qdisc_dev(tp->q));
+
 	return 0;
 }
 
@@ -300,7 +303,7 @@ static int cls_bpf_modify_existing(struct net *net, struct tcf_proto *tp,
 	classid = nla_get_u32(tb[TCA_BPF_CLASSID]);
 
 	ret = is_bpf ? cls_bpf_prog_from_ops(tb, prog, classid) :
-		       cls_bpf_prog_from_efd(tb, prog, classid);
+		       cls_bpf_prog_from_efd(tb, prog, classid, tp);
 	if (ret < 0) {
 		tcf_exts_destroy(&exts);
 		return ret;
Signature · 1 line
1.9.3
author maintainer
#9 lore raw

PATCH 4/4 net: {cls,act}_bpf: make skb->priority writable

In reply to Daniel Borkmann

Now with the new ->preclassify() handler in place, {cls,act}_bpf can set the skb->priority from an eBPF program based on various critera, so that qdiscs like mqprio can pick it up.

---
 net/core/filter.c | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)
net/core/filter.c +7 -2
index 3765ae1..a88ea8e 100644
@@ -1674,6 +1674,7 @@ static bool tc_cls_act_is_valid_access(int off, int size,
 		switch (off) {
 		case offsetof(struct __sk_buff, mark):
 		case offsetof(struct __sk_buff, tc_index):
+		case offsetof(struct __sk_buff, priority):
 		case offsetof(struct __sk_buff, cb[0]) ...
 			offsetof(struct __sk_buff, cb[4]):
 			break;
@@ -1715,8 +1716,12 @@ static u32 bpf_net_convert_ctx_access(enum bpf_access_type type, int dst_reg,
 	case offsetof(struct __sk_buff, priority):
 		BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, priority) != 4);
 
-		*insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg,
-				      offsetof(struct sk_buff, priority));
+		if (type == BPF_WRITE)
+			*insn++ = BPF_STX_MEM(BPF_W, dst_reg, src_reg,
+					      offsetof(struct sk_buff, priority));
+		else
+			*insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg,
+					      offsetof(struct sk_buff, priority));
 		break;
 
 	case offsetof(struct __sk_buff, ingress_ifindex):
Signature · 1 line
1.9.3