net_cls: traffic counter based on classification control cgroup
Subsystems (6)
RFC v2 1/1 net_cls: traffic counter based on classification control cgroup
Hello.
It's second version of patch I already sent to netdev.
The main goal of this patch it's counting traffic for process placed to net_cls cgroup (ingress and egress). It's based on res_counters and holds counter per network interfaces.
Description of patch.
It handles packets in net/core/dev.c for egress and in /net/ipv4/tcp.c|udp.c for ingress. These places were chosen because we need to know also network interface.
Cgroup fs interface provides following files additional to existing net_cls files:
net_cls.ifacename.usage_in_bytes
Containing rcv/snd lines.
Also this patch adds to net_cls ability to handle a network device registration.
It could be included or excluded in compile time.
I moved the menu entry for "Control group classifier" from network/QoS to General Option/Control Group.
I'm waiting for you comments.
Signature · 9 lines
Best regards, Alexey Perevalov, Leading Software Engineer, phone: +7 (495) 797 25 00 ext 3969 e-mail: [email protected] <mailto:a.perevalov-LDgkF/uwaJFWk0Htik3J/[email protected]> Mobile group, Moscow Samsung Research Center 12 Dvintsev street, building 1 127018, Moscow, Russian Federation
From bc59794d7fcc75de0c7a408860fd6ec3be8c50fe Mon Sep 17 00:00:00 2001
From: Alexey Perevalov <[email protected]>
Date: Fri, 26 Oct 2012 17:45:44 +0400
Subject: [PATCH] Traffic statistics based on packet classification control
group
--- include/net/cls_cgroup.h | 203 ++++++++++++++++++++++++++++++++++---- include/net/cls_counter_holder.h | 26 +++++ init/Kconfig | 25 +++++ kernel/cgroup.c | 2 + kernel/res_counter.c | 4 + net/core/dev.c | 6 ++ net/ipv4/tcp.c | 27 ++++- net/ipv4/udp.c | 6 ++ net/sched/Kconfig | 11 --- net/sched/Makefile | 1 + net/sched/cls_cgroup.c | 192 ++++++++++++++++++++++++++++++++++- net/sched/cls_counter_holder.c | 145 +++++++++++++++++++++++++++ 12 files changed, 613 insertions(+), 35 deletions(-) create mode 100644 include/net/cls_counter_holder.h create mode 100644 net/sched/cls_counter_holder.c
include/net/cls_cgroup.h +183 -20
@@ -17,50 +17,198 @@#include <linux/hardirq.h> #include <linux/rcupdate.h> +#if IS_ENABLED (CONFIG_NET_CLS_COUNTER ) +#include <linux/nsproxy.h> +#include <linux/res_counter.h> +#include <linux/string.h> +#include <linux/skbuff.h> +#include <net/cls_counter_holder.h> +#include <net/sock.h> + +/*TODO hide all it to separate file*/ + +struct cls_iface_cntrs { +char *dev_name; +struct res_counter snd_counter; +struct res_counter rcv_counter; +struct list_head link; +}; + +#endif /*CONFIG_NET_CLS_COUNTER*/ + +#if IS_ENABLED (CONFIG_NET_CLS_CGROUP )struct cgroup_cls_state {struct cgroup_subsys_state css;u32 classid; +#if IS_ENABLED (CONFIG_NET_CLS_COUNTER ) +struct cls_iface_cntrs iface_stats; +#endif /*CONFIG_NET_CLS_COUNTER*/ };extern void sock_update_classid (struct sock *sk,struct task_struct *task); -#if IS_BUILTIN (CONFIG_NET_CLS_CGROUP ) -static inline u32 task_cls_classid (struct task_struct *p) +#if IS_MODULE (CONFIG_NET_CLS_CGROUP ) +static inline struct cgroup_cls_state *get_cls_cgroup (struct task_struct *p) { -u32 classid; +struct cgroup_subsys_state *css =task_subsys_state (p, + net_cls_subsys_id); +if (css) +return container_of (css, + struct cgroup_cls_state, css); +return NULL ; +} +#elif IS_BUILTIN (CONFIG_NET_CLS_CGROUP ) +static inline struct cgroup_cls_state *get_cls_cgroup (struct task_struct *p) +{ +return container_of (task_subsys_state (p, net_cls_subsys_id), + struct cgroup_cls_state, css); +} +#endif -if (in_interrupt ()) -return 0 ; -rcu_read_lock (); - classid =container_of (task_subsys_state (p, net_cls_subsys_id), - struct cgroup_cls_state, css)->classid; -rcu_read_unlock (); +#endif /*CONFIG_NET_CLS_CGROUP*/ -return classid; +#if IS_ENABLED (CONFIG_NET_CLS_CGROUP ) +#if IS_ENABLED (CONFIG_NET_CLS_COUNTER ) +static inline u32 skb_cls_classid (const struct sk_buff *skb) +{ +return (skb && skb->sk) ? skb->sk->sk_classid :0 ; +} + +static inline int get_ifindex_from_skb (const struct sk_buff *skb) +{ +int ifindex =0 ; +if (skb) + ifindex = skb->skb_iif; +return ifindex; +} + +static struct cls_iface_cntrs *find_cls_counter ( +struct cgroup_cls_state *cls_cgroup, +const char *dev_name, +bool create) +{ +/*TODO Add lock*/ +struct cls_iface_cntrs *entry =NULL ; + +if (!dev_name) { +pr_err ("cls please provide valid dev name" ); +return NULL ; + } + +list_for_each_entry (entry, &cls_cgroup->iface_stats.link, link) +if (!strcmp (entry->dev_name, dev_name)) +return entry; + +if (!create) +return entry; + +/*not found, insert*/ + entry =kmalloc (sizeof (struct cls_iface_cntrs ),GFP_ATOMIC ); + entry->dev_name =kstrdup (dev_name,GFP_ATOMIC ); +memset (&entry->rcv_counter,0 ,sizeof (struct res_counter )); +memset (&entry->snd_counter,0 ,sizeof (struct res_counter )); +res_counter_init (&entry->rcv_counter,NULL ); +res_counter_init (&entry->snd_counter,NULL ); +list_add_tail (&entry->link, &cls_cgroup->iface_stats.link); +return entry; } -#elif IS_MODULE (CONFIG_NET_CLS_CGROUP ) + +static void charge_net_cls_snd (struct cgroup_cls_state *cls_cgroup, +const u32 copied,const char *dev_name) +{ +struct res_counter *fail_res; +int res; +struct cls_iface_cntrs *cnt =find_cls_counter (cls_cgroup, + dev_name, true); + +if (!cnt) +return ; + + res =res_counter_charge (&cnt->snd_counter, copied, &fail_res); +} + +static char *get_dev_name (const int ifindex) +{ +struct net *net =NULL ; +struct nsproxy *nsproxy =NULL ; +struct net_device *net_dev =NULL ; + + nsproxy =task_nsproxy (current); +if (!nsproxy) { +pr_debug ("cls cant find task_nsproxy" ); +return NULL ; + } + + net =get_net (nsproxy->net_ns); +if (!net) { +pr_debug ("cls cant find net" ); +return NULL ; + } + net_dev =dev_get_by_index (net, ifindex); + +return net_dev ? net_dev->name :NULL ; +} + +static void charge_net_cls_rcv (struct cgroup_cls_state *cls_cgroup, +const u32 copied,const int ifindex) +{ +char *dev_name =get_dev_name (ifindex); +struct res_counter *fail_res; +int res; +struct cls_iface_cntrs *cnt =find_cls_counter (cls_cgroup, + dev_name, true); + +if (!cnt) +return ; + + res =res_counter_charge (&cnt->rcv_counter, copied, &fail_res); +} + +static inline void count_cls_rcv (struct task_struct *p,const u32 copied,const int ifindex) +{ +struct cgroup_cls_state *cls_cgroup; + + cls_cgroup =get_cls_cgroup (p); + +if (cls_cgroup) +charge_net_cls_rcv (cls_cgroup, copied, ifindex); +} + +static inline void count_cls_snd (u32 classid,const u32 copied, +const char *dev_name) +{ +struct cgroup_cls_state *cls_cgroup; + + cls_cgroup =find_cls_cgroup_by_classid (classid); + +if (cls_cgroup) +charge_net_cls_snd (cls_cgroup, copied, dev_name); +} +#endif /*CONFIG_NET_CLS_COUNTER*/ +static inline u32 task_cls_classid (struct task_struct *p) { -struct cgroup_subsys_state *css; -u32 classid =0 ; +int classid =0 ; +struct cgroup_cls_state *cls_cgroup =NULL ;if (in_interrupt ())return 0 ;rcu_read_lock (); - css =task_subsys_state (p, net_cls_subsys_id); -if (css) - classid =container_of (css, - struct cgroup_cls_state, css)->classid; + + cls_cgroup =get_cls_cgroup (p); +if (cls_cgroup) + classid = cls_cgroup->classid; +rcu_read_unlock ();return classid; } -#endif -#else /* !CGROUP_NET_CLS_CGROUP */ + +#else /* !CONFIG_NET_CLS_CGROUP */ static inline void sock_update_classid (struct sock *sk,struct task_struct *task) { }
@@ -69,5 +217,20 @@ static inline u32 task_cls_classid(struct task_struct *p) {return 0 ; } -#endif /* CGROUP_NET_CLS_CGROUP */ +#endif /* CONFIG_NET_CLS_CGROUP */ + +#if !IS_ENABLED (CONFIG_NET_CLS_CGROUP ) || !IS_ENABLED (CONFIG_NET_CLS_COUNTER ) +static inline void count_cls_rcv (struct task_struct *p,const u32 copied,const int ifindex) +{ +} + +static inline void count_cls_snd (u32 classid,const u32 copied,const char *dev_name) +{ +} + +static inline u32 skb_cls_classid (const struct sk_buff *skb) +{ +return 0 ; +} +#endif #endif /* _NET_CLS_CGROUP_H */
include/net/cls_counter_holder.h new file +26 -0
@@ -0,0 +1,26 @@ +/* +* cls_counter_holder.c Interface for holding references of the +* net cls cgroup instances. +* +* Authors: Alexey Perevalov, <[email protected]> +* +* Changes: +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version +* 2 of the License, or (at your option) any later version. +*/ + +#ifndef _NET_CLS_COUNTER_HOLDER_H_ +#define _NET_CLS_COUNTER_HOLDER_H_ + +#include <net/cls_cgroup.h> + +struct cgroup_cls_state ; + +void insert_cls_cgroup_entry (struct cgroup_cls_state *obj); +void delete_cls_cgroup_entry (const u32 classid); +struct cgroup_cls_state *find_cls_cgroup_by_classid (const u32 classid); + + +#endif /* _NET_CLS_COUNTER_HOLDER_H_ */
init/Kconfig +25 -0
@@ -841,6 +841,31 @@ config CGROUP_HUGETLB control group is tracked in the third page lru pointer. This means that we cannot use the controller with huge page less than 3 pages. +menuconfig NET_CLS_CGROUP + tristate "Control Group Classifier" + select NET_CLS + depends on CGROUPS + ---help--- + Say Y here if you want to classify packets based on the control + cgroup of their process. + + To compile this code as a module, choose M here: the + module will be called cls_cgroup. + +if NET_CLS_CGROUP +config NET_CLS_COUNTER + bool "Network traffic counter for network Control Group Classifier" + select NET_CLS + default n + depends on NET_CLS_CGROUP && RESOURCE_COUNTERS + ---help--- + Say Y here if you want to count traffic associate with the control + cgroup. + + To add functionality to cls_cgroup select y. + +endif #NET_CLS_CGROUP + config CGROUP_PERF bool "Enable perf_event per-cpu per-container group (cgroup) monitoring" depends on PERF_EVENTS && CGROUPS
kernel/cgroup.c +2 -0
@@ -2966,6 +2966,8 @@ int cgroup_rm_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)cgroup_cfts_commit (ss,NULL , false);return -ENOENT ; } +EXPORT_SYMBOL_GPL (cgroup_rm_cftypes); +/** * cgroup_task_count - count the number of tasks in a cgroup.
kernel/res_counter.c +4 -0
@@ -13,6 +13,8 @@#include <linux/res_counter.h> #include <linux/uaccess.h> #include <linux/mm.h> +#include <linux/export.h> +void res_counter_init (struct res_counter *counter,struct res_counter *parent) {
@@ -21,6 +23,7 @@ void res_counter_init(struct res_counter *counter, struct res_counter *parent) counter->soft_limit =RESOURCE_MAX ; counter->parent = parent; } +EXPORT_SYMBOL (res_counter_init);int res_counter_charge_locked (struct res_counter *counter,unsigned long val,bool force)
@@ -170,6 +173,7 @@ u64 res_counter_read_u64(struct res_counter *counter, int member) return *res_counter_member(counter, member); }#endif +EXPORT_SYMBOL (res_counter_read_u64);int res_counter_memparse_write_strategy (const char *buf,unsigned long long *res)
net/core/dev.c +6 -0
@@ -135,6 +135,7 @@#include <linux/net_tstamp.h> #include <linux/static_key.h> #include <net/flow_keys.h> +#include <net/cls_cgroup.h> #include "net-sysfs.h"
@@ -2570,6 +2571,11 @@ int dev_queue_xmit(struct sk_buff *skb)*/ rcu_read_lock_bh (); +#if IS_ENABLED (CONFIG_NET_CLS_COUNTER ) +if (dev) +count_cls_snd (skb_cls_classid (skb), skb->len, dev->name); +#endif +skb_update_prio (skb); txq =netdev_pick_tx (dev, skb);
net/ipv4/tcp.c +26 -1
@@ -276,6 +276,7 @@#include <net/ip.h> #include <net/netdma.h> #include <net/sock.h> +#include <net/cls_cgroup.h> #include <asm/uaccess.h> #include <asm/ioctls.h>
@@ -1467,6 +1468,9 @@ int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,u32 seq = tp->copied_seq;u32 offset;int copied =0 ; +#if IS_ENABLED (CONFIG_NET_CLS_COUNTER ) +int ifindex =0 ; +#endif if (sk->sk_state ==TCP_LISTEN )return -ENOTCONN ;
@@ -1509,6 +1513,9 @@ int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, ++seq;break ; } +#if IS_ENABLED (CONFIG_NET_CLS_COUNTER ) + ifindex =get_ifindex_from_skb (skb); +#endif sk_eat_skb (sk, skb, false);if (!desc->count)break ;
@@ -1519,8 +1526,12 @@ int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,tcp_rcv_space_adjust (sk);/* Clean up data we have read: This will do ACK frames. */ -if (copied >0 ) +if (copied >0 ) {tcp_cleanup_rbuf (sk, copied); +#if IS_ENABLED (CONFIG_NET_CLS_COUNTER ) +count_cls_rcv (current, copied, ifindex); +#endif + }return copied; }EXPORT_SYMBOL (tcp_read_sock);
@@ -1548,6 +1559,9 @@ int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,bool copied_early = false;struct sk_buff *skb;u32 urg_hole =0 ; +#if IS_ENABLED (CONFIG_NET_CLS_COUNTER ) +int ifindex =0 ; +#endif lock_sock (sk);
@@ -1872,6 +1886,9 @@ skip_copy:if (tcp_hdr (skb)->fin) goto found_fin_ok;if (!(flags &MSG_PEEK )) { +#if IS_ENABLED (CONFIG_NET_CLS_COUNTER ) + ifindex =get_ifindex_from_skb (skb); +#endif sk_eat_skb (sk, skb, copied_early); copied_early = false; }
@@ -1881,6 +1898,9 @@ skip_copy:/* Process the FIN. */ ++*seq;if (!(flags &MSG_PEEK )) { +#if IS_ENABLED (CONFIG_NET_CLS_COUNTER ) + ifindex =get_ifindex_from_skb (skb); +#endif sk_eat_skb (sk, skb, copied_early); copied_early = false; }
@@ -1923,6 +1943,11 @@ skip_copy:/* Clean up data we have read: This will do ACK frames. */ tcp_cleanup_rbuf (sk, copied); +#if IS_ENABLED (CONFIG_NET_CLS_COUNTER ) +if (copied >0 ) +count_cls_rcv (current, copied, ifindex); +#endif +release_sock (sk);return copied;
net/ipv4/udp.c +6 -0
@@ -101,6 +101,7 @@#include <linux/skbuff.h> #include <linux/proc_fs.h> #include <linux/seq_file.h> +#include <net/cls_cgroup.h> #include <net/net_namespace.h> #include <net/icmp.h> #include <net/route.h>
@@ -1254,6 +1255,11 @@ try_again:if (flags &MSG_TRUNC ) err = ulen; +#if IS_ENABLED (CONFIG_NET_CLS_COUNTER ) +if (ulen >0 ) +count_cls_rcv (current, ulen,get_ifindex_from_skb (skb)); +#endif + out_free:skb_free_datagram_locked (sk, skb); out:
net/sched/Kconfig +0 -11
@@ -418,17 +418,6 @@ config NET_CLS_FLOW To compile this code as a module, choose M here: the module will be called cls_flow. -config NET_CLS_CGROUP - tristate "Control Group Classifier" - select NET_CLS - depends on CGROUPS - ---help--- - Say Y here if you want to classify packets based on the control - cgroup of their process. - - To compile this code as a module, choose M here: the - module will be called cls_cgroup. - config NET_EMATCH bool "Extended Matches" select NET_CLS
net/sched/Makefile +1 -0
@@ -49,6 +49,7 @@ obj-$(CONFIG_NET_CLS_RSVP6) += cls_rsvp6.o obj-$(CONFIG_NET_CLS_BASIC) += cls_basic.o obj-$(CONFIG_NET_CLS_FLOW) += cls_flow.o obj-$(CONFIG_NET_CLS_CGROUP) += cls_cgroup.o +obj-$(CONFIG_NET_CLS_COUNTER) += cls_counter_holder.o obj-$(CONFIG_NET_EMATCH) += ematch.o obj-$(CONFIG_NET_EMATCH_CMP) += em_cmp.o obj-$(CONFIG_NET_EMATCH_NBYTE) += em_nbyte.o
net/sched/cls_cgroup.c +189 -3
238 lines not on this page · raw message
net/sched/cls_counter_holder.c new file +145 -0
145 lines not on this page · raw message
Signature · 1 line
1.7.9.5
Quoted 20 lines · Alexey Perevalov
Hello.
It's second version of patch I already sent to netdev.
The main goal of this patch it's counting traffic for process placed to
net_clscgroup (ingress and egress).
It's based onres_countersand holds counter per network interfaces.Description of patch.
It handles packets innet/core/dev.cfor egress and in /net/ipv4/tcp.c|udp.c for ingress.
These places were chosen because we need to know also network interface.Cgroup fs interface provides following files additional to existing
net_clsfiles:
net_cls.ifacename.usage_in_bytes
Containing rcv/snd lines.
Also this patch adds tonet_clsability to handle a network device registration.
Quoted 6 lines
It could be included or excluded in compile time.
I moved the menu entry for "Control group classifier" from network/QoS to General Option/Control Group.I'm waiting for you comments.
Daniel Wagner is working on something a lot similar.
Maybe you should be in contact, in case you are not yet.
A few general comments:
1) res_counters are incredibly expensive. If you are more interested in counting than you are in limiting, they may not be your best choice.
2) When Daniel exposed his use case to me, it gave me the impression that "counting traffic" is something that is totally doable by having a dedicated interface in a separate namespace. Basically, we already count traffic (rx and tx) for all interfaces anyway, so it suggests that it could be an interesting way to see the problem.
AFAIK, Daniel is still measuring this. But it would be great to know if that could work for your use case as well.
Thanks.
A message that is not in the archive (sent off-list or not yet received): [email protected]
Hi Alexey,
Quoted 21 lines · Glauber Costa
Quoted 20 lines · Alexey Perevalov
Hello.
It's second version of patch I already sent to netdev.
The main goal of this patch it's counting traffic for process placed to
net_clscgroup (ingress and egress).
It's based onres_countersand holds counter per network interfaces.Description of patch.
It handles packets innet/core/dev.cfor egress and in /net/ipv4/tcp.c|udp.c for ingress.
These places were chosen because we need to know also network interface.Cgroup fs interface provides following files additional to existing
net_clsfiles:
net_cls.ifacename.usage_in_bytes
Containing rcv/snd lines.
Also this patch adds tonet_clsability to handle a network device registration.
Quoted 8 lines
Quoted 6 lines
It could be included or excluded in compile time.
I moved the menu entry for "Control group classifier" from network/QoS to General Option/Control Group.I'm waiting for you comments.
Daniel Wagner is working on something a lot similar.
Yes, basically what I try to do is explained by this excellent article
https://lwn.net/Articles/523058/
The short version: Per application routing and statistics.
I have two PoC implementation doing this. Both implementation have the same key idea which is to set SO_MARK per application. The routing and statistics would then be done by a bunch iptables rules.
In the first implementation extends net_cls to set SO_MARK:
void sock_update_classid(struct sock *sk, struct task_struct *task)
{
u32 classid;+ u32 mark;
classid = task_cls_classid(task);
if (classid != sk->sk_classid)
sk->sk_classid = classid;+
+ mark = task_cls_mark(task);
+ if (mark != sk->sk_mark)
+ sk->sk_mark = mark;
}
The second implementation is adding a new iptables matcher which matches on LSM contexts. Then you can do something like this:
iptables -t mangle -A OUTPUT -m secmark --secctx unconfined_u:unconfined_r:foo_t:s0-s0:c0.c1023 -j MARK --set-mark 200
Quoted 11 lines
Maybe you should be in contact, in case you are not yet.
A few general comments:
1)res_countersare incredibly expensive. If you are more interested in counting than you are in limiting, they may not be your best choice.2) When Daniel exposed his use case to me, it gave me the impression that "counting traffic" is something that is totally doable by having a dedicated interface in a separate namespace. Basically, we already count traffic (rx and tx) for all interfaces anyway, so it suggests that it could be an interesting way to see the problem.
Moving applications into separate net namespaces is for sure a valid solution. Though there is a one drawback in this approach. The namespaces need to be attached to a bridge and then some NATting. That means every application would get it's own IP address. This might be okay for your certain use cases but I am still trying to work around this. Glauber and I had some discussion about this and he suggested to allow the physical networking device to be attached to several namespaces (e.g. via macvlan). Every namespace would get the same IP address. Unfortunately, this would result in the same mess as several physical devices on a network get the same IP address assigned.
Quoted 2 lines
AFAIK, Daniel is still measuring this. But it would be great to know if that could work for your use case as well.
I have not started to measure :(
cheers,
daniel
A message that is not in the archive (sent off-list or not yet received): [email protected]
Quoted 28 lines · Daniel Wagner
Hi Alexey,
Quoted 25 lines · Glauber Costa
Quoted 24 lines · Alexey Perevalov
Hello.
It's second version of patch I already sent to netdev.
The main goal of this patch it's counting traffic for process placed to
net_clscgroup (ingress and egress).
It's based onres_countersand holds counter per network interfaces.Description of patch.
It handles packets innet/core/dev.cfor egress and in /net/ipv4/tcp.c|udp.c for ingress.
These places were chosen because we need to know also network interface.Cgroup fs interface provides following files additional to existing
net_clsfiles:
net_cls.ifacename.usage_in_bytes
Containing rcv/snd lines.
Also this patch adds tonet_clsability to handle a network device registration.It could be included or excluded in compile time.
I moved the menu entry for "Control group classifier" from network/QoS to General Option/Control Group.
Quoted 6 lines
Quoted 3 lines
Quoted 2 lines
I'm waiting for you comments.
Daniel Wagner is working on something a lot similar.
Yes, basically what I try to do is explained by this excellent article
I read articles and agreed with aspects.
But problem of selecting preferred network for application can be solved using netprio cgroup.
Quoted 22 lines
The short version: Per application routing and statistics.
I have two PoC implementation doing this. Both implementation have the same key idea which is to set
SO_MARKper application. The routing and statistics would then be done by a bunch iptables rules.In the first implementation extends
net_clsto setSO_MARK:void
sock_update_classid(struct sock *sk, struct task_struct *task){ u32 classid;(quoted diff) +5
-0+ u32 mark; classid = task_cls_classid(task); if (classid != sk->sk_classid) sk->sk_classid = classid; + + mark = task_cls_mark(task); + if (mark != sk->sk_mark) + sk->sk_mark = mark; }
Quoted 4 lines
The second implementation is adding a new iptables matcher which matches on LSM contexts. Then you can do something like this:
iptables -t mangle -A OUTPUT -m secmark --secctx unconfined_u:unconfined_r:foo_t:s0-s0:c0.c1023 -j MARK --set-mark 200
As I understand in LSM context it works for egress and ingress.
Quoted 5 lines
Quoted 5 lines
Maybe you should be in contact, in case you are not yet.
A few general comments:
1)res_countersare incredibly expensive. If you are more interested in counting than you are in limiting, they may not be your best choice.
You right, I have a plan now to limit traffic too here. Or as a variant in QoS in this case atomic is better here.
Quoted 5 lines
Quoted 5 lines
2) When Daniel exposed his use case to me, it gave me the impression that "counting traffic" is something that is totally doable by having a dedicated interface in a separate namespace. Basically, we already count traffic (rx and tx) for all interfaces anyway, so it suggests that it could be an interesting way to see the problem.
Quoted 10 lines
Moving applications into separate net namespaces is for sure a valid solution. Though there is a one drawback in this approach. The namespaces need to be attached to a bridge and then some NATting. That means every application would get it's own IP address. This might be okay for your certain use cases but I am still trying to work around this. Glauber and I had some discussion about this and he suggested to allow the physical networking device to be attached to several namespaces (e.g. via macvlan). Every namespace would get the same IP address. Unfortunately, this would result in the same mess as several physical devices on a network get the same IP address assigned.
Is I truly understand what to make statistics works we need to put process to separate namespace? Approach to keep counter in cgroup hasn't such side effects, but it has another ).
Quoted 10 lines
Quoted 4 lines
AFAIK, Daniel is still measuring this. But it would be great to know if that could work for your use case as well.
I have not started to measure :(
cheers,
daniel
Thank you Daniel and Glauber!
Signature · 2 lines
BR Alexey
A message that is not in the archive (sent off-list or not yet received): [email protected]
Hi Alexey,
Quoted 7 lines · Alexey Perevalov
Quoted 4 lines
Quoted 1 line
Daniel Wagner is working on something a lot similar.
Yes, basically what I try to do is explained by this excellent article
I read articles and agreed with aspects.
But problem of selecting preferred network for application can be solved using netprio cgroup.
Choosing the which network to connect to is job of a connection manager. I don't see how a cgroup controller can help you there. I guess I do not understand your statement. Can you rephrase please?
Quoted 6 lines
Quoted 5 lines
The second implementation is adding a new iptables matcher which matches on LSM contexts. Then you can do something like this:
iptables -t mangle -A OUTPUT -m secmark --secctx unconfined_u:unconfined_r:foo_t:s0-s0:c0.c1023 -j MARK --set-mark 200
As I understand in LSM context it works for egress and ingress.
Yes, I am using CONNMARK in conjunction with the the above LSM context matcher. I am still playing around, but it looks quite promising.
Quoted 18 lines
Quoted 5 lines
Quoted 5 lines
2) When Daniel exposed his use case to me, it gave me the impression that "counting traffic" is something that is totally doable by having a dedicated interface in a separate namespace. Basically, we already count traffic (rx and tx) for all interfaces anyway, so it suggests that it could be an interesting way to see the problem.
Quoted 13 lines
Moving applications into separate net namespaces is for sure a valid solution.
Though there is a one drawback in this approach. The namespaces need to be
attached to a bridge and then some NATting. That means every application would get it's own IP address. This might be okay for your certain use cases but I am still trying to work around this. Glauber and I had some discussion about this and he suggested to allow the physical networking device to be attached to several namespaces (e.g. via macvlan). Every namespace would get the same IP address. Unfortunately, this would result in
the same mess as several physical devices on a network get the same IP address assigned.
Quoted 2 lines
Is I truly understand what to make statistics works we need to put process to separate namespace?
If a process lives in its own network namespace then you can count the packets/bytes on the network interface level. The side effect is that is that each namespace is obviously a new network and has to be treated as such.
Quoted 2 lines
Approach to keep counter in cgroup hasn't such side effects, but it has another ).
cgroups are not for free. Currently a lot of effort is put into getting a reasonable performance and behavior into cgroups. In this situation any new feature added to cgroups will need a pretty good justification why it is needed and why it cant be done with existing infrastructure.
Here is some background information on the state of cgroups:
http://thread.gmane.org/gmane.linux.kernel.containers/23698
cheers,
daniel
A message that is not in the archive (sent off-list or not yet received): [email protected]
Quoted 6 lines · Daniel Wagner
Hi Alexey,
Quoted 3 lines · Alexey Perevalov
Quoted 3 lines
Quoted 1 line
Daniel Wagner is working on something a lot similar.
Yes, basically what I try to do is explained by this excellent article
Quoted 7 lines
Quoted 4 lines
Quoted 1 line
I read articles and agreed with aspects.
But problem of selecting preferred network for application can be solved using netprio cgroup.Choosing the which network to connect to is job of a connection manager. I don't see how a cgroup controller can help you there. I guess I do not understand your statement. Can you rephrase please?
I meant choosing preferred network interface for application's traffic. I can be done by metric in routing table or (I wrote) by netprio cgroup.
Quoted 35 lines
Quoted 7 lines
Quoted 6 lines
The second implementation is adding a new iptables matcher which matches on LSM contexts. Then you can do something like this:
iptables -t mangle -A OUTPUT -m secmark --secctx unconfined_u:unconfined_r:foo_t:s0-s0:c0.c1023 -j MARK --set-mark 200
As I understand in LSM context it works for egress and ingress.
Yes, I am using CONNMARK in conjunction with the the above LSM context matcher. I am still playing around, but it looks quite promising.
Quoted 18 lines
Quoted 5 lines
Quoted 5 lines
2) When Daniel exposed his use case to me, it gave me the impression that "counting traffic" is something that is totally doable by having a dedicated interface in a separate namespace. Basically, we already count traffic (rx and tx) for all interfaces anyway, so it suggests that it could be an interesting way to see the problem.
Quoted 13 lines
Moving applications into separate net namespaces is for sure a valid solution.
Though there is a one drawback in this approach. The namespaces need to be
attached to a bridge and then some NATting. That means every application would get it's own IP address. This might be okay for your certain use cases but I am still trying to work around this. Glauber and I had some discussion about this and he suggested to allow the physical networking device to be attached to several namespaces (e.g. via macvlan). Every namespace would get the same IP address. Unfortunately, this would result in
the same mess as several physical devices on a network get the same IP address assigned.Quoted 2 lines
Is I truly understand what to make statistics works we need to put process to separate namespace?
If a process lives in its own network namespace then you can count the packets/bytes on the network interface level. The side effect is that is that each namespace is obviously a new network and has to be treated as such.
Quoted 6 lines
Quoted 2 lines
Approach to keep counter in cgroup hasn't such side effects, but it has another ).
cgroups are not for free. Currently a lot of effort is put into getting a reasonable performance and behavior into cgroups. In this situation any new feature added to cgroups will need a pretty good justification why it is needed and why it cant be done with existing infrastructure.
I want to figure out in yours proposed design:
+------------------------------------------------+ |network namespace1: pid1, pid2,... | | | +---------------------------+ | network stack, network iface | | | | nf hooks +------->| physical network | +------------------------------------------------+ | interface | | | | | +------------------------------------------------+ | | |network namespace2: pid1, pid2,... | | | | +------->| | | network stack, network iface | | | | nf hooks | | | +------------------------------------------------+ | | +---------------------------+ ... ^ +------------------------------------------------+ | |network namespace3: pid1, pid2,... | | | | | | network stack, network iface +-----------+ | nf hooks | +------------------------------------------------+
Question, in case of one physical networking device connected to several namespaces,
is it allow to tweak network packet scheduler (qdisc instance) using traffic control tool for one physical network interface? The same question is about netfilter hooks. I have seen the code, it seems to me nf hooks is registering per network stack now.
CGroup framework has an notification mechanism based on eventfd. For example I can just send notification to user space about network activity. Is there such mechanism in standard infrastructure to notify user space apps on activity on monitored application (maybe nf_queue)?
Quoted 4 lines
Here is some background information on the state of cgroups:
Thank you, I have read.
It seems to me my patch has technical defect with inherited groups.
Quoted 4 lines
cheers,
daniel
Signature · 2 lines
BR, Alexey
A message that is not in the archive (sent off-list or not yet received): [email protected]
Hi Alexey
Quoted 70 lines · Alexey Perevalov
Quoted 28 lines · Daniel Wagner
Quoted 21 lines
Quoted 7 lines
Quoted 7 lines
2) When Daniel exposed his use case to me, it gave me the impression that "counting traffic" is something that is totally doable by having a
dedicated interface in a separate namespace. Basically, we already count
traffic (rx and tx) for all interfaces anyway, so it suggests that it could be an interesting way to see the problem.Quoted 14 lines
Moving applications into separate net namespaces is for sure a valid solution.
Though there is a one drawback in this approach. The namespaces need to be
attached to a bridge and then some NATting. That means every application
would get it's own IP address. This might be okay for your certain use cases but I am still trying to work around this. Glauber and I had some discussion about this and he suggested to allow the physical networking device to be attached to several namespaces (e.g. via macvlan). Every namespace would get the same IP address. Unfortunately, this would result in
the same mess as several physical devices on a network get the same IP address assigned.Quoted 2 lines
Is I truly understand what to make statistics works we need to put process to separate namespace?
If a process lives in its own network namespace then you can count the packets/bytes on the network interface level. The side effect is that is that each namespace is obviously a new network and has to be treated as such.
Quoted 6 lines
Quoted 2 lines
Approach to keep counter in cgroup hasn't such side effects, but it has another ).
cgroups are not for free. Currently a lot of effort is put into getting a reasonable performance and behavior into cgroups. In this situation any new feature added to cgroups will need a pretty good justification why it is needed and why it cant be done with existing infrastructure.
I want to figure out in yours proposed design:
+------------------------------------------------+ |network namespace1: pid1, pid2,... | | | +---------------------------+ | network stack, network iface | | | | nf hooks +------->| physical network | +------------------------------------------------+ | interface | | | | | +------------------------------------------------+ | | |network namespace2: pid1, pid2,... | | | | +------->| | | network stack, network iface | | | | nf hooks | | | +------------------------------------------------+ | | +---------------------------+ ... ^ +------------------------------------------------+ | |network namespace3: pid1, pid2,... | | | | | | network stack, network iface +-----------+ | nf hooks | +------------------------------------------------+
Quoted 2 lines
Question, in case of one physical networking device connected to several namespaces,
Currently, a physical device can only live in one namespace. The idea was to see if that 'limitation' could be removed, e.g. by modifying macvlan.
Quoted 4 lines
is it allow to tweak network packet scheduler (qdisc instance) using traffic control tool for one physical network interface? The same question is about netfilter hooks. I have seen the code, it seems to me nf hooks is registering per network stack now.
As I said this was just and idea and maybe it is not possible.
Quoted 4 lines
CGroup framework has an notification mechanism based on eventfd. For example I can just send notification to user space about network activity. Is there such mechanism in standard infrastructure to notify user space apps on activity on monitored application (maybe
nf_queue)?
My current plan is to use IDLETIMER for this stuff.
cheers,
daniel