ICMP flow improvements
Subsystems (5)
v2 cover ICMP flow improvements
This series improves the flow inspector handling of ICMP packets:
The first two patches just add some comments in the code which would have saved me a few minutes of time, and refactor a piece of code.
The third one adds to the flow inspector the capability to extract the Identifier field, if present, so echo requests and replies are classified as part of the same flow.
The fourth patch uses the function introduced earlier to the bonding driver, so echo replies can be balanced across bonding slaves.
v1 -> v2:
- remove unused struct members
- add an helper to check for the Id field
- use a local flow_dissector_key in the bonding to avoid changing behaviour of the flow dissector
Matteo Croce (4):
flow_dissector: add meaningful comments
flow_dissector: skip the ICMP dissector for non ICMP packets
flow_dissector: extract more ICMP information
bonding: balance ICMP echoes in layer3+4 mode
drivers/net/bonding/bond_main.c | 77 ++++++++++++++++++++--- include/net/flow_dissector.h | 20 +++--- net/core/flow_dissector.c | 108 +++++++++++++++++++++++--------- 3 files changed, 160 insertions(+), 45 deletions(-)
Signature · 1 line
2.21.0
PATCH v2 1/4 flow_dissector: add meaningful comments
Documents two piece of code which can't be understood at a glance.
--- include/net/flow_dissector.h | 1 + net/core/flow_dissector.c | 6 ++++++ 2 files changed, 7 insertions(+)
include/net/flow_dissector.h +1 -0
@@ -282,6 +282,7 @@ struct flow_keys {struct flow_dissector_key_vlan cvlan;struct flow_dissector_key_keyid keyid;struct flow_dissector_key_ports ports; +/* 'addrs' must be the last member */ struct flow_dissector_key_addrs addrs; };
net/core/flow_dissector.c +6 -0
@@ -1374,6 +1374,9 @@ static inline size_t flow_keys_hash_length(const struct flow_keys *flow) {size_t diff =FLOW_KEYS_HASH_OFFSET +sizeof (flow->addrs);BUILD_BUG_ON ((sizeof (*flow) -FLOW_KEYS_HASH_OFFSET ) %sizeof (u32)); +/* flow.addrs MUST be the last member in struct flow_keys because +* different L3 protocols have different address length +*/ BUILD_BUG_ON(offsetof(typeof(*flow), addrs) != sizeof(*flow) - sizeof(flow->addrs));
@@ -1421,6 +1424,9 @@ __be32 flow_get_u32_dst(const struct flow_keys *flow) }EXPORT_SYMBOL (flow_get_u32_dst); +/* Sort the source and destination IP (and the ports if the IP are the same), +* to have consistent hash within the two directions +*/ static inline void __flow_hash_consistentify (struct flow_keys *keys) {int addr_diff, i;
Signature · 1 line
2.21.0
PATCH v2 2/4 flow_dissector: skip the ICMP dissector for non ICMP packets
In reply to Matteo CroceFLOW_DISSECTOR_KEY_ICMP is checked for every packet, not only ICMP ones. Even if the test overhead is probably negligible, move the ICMP dissector code under the big 'switch(ip_proto)' so it gets called only for ICMP packets.
--- net/core/flow_dissector.c | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-)
net/core/flow_dissector.c +25 -9
@@ -203,6 +203,25 @@ __be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto, }EXPORT_SYMBOL (__skb_flow_get_ports); +/* If FLOW_DISSECTOR_KEY_ICMP is set, get the Type and Code from an ICMP packet +* using skb_flow_get_be16(). +*/ +static void __skb_flow_dissect_icmp (const struct sk_buff *skb, +struct flow_dissector *flow_dissector, +void *target_container, +void *data,int thoff,int hlen) +{ +struct flow_dissector_key_icmp *key_icmp; + +if (!dissector_uses_key (flow_dissector,FLOW_DISSECTOR_KEY_ICMP )) +return ; + + key_icmp =skb_flow_dissector_target (flow_dissector, +FLOW_DISSECTOR_KEY_ICMP , + target_container); + key_icmp->icmp =skb_flow_get_be16 (skb, thoff, data, hlen); +} +void skb_flow_dissect_meta (const struct sk_buff *skb,struct flow_dissector *flow_dissector,void *target_container)
@@ -853,7 +872,6 @@ bool __skb_flow_dissect(const struct net *net,struct flow_dissector_key_basic *key_basic;struct flow_dissector_key_addrs *key_addrs;struct flow_dissector_key_ports *key_ports; -struct flow_dissector_key_icmp *key_icmp;struct flow_dissector_key_tags *key_tags;struct flow_dissector_key_vlan *key_vlan;struct bpf_prog *attached =NULL ;
@@ -1295,6 +1313,12 @@ bool __skb_flow_dissect(const struct net *net, data, nhoff, hlen);break ; +case IPPROTO_ICMP : +case IPPROTO_ICMPV6 : +__skb_flow_dissect_icmp (skb, flow_dissector, target_container, + data, nhoff, hlen); +break ; +default :break ; }
@@ -1308,14 +1332,6 @@ bool __skb_flow_dissect(const struct net *net, data, hlen); } -if (dissector_uses_key (flow_dissector, -FLOW_DISSECTOR_KEY_ICMP )) { - key_icmp =skb_flow_dissector_target (flow_dissector, -FLOW_DISSECTOR_KEY_ICMP , - target_container); - key_icmp->icmp =skb_flow_get_be16 (skb, nhoff, data, hlen); - } -/* Process result of IP proto processing */ switch (fdret) {case FLOW_DISSECT_RET_PROTO_AGAIN :
Signature · 1 line
2.21.0
PATCH v2 3/4 flow_dissector: extract more ICMP information
In reply to Matteo CroceThe ICMP flow dissector currently parses only the Type and Code fields. Some ICMP packets (echo, timestamp) have a 16 bit Identifier field which is used to correlate packets.
Add such field in flow_dissector_key_icmp and replace skb_flow_get_be16() with a more complex function which populate this field.
--- include/net/flow_dissector.h | 19 +++++---- net/core/flow_dissector.c | 74 ++++++++++++++++++++++++------------ 2 files changed, 61 insertions(+), 32 deletions(-)
include/net/flow_dissector.h +11 -8
@@ -6,6 +6,8 @@#include <linux/in6.h> #include <uapi/linux/if_ether.h> +struct sk_buff ; +/** * struct flow_dissector_key_control: * @thoff: Transport header offset
@@ -156,19 +158,16 @@ struct flow_dissector_key_ports {/** * flow_dissector_key_icmp: -* @ports: type and code of ICMP header -* icmp: ICMP type (high) and code (low) * type: ICMP type * code: ICMP code +* id: session identifier */ struct flow_dissector_key_icmp { -union { -__be16 icmp; -struct { -u8 type; -u8 code; - }; +struct { +u8 type; +u8 code; }; +u16 id; };/**
@@ -282,6 +281,7 @@ struct flow_keys {struct flow_dissector_key_vlan cvlan;struct flow_dissector_key_keyid keyid;struct flow_dissector_key_ports ports; +struct flow_dissector_key_icmp icmp;/* 'addrs' must be the last member */ struct flow_dissector_key_addrs addrs; };
@@ -316,6 +316,9 @@ static inline bool flow_keys_have_l4(const struct flow_keys *keys) }u32 flow_hash_from_keys (struct flow_keys *keys); +void skb_flow_get_icmp_tci (const struct sk_buff *skb, +struct flow_dissector_key_icmp *key_icmp, +void *data,int thoff,int hlen);static inline bool dissector_uses_key (const struct flow_dissector *flow_dissector,enum flow_dissector_key_id key_id)
net/core/flow_dissector.c +50 -24
92 lines not on this page · raw message
Signature · 1 line
2.21.0
PATCH v2 4/4 bonding: balance ICMP echoes in layer3+4 mode
In reply to Matteo CroceThe bonding uses the L4 ports to balance flows between slaves. As the ICMP protocol has no ports, those packets are sent all to the same device:
# tcpdump -qltnni veth0 ip |sed 's/^/0: /' &
# tcpdump -qltnni veth1 ip |sed 's/^/1: /' &
# ping -qc1 192.168.0.2
1: IP 192.168.0.1 > 192.168.0.2: ICMP echo request, id 315, seq 1, length 64
1: IP 192.168.0.2 > 192.168.0.1: ICMP echo reply, id 315, seq 1, length 64
# ping -qc1 192.168.0.2
1: IP 192.168.0.1 > 192.168.0.2: ICMP echo request, id 316, seq 1, length 64
1: IP 192.168.0.2 > 192.168.0.1: ICMP echo reply, id 316, seq 1, length 64
# ping -qc1 192.168.0.2
1: IP 192.168.0.1 > 192.168.0.2: ICMP echo request, id 317, seq 1, length 64
1: IP 192.168.0.2 > 192.168.0.1: ICMP echo reply, id 317, seq 1, length 64But some ICMP packets have an Identifier field which is
used to match packets within sessions, let's use this value in the hash function to balance these packets between bond slaves:
# ping -qc1 192.168.0.2
0: IP 192.168.0.1 > 192.168.0.2: ICMP echo request, id 303, seq 1, length 64
0: IP 192.168.0.2 > 192.168.0.1: ICMP echo reply, id 303, seq 1, length 64
# ping -qc1 192.168.0.2
1: IP 192.168.0.1 > 192.168.0.2: ICMP echo request, id 304, seq 1, length 64
1: IP 192.168.0.2 > 192.168.0.1: ICMP echo reply, id 304, seq 1, length 64Aso, let's use a flow_dissector_key which defines FLOW_DISSECTOR_KEY_ICMP, so we can balance pings encapsulated in a tunnel when using mode encap3+4:
# ping -q 192.168.1.2 -c1
0: IP 192.168.0.1 > 192.168.0.2: GREv0, length 102: IP 192.168.1.1 > 192.168.1.2: ICMP echo request, id 585, seq 1, length 64
0: IP 192.168.0.2 > 192.168.0.1: GREv0, length 102: IP 192.168.1.2 > 192.168.1.1: ICMP echo reply, id 585, seq 1, length 64
# ping -q 192.168.1.2 -c1
1: IP 192.168.0.1 > 192.168.0.2: GREv0, length 102: IP 192.168.1.1 > 192.168.1.2: ICMP echo request, id 586, seq 1, length 64
1: IP 192.168.0.2 > 192.168.0.1: GREv0, length 102: IP 192.168.1.2 > 192.168.1.1: ICMP echo reply, id 586, seq 1, length 64--- drivers/net/bonding/bond_main.c | 77 ++++++++++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 7 deletions(-)
drivers/net/bonding/bond_main.c +70 -7
@@ -200,6 +200,51 @@ atomic_t netpoll_block_tx = ATOMIC_INIT(0);unsigned int bond_net_id__read_mostly ; +static const struct flow_dissector_key flow_keys_bonding_keys[] = { + { + .key_id =FLOW_DISSECTOR_KEY_CONTROL , + .offset = offsetof(struct flow_keys , control), + }, + { + .key_id =FLOW_DISSECTOR_KEY_BASIC , + .offset = offsetof(struct flow_keys , basic), + }, + { + .key_id =FLOW_DISSECTOR_KEY_IPV4_ADDRS , + .offset = offsetof(struct flow_keys, addrs.v4addrs), + }, + { + .key_id =FLOW_DISSECTOR_KEY_IPV6_ADDRS , + .offset = offsetof(struct flow_keys, addrs.v6addrs), + }, + { + .key_id =FLOW_DISSECTOR_KEY_TIPC , + .offset = offsetof(struct flow_keys, addrs.tipckey), + }, + { + .key_id =FLOW_DISSECTOR_KEY_PORTS , + .offset = offsetof(struct flow_keys , ports), + }, + { + .key_id =FLOW_DISSECTOR_KEY_ICMP , + .offset = offsetof(struct flow_keys , icmp), + }, + { + .key_id =FLOW_DISSECTOR_KEY_VLAN , + .offset = offsetof(struct flow_keys , vlan), + }, + { + .key_id =FLOW_DISSECTOR_KEY_FLOW_LABEL , + .offset = offsetof(struct flow_keys , tags), + }, + { + .key_id =FLOW_DISSECTOR_KEY_GRE_KEYID , + .offset = offsetof(struct flow_keys , keyid), + }, +}; + +static struct flow_dissector flow_keys_bonding__read_mostly ; +/*-------------------------- Forward declarations ---------------------------*/ static int bond_init (struct net_device *bond_dev);
@@ -3263,10 +3308,14 @@ static bool bond_flow_dissect(struct bonding *bond, struct sk_buff *skb,const struct iphdr *iph;int noff, proto =-1 ; -if (bond->params.xmit_policy >BOND_XMIT_POLICY_LAYER23 ) -return skb_flow_dissect_flow_keys (skb, fk,0 ); +if (bond->params.xmit_policy >BOND_XMIT_POLICY_LAYER23 ) { +memset (fk,0 ,sizeof (*fk)); +return __skb_flow_dissect (NULL , skb, &flow_keys_bonding, + fk,NULL ,0 ,0 ,0 ,0 ); + } fk->ports.ports =0 ; +memset (&fk->icmp,0 ,sizeof (fk->icmp)); noff =skb_network_offset (skb);if (skb->protocol ==htons (ETH_P_IP )) {if (unlikely (!pskb_may_pull (skb, noff +sizeof (*iph))))
@@ -3286,8 +3335,14 @@ static bool bond_flow_dissect(struct bonding *bond, struct sk_buff *skb, }else {return false; } -if (bond->params.xmit_policy ==BOND_XMIT_POLICY_LAYER34 && proto >=0 ) - fk->ports.ports =skb_flow_get_ports (skb, noff, proto); +if (bond->params.xmit_policy ==BOND_XMIT_POLICY_LAYER34 && proto >=0 ) { +if (proto ==IPPROTO_ICMP || proto ==IPPROTO_ICMPV6 ) +skb_flow_get_icmp_tci (skb, &fk->icmp, skb->data, +skb_transport_offset (skb), +skb_headlen (skb)); +else + fk->ports.ports =skb_flow_get_ports (skb, noff, proto); + }return true; }
@@ -3314,10 +3369,14 @@ u32 bond_xmit_hash(struct bonding *bond, struct sk_buff *skb)return bond_eth_hash (skb);if (bond->params.xmit_policy ==BOND_XMIT_POLICY_LAYER23 || - bond->params.xmit_policy ==BOND_XMIT_POLICY_ENCAP23 ) + bond->params.xmit_policy ==BOND_XMIT_POLICY_ENCAP23 ) { hash =bond_eth_hash (skb); -else - hash = (__force u32 )flow.ports.ports; + }else { +if (flow.icmp.id) +memcpy (&hash, &flow.icmp,sizeof (hash)); +else +memcpy (&hash, &flow.ports.ports,sizeof (hash)); + } hash ^= (__force u32 )flow_get_u32_dst (&flow) ^ (__force u32 )flow_get_u32_src (&flow); hash ^= (hash >>16 );
@@ -4901,6 +4960,10 @@ static int __init bonding_init(void) goto err; } +skb_flow_dissector_init (&flow_keys_bonding, + flow_keys_bonding_keys, +ARRAY_SIZE (flow_keys_bonding_keys)); +register_netdevice_notifier (&bond_netdev_notifier); out:return res;
Signature · 1 line
2.21.0
Re: [PATCH net-next v2 4/4] bonding: balance ICMP echoes in layer3+4 mode
Quoted 19 lines · Matteo Croce
The bonding uses the L4 ports to balance flows between slaves. As the ICMP protocol has no ports, those packets are sent all to the same device:
# tcpdump -qltnni veth0 ip |sed 's/^/0: /' & # tcpdump -qltnni veth1 ip |sed 's/^/1: /' & # ping -qc1 192.168.0.2 1: IP 192.168.0.1 > 192.168.0.2: ICMP echo request, id 315, seq 1, length 64 1: IP 192.168.0.2 > 192.168.0.1: ICMP echo reply, id 315, seq 1, length 64 # ping -qc1 192.168.0.2 1: IP 192.168.0.1 > 192.168.0.2: ICMP echo request, id 316, seq 1, length 64 1: IP 192.168.0.2 > 192.168.0.1: ICMP echo reply, id 316, seq 1, length 64 # ping -qc1 192.168.0.2 1: IP 192.168.0.1 > 192.168.0.2: ICMP echo request, id 317, seq 1, length 64 1: IP 192.168.0.2 > 192.168.0.1: ICMP echo reply, id 317, seq 1, length 64But some ICMP packets have an Identifier field which is
used to match packets within sessions, let's use this value in the hash function to balance these packets between bond slaves:
Quoted 8 lines
# ping -qc1 192.168.0.2 0: IP 192.168.0.1 > 192.168.0.2: ICMP echo request, id 303, seq 1, length 64 0: IP 192.168.0.2 > 192.168.0.1: ICMP echo reply, id 303, seq 1, length 64 # ping -qc1 192.168.0.2 1: IP 192.168.0.1 > 192.168.0.2: ICMP echo request, id 304, seq 1, length 64 1: IP 192.168.0.2 > 192.168.0.1: ICMP echo reply, id 304, seq 1, length 64Aso, let's use a
flow_dissector_keywhich definesFLOW_DISSECTOR_KEY_ICMP,
Also ?
Quoted 9 lines
so we can balance pings encapsulated in a tunnel when using mode encap3+4:
# ping -q 192.168.1.2 -c1 0: IP 192.168.0.1 > 192.168.0.2: GREv0, length 102: IP 192.168.1.1 > 192.168.1.2: ICMP echo request, id 585, seq 1, length 64 0: IP 192.168.0.2 > 192.168.0.1: GREv0, length 102: IP 192.168.1.2 > 192.168.1.1: ICMP echo reply, id 585, seq 1, length 64 # ping -q 192.168.1.2 -c1 1: IP 192.168.0.1 > 192.168.0.2: GREv0, length 102: IP 192.168.1.1 > 192.168.1.2: ICMP echo request, id 586, seq 1, length 64 1: IP 192.168.0.2 > 192.168.0.1: GREv0, length 102: IP 192.168.1.2 > 192.168.1.1: ICMP echo reply, id 586, seq 1, length 64
Quoted 5 lines
Signed-off-by Matteo Croce <[email protected]>--- drivers/net/bonding/bond_main.c | 77 ++++++++++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 7 deletions(-)
Hi Matteo,
Wouldn't it be more useful and simpler to use some field to choose the slave (override the hash completely) in a deterministic way from user-space ?
For example the mark can be interpreted as a slave id in the bonding (should be optional, to avoid breaking existing setups). ping already supports -m and anything else can set it, this way it can be used to do monitoring for a specific slave with any protocol and would be a much simpler change.
User-space can then implement any logic for the monitoring case and as a minor bonus can monitor the slaves in parallel. And the opposite as well - if people don't want these balanced for some reason, they wouldn't enable it.
Or maybe I've misunderstood why this change is needed. :)
It would actually be nice to include the use-case which brought this on in the commit message.
Cheers,
Nik
Re: [PATCH net-next v2 4/4] bonding: balance ICMP echoes in layer3+4 mode
Quoted 46 lines · Nikolay Aleksandrov
Quoted 19 lines · Matteo Croce
The bonding uses the L4 ports to balance flows between slaves. As the ICMP protocol has no ports, those packets are sent all to the same device:
# tcpdump -qltnni veth0 ip |sed 's/^/0: /' & # tcpdump -qltnni veth1 ip |sed 's/^/1: /' & # ping -qc1 192.168.0.2 1: IP 192.168.0.1 > 192.168.0.2: ICMP echo request, id 315, seq 1, length 64 1: IP 192.168.0.2 > 192.168.0.1: ICMP echo reply, id 315, seq 1, length 64 # ping -qc1 192.168.0.2 1: IP 192.168.0.1 > 192.168.0.2: ICMP echo request, id 316, seq 1, length 64 1: IP 192.168.0.2 > 192.168.0.1: ICMP echo reply, id 316, seq 1, length 64 # ping -qc1 192.168.0.2 1: IP 192.168.0.1 > 192.168.0.2: ICMP echo request, id 317, seq 1, length 64 1: IP 192.168.0.2 > 192.168.0.1: ICMP echo reply, id 317, seq 1, length 64But some ICMP packets have an Identifier field which is
used to match packets within sessions, let's use this value in the hash function to balance these packets between bond slaves:Quoted 8 lines
# ping -qc1 192.168.0.2 0: IP 192.168.0.1 > 192.168.0.2: ICMP echo request, id 303, seq 1, length 64 0: IP 192.168.0.2 > 192.168.0.1: ICMP echo reply, id 303, seq 1, length 64 # ping -qc1 192.168.0.2 1: IP 192.168.0.1 > 192.168.0.2: ICMP echo request, id 304, seq 1, length 64 1: IP 192.168.0.2 > 192.168.0.1: ICMP echo reply, id 304, seq 1, length 64Aso, let's use a
flow_dissector_keywhich definesFLOW_DISSECTOR_KEY_ICMP,Also ?
Quoted 9 lines
so we can balance pings encapsulated in a tunnel when using mode encap3+4:
# ping -q 192.168.1.2 -c1 0: IP 192.168.0.1 > 192.168.0.2: GREv0, length 102: IP 192.168.1.1 > 192.168.1.2: ICMP echo request, id 585, seq 1, length 64 0: IP 192.168.0.2 > 192.168.0.1: GREv0, length 102: IP 192.168.1.2 > 192.168.1.1: ICMP echo reply, id 585, seq 1, length 64 # ping -q 192.168.1.2 -c1 1: IP 192.168.0.1 > 192.168.0.2: GREv0, length 102: IP 192.168.1.1 > 192.168.1.2: ICMP echo request, id 586, seq 1, length 64 1: IP 192.168.0.2 > 192.168.0.1: GREv0, length 102: IP 192.168.1.2 > 192.168.1.1: ICMP echo reply, id 586, seq 1, length 64Quoted 5 lines
Signed-off-by Matteo Croce <[email protected]>--- drivers/net/bonding/bond_main.c | 77 ++++++++++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 7 deletions(-)
Quoted 11 lines
Hi Matteo,
Wouldn't it be more useful and simpler to use some field to choose the slave (override the hash completely) in a deterministic way from user-space ?
For example the mark can be interpreted as a slave id in the bonding (should be optional, to avoid breaking existing setups). ping already supports -m and anything else can set it, this way it can be used to do monitoring for a specific slave with any protocol and would be a much simpler change.
User-space can then implement any logic for the monitoring case and as a minor bonus can monitor the slaves in parallel. And the opposite as well - if people don't want these balanced for some reason, they wouldn't enable it.
Ooh I just noticed you'd like to balance replies as well. Nevermind
Quoted 7 lines
Or maybe I've misunderstood why this change is needed. :)
It would actually be nice to include the use-case which brought this on in the commit message.Cheers,
Nik
Re: [PATCH net-next v2 4/4] bonding: balance ICMP echoes in layer3+4 mode
Quoted 14 lines · Nikolay Aleksandrov
Quoted 11 lines · Nikolay Aleksandrov
Hi Matteo,
Wouldn't it be more useful and simpler to use some field to choose the slave (override the hash completely) in a deterministic way from user-space ?
For example the mark can be interpreted as a slave id in the bonding (should be optional, to avoid breaking existing setups). ping already supports -m and anything else can set it, this way it can be used to do monitoring for a specific slave with any protocol and would be a much simpler change.
User-space can then implement any logic for the monitoring case and as a minor bonus can monitor the slaves in parallel. And the opposite as well - if people don't want these balanced for some reason, they wouldn't enable it.
Quoted 2 lines
Ooh I just noticed you'd like to balance replies as well. Nevermind
Also, the bonding could be in a router in the middle so no way to read the mark.
Signature · 2 lines
Matteo Croce per aspera ad upstream
Nikolay Aleksandrov
Reviewed-byRe: [PATCH net-next v2 4/4] bonding: balance ICMP echoes in layer3+4 mode
Quoted 16 lines · Matteo Croce
Quoted 14 lines · Nikolay Aleksandrov
Quoted 11 lines · Nikolay Aleksandrov
Hi Matteo,
Wouldn't it be more useful and simpler to use some field to choose the slave (override the hash completely) in a deterministic way from user-space ?
For example the mark can be interpreted as a slave id in the bonding (should be optional, to avoid breaking existing setups). ping already supports -m and anything else can set it, this way it can be used to do monitoring for a specific slave with any protocol and would be a much simpler change.
User-space can then implement any logic for the monitoring case and as a minor bonus can monitor the slaves in parallel. And the opposite as well - if people don't want these balanced for some reason, they wouldn't enable it.
Quoted 5 lines
Quoted 2 lines
Ooh I just noticed you'd like to balance replies as well. Nevermind
Also, the bonding could be in a router in the middle so no way to read the mark.
Yeah, of course. I was just thinking from the host monitoring POV as I thought that was the initial intent (reading the last set's discussion).
Anyway the patch looks good to me,
Re: [PATCH net-next v2 4/4] bonding: balance ICMP echoes in layer3+4 mode
In reply to Nikolay AleksandrovQuoted 11 lines · Nikolay Aleksandrov
Hi Matteo,
Wouldn't it be more useful and simpler to use some field to choose the slave (override the hash completely) in a deterministic way from user-space ?
For example the mark can be interpreted as a slave id in the bonding (should be optional, to avoid breaking existing setups). ping already supports -m and anything else can set it, this way it can be used to do monitoring for a specific slave with any protocol and would be a much simpler change.
User-space can then implement any logic for the monitoring case and as a minor bonus can monitor the slaves in parallel. And the opposite as well - if people don't want these balanced for some reason, they wouldn't enable it.
I kind of agree giving user more control. But I do not believe we need to use the mark (this might be already used by other layers)
TCP uses sk->sk_hash to feed skb->hash.
Anything using skb_set_owner_w() is also using sk->sk_hash if set.
So presumably we could add a generic SO_TXHASH socket option to let user space read/set this field.
Re: [PATCH net-next v2 4/4] bonding: balance ICMP echoes in layer3+4 mode
Quoted 19 lines · Eric Dumazet
Quoted 11 lines · Nikolay Aleksandrov
Hi Matteo,
Wouldn't it be more useful and simpler to use some field to choose the slave (override the hash completely) in a deterministic way from user-space ?
For example the mark can be interpreted as a slave id in the bonding (should be optional, to avoid breaking existing setups). ping already supports -m and anything else can set it, this way it can be used to do monitoring for a specific slave with any protocol and would be a much simpler change.
User-space can then implement any logic for the monitoring case and as a minor bonus can monitor the slaves in parallel. And the opposite as well - if people don't want these balanced for some reason, they wouldn't enable it.I kind of agree giving user more control. But I do not believe we need to use the mark (this might be already used by other layers)
Quoted 7 lines
TCP uses
sk->sk_hashto feedskb->hash.Anything using
skb_set_owner_w()is also usingsk->sk_hashif set.So presumably we could add a generic
SO_TXHASHsocket option to let user space read/set this field.
Right, I was just giving it as an example. Your suggestion sounds much better and wouldn't interfere with other layers, plus we already use skb->hash in bond_xmit_hash() and skb_set_owner_w() sets l4_hash if txhash is present which is perfect.
One thing - how do we deal with sk_rethink_txhash() ? I guess we'll need some way to signal that the user specified the txhash and it is not to be recomputed ? That can also be used to avoid the connect txhash set as well if SO_TXHASH was set prior to the connect. It's quite late here, I'll look into it more tomorrow. :)
Thanks,
Nik
Re: [PATCH net-next v2 4/4] bonding: balance ICMP echoes in layer3+4 mode
Quoted 8 lines · Nikolay Aleksandrov
Right, I was just giving it as an example. Your suggestion sounds much better and wouldn't interfere with other layers, plus we already use
skb->hashinbond_xmit_hash()andskb_set_owner_w()setsl4_hashif txhash is present which is perfect.One thing - how do we deal with
sk_rethink_txhash()? I guess we'll need some way to signal that the user specified the txhash and it is not to be recomputed ? That can also be used to avoid the connect txhash set as well ifSO_TXHASHwas set prior to the connect. It's quite late here, I'll look into it more tomorrow. :)
I guess that we have something similar with SO_RCVBUF/SO_SNDBUF
autotuning is disabled when/if they are used :
SOCK_RCVBUF_LOCK & SOCK_SNDBUF_LOCK
We could add a SOCK_TXHASH_LOCK so that sk_rethink_txhash() does nothing if user forced a TXHASH value.
Something like the following (probably not complete) patch.
include/net/sock.h +3 -1
@@ -1354,6 +1354,7 @@ static inline int __sk_prot_rehash(struct sock *sk)#define SOCK_RCVBUF_LOCK 2#define SOCK_BINDADDR_LOCK 4#define SOCK_BINDPORT_LOCK 8 +#define SOCK_TXHASH_LOCK 16struct socket_alloc {struct socket socket;
@@ -1852,7 +1853,8 @@ static inline u32 net_tx_rndhash(void)static inline void sk_set_txhash (struct sock *sk) { - sk->sk_txhash =net_tx_rndhash (); +if (!(sk->sk_userlocks &SOCK_TXHASH_LOCK )) + sk->sk_txhash =net_tx_rndhash (); }static inline void sk_rethink_txhash (struct sock *sk)
include/uapi/asm-generic/socket.h +1 -0
@@ -118,6 +118,7 @@#define SO_SNDTIMEO_NEW 67#define SO_DETACH_REUSEPORT_BPF 68 +#define SO_TXHASH 69#if !defined(__KERNEL__)
net/core/sock.c +8 -0
@@ -770,6 +770,10 @@ int sock_setsockopt(struct socket *sock, int level, int optname,case SO_BROADCAST :sock_valbool_flag (sk,SOCK_BROADCAST , valbool);break ; +case SO_TXHASH : + sk->sk_txhash = val; + sk->sk_userlocks |=SOCK_TXHASH_LOCK ; +break ;case SO_SNDBUF :/* Don't error on this BSD doesn't and if you think * about it this is right. Otherwise apps have to
@@ -1249,6 +1253,10 @@ int sock_getsockopt(struct socket *sock, int level, int optname, v.val =sock_flag (sk,SOCK_BROADCAST );break ; +case SO_TXHASH : + v.val = sk->sk_txhash; +break ; +case SO_SNDBUF : v.val = sk->sk_sndbuf;break ;
Re: [PATCH net-next v2 4/4] bonding: balance ICMP echoes in layer3+4 mode
Quoted 15 lines · Eric Dumazet
Quoted 8 lines · Nikolay Aleksandrov
Right, I was just giving it as an example. Your suggestion sounds much better and wouldn't interfere with other layers, plus we already use
skb->hashinbond_xmit_hash()andskb_set_owner_w()setsl4_hashif txhash is present which is perfect.One thing - how do we deal with
sk_rethink_txhash()? I guess we'll need some way to signal that the user specified the txhash and it is not to be recomputed ? That can also be used to avoid the connect txhash set as well ifSO_TXHASHwas set prior to the connect. It's quite late here, I'll look into it more tomorrow. :)I guess that we have something similar with SO_RCVBUF/SO_SNDBUF
Quoted 9 lines
autotuning is disabled when/if they are used :
SOCK_RCVBUF_LOCK&SOCK_SNDBUF_LOCKWe could add a
SOCK_TXHASH_LOCKso thatsk_rethink_txhash()does nothing if user forced a TXHASH value.Something like the following (probably not complete) patch.
Actually I think it's ok. I had a similar change last night sans the userlocks. I just built and tested a kernel with it successfully using the bonding. The only case that doesn't seem to work is a raw socket without hdrincl, IIUC due to the direct alloc_skb() (transhdrlen == 0) in ip_append_data().
Unless you have other concerns could you please submit it formally ?
Quoted 61 lines
include/net/sock.h +3
-1@@ -1354,6 +1354,7 @@ static inline int __sk_prot_rehash(struct sock *sk)#define SOCK_RCVBUF_LOCK 2#define SOCK_BINDADDR_LOCK 4#define SOCK_BINDPORT_LOCK 8 +#define SOCK_TXHASH_LOCK 16struct socket_alloc {struct socket socket;@@ -1852,7 +1853,8 @@ static inline u32 net_tx_rndhash(void)static inline void sk_set_txhash (struct sock *sk) { - sk->sk_txhash =net_tx_rndhash (); +if (!(sk->sk_userlocks &SOCK_TXHASH_LOCK )) + sk->sk_txhash =net_tx_rndhash (); }static inline void sk_rethink_txhash (struct sock *sk)include/uapi/asm-generic/socket.h +1
-0@@ -118,6 +118,7 @@#define SO_SNDTIMEO_NEW 67#define SO_DETACH_REUSEPORT_BPF 68 +#define SO_TXHASH 69#if !defined(__KERNEL__)net/core/sock.c +8
-0@@ -770,6 +770,10 @@ int sock_setsockopt(struct socket *sock, int level, int optname,case SO_BROADCAST :sock_valbool_flag (sk,SOCK_BROADCAST , valbool);break ; +case SO_TXHASH : + sk->sk_txhash = val; + sk->sk_userlocks |=SOCK_TXHASH_LOCK ; +break ;case SO_SNDBUF :/* Don't error on this BSD doesn't and if you think * about it this is right. Otherwise apps have to @@ -1249,6 +1253,10 @@ int sock_getsockopt(struct socket *sock, int level, int optname, v.val =sock_flag (sk,SOCK_BROADCAST );break ; +case SO_TXHASH : + v.val = sk->sk_txhash; +break ; +case SO_SNDBUF : v.val = sk->sk_sndbuf;break ;
Re: [PATCH net-next v2 4/4] bonding: balance ICMP echoes in layer3+4 mode
In reply to Eric DumazetQuoted 20 lines · Eric Dumazet
Quoted 11 lines · Nikolay Aleksandrov
Hi Matteo,
Wouldn't it be more useful and simpler to use some field to choose the slave (override the hash completely) in a deterministic way from user-space ?
For example the mark can be interpreted as a slave id in the bonding (should be optional, to avoid breaking existing setups). ping already supports -m and anything else can set it, this way it can be used to do monitoring for a specific slave with any protocol and would be a much simpler change.
User-space can then implement any logic for the monitoring case and as a minor bonus can monitor the slaves in parallel. And the opposite as well - if people don't want these balanced for some reason, they wouldn't enable it.I kind of agree giving user more control. But I do not believe we need to use the mark (this might be already used by other layers)
Quoted 7 lines
TCP uses
sk->sk_hashto feedskb->hash.Anything using
skb_set_owner_w()is also usingsk->sk_hashif set.So presumably we could add a generic
SO_TXHASHsocket option to let user space read/set this field.
Hi Eric,
this would work for locally generated echoes, but what about forwarded packets? The point behind my changeset is to provide consistent results within a session by using the same path for request and response,
but avoid all sessions flowing to the same path.
This should resemble what happens with TCP and UDP: different connections, different port, probably a different path. And by doing this in the flow dissector, other applications could benefit it.
Also, this should somewhat balance the traffic of a router forwarding those packets. Maybe it's not so much in percentage, but in some gateways be a considerable volume.
Regards,
Signature · 2 lines
Matteo Croce per aspera ad upstream
Re: [PATCH net-next v2 4/4] bonding: balance ICMP echoes in layer3+4 mode
Quoted 9 lines · Matteo Croce
Hi Eric,
this would work for locally generated echoes, but what about forwarded packets? The point behind my changeset is to provide consistent results within a session by using the same path for request and response,
but avoid all sessions flowing to the same path.
This should resemble what happens with TCP and UDP: different connections, different port, probably a different path. And by doing this in the flow dissector, other applications could benefit it.
In principle it is fine, but I was not sure of overall impact of your change on performance for 99.9% of packets that are not ICMP :)
Re: [PATCH net-next v2 4/4] bonding: balance ICMP echoes in layer3+4 mode
Quoted 15 lines · Eric Dumazet
Quoted 9 lines · Matteo Croce
Hi Eric,
this would work for locally generated echoes, but what about forwarded packets? The point behind my changeset is to provide consistent results within a session by using the same path for request and response,
but avoid all sessions flowing to the same path.
This should resemble what happens with TCP and UDP: different connections, different port, probably a different path. And by doing this in the flow dissector, other applications could benefit it.
Quoted 3 lines
In principle it is fine, but I was not sure of overall impact of your change on performance for 99.9% of packets that are not ICMP :)
Good point. I didn't measure it (I will) but all the code additions are under some if (proto == ICMP) or similar.
My guess is that performance shouldn't change for non ICMP traffic, but I'm curious to test it.
Signature · 2 lines
Matteo Croce per aspera ad upstream
Re: [PATCH net-next v2 4/4] bonding: balance ICMP echoes in layer3+4 mode
Quoted 17 lines · Matteo Croce
Quoted 15 lines · Eric Dumazet
Quoted 9 lines · Matteo Croce
Hi Eric,
this would work for locally generated echoes, but what about forwarded packets? The point behind my changeset is to provide consistent results within a session by using the same path for request and response,
but avoid all sessions flowing to the same path.
This should resemble what happens with TCP and UDP: different connections, different port, probably a different path. And by doing this in the flow dissector, other applications could benefit it.
Quoted 9 lines
Quoted 3 lines
In principle it is fine, but I was not sure of overall impact of your change on performance for 99.9% of packets that are not ICMP :)
Good point. I didn't measure it (I will) but all the code additions are under some if (proto == ICMP) or similar.
My guess is that performance shouldn't change for non ICMP traffic, but I'm curious to test it.
Indeed if there is some impact it's way below the measurement uncertainty. I've bonded two veth pairs and added a tc drop to the peers, then started mausezahn to generate UDP traffic.
Traffic is measured on the veth peers:
Stock 5.4-rc5:
rx: 261.5 Mbps 605.4 Kpps
rx: 261.2 Mbps 604.6 Kpps
rx: 261.6 Mbps 605.5 Kpps
patched:
rx: 261.4 Mbps 605.1 Kpps
rx: 261.1 Mbps 604.4 Kpps
rx: 260.3 Mbps 602.5 Kpps
perf top shows no significatn change in bond* and skb_flow* functions
Regards,
Signature · 2 lines
Matteo Croce per aspera ad upstream
David Miller
maintainerFrom: Matteo Croce <[email protected]>
Date: Tue, 29 Oct 2019 14:50:49 +0100
Quoted 9 lines
This series improves the flow inspector handling of ICMP packets:
The first two patches just add some comments in the code which would have saved me a few minutes of time, and refactor a piece of code.
The third one adds to the flow inspector the capability to extract the Identifier field, if present, so echo requests and replies are classified as part of the same flow.
The fourth patch uses the function introduced earlier to the bonding driver, so echo replies can be balanced across bonding slaves.
Quoted 5 lines
v1 -> v2:
- remove unused struct members
- add an helper to check for the Id field
- use a localflow_dissector_keyin the bonding to avoid changing behaviour of the flow dissector
Series applied to net-next, thanks.