IPv6 ifstats separation
Subsystems (3)
PATCH v2 1/1 IPv6 ifstats separation
Separate IPv6 ifstats into the ones that are hit on fast path and the ones that aren't. The ones that are not can be removed as needed using sysctls.
The separation minimizes percpu data. It reduces from 288 bytes per cpu to 56 bytes. When the other stats are disabled, then it adds 4144 bytes saving per netdev.
The stats that are hit on fast path are percpu and the existing macro that increments them is added with a constant check that should be optimized by the preprocessor.
This is done based on some feedbacks in the past to make IPv6 stats optional. It doesn't fully accomplish that but it enables all but 6 counters be optional. Those optional counters however are still enabled by default to preserve the current behavior.
Changes from v1:
- More elaborate changelog (per Eric Dumazet)
--- include/linux/ipv6.h | 3 + include/net/if_inet6.h | 3 +- include/net/ipv6.h | 28 ++- include/net/snmp.h | 22 +++ include/uapi/linux/ipv6.h | 3 + include/uapi/linux/snmp.h | 3 +- net/ipv6/addrconf.c | 380 +++++++++++++++++++++++++++++++++++--- net/ipv6/addrconf_core.c | 3 +- net/ipv6/proc.c | 57 +++++- 9 files changed, 462 insertions(+), 40 deletions(-)
include/linux/ipv6.h +3 -0
@@ -74,6 +74,9 @@ struct ipv6_devconf {__u32 addr_gen_mode;__s32 disable_policy;__s32 ndisc_tclass; +__s32 extended_ipstats; +__s32 icmpstats; +__s32 icmpmsgstats;struct ctl_table_header *sysctl_header; };
include/net/if_inet6.h +2 -1
@@ -158,7 +158,8 @@ struct ifacaddr6 {struct ipv6_devstat {struct proc_dir_entry *proc_dir_entry; - DEFINE_SNMP_STAT(struct ipstats_mib, ipv6); + DEFINE_SNMP_STAT(struct ipstats_mib_device_fast, ipv6dev_fast); + DEFINE_SNMP_STAT_ATOMIC(struct ipstats_mib_device, ipv6dev); DEFINE_SNMP_STAT_ATOMIC(struct icmpv6_mib_device, icmpv6dev); DEFINE_SNMP_STAT_ATOMIC(struct icmpv6msg_mib_device, icmpv6msgdev); };
include/net/ipv6.h +20 -8
@@ -166,8 +166,12 @@ extern int sysctl_mld_qrv;#define _DEVINC (net, statname, mod, idev, field) \ ({ \ struct inet6_dev *_idev = (idev); \ - if (likely(_idev != NULL)) \ - mod##SNMP_INC_STATS64((_idev)->stats.statname, (field));\ + if (likely(_idev != NULL)) { \ + if (field < __IPSTATS_MIB_FAST_MAX) \ + mod##SNMP_INC_STATS64((_idev)->stats.statname##dev_fast, (field)); \ + else if (likely((_idev)->stats.statname##dev != NULL)) \ + SNMP_INC_STATS_ATOMIC_LONG((_idev)->stats.statname##dev, (field)); \ + } \ mod##SNMP_INC_STATS64((net)->mib.statname##_statistics, (field));\ })
@@ -175,7 +179,7 @@ extern int sysctl_mld_qrv;#define _DEVINCATOMIC (net, statname, mod, idev, field) \ ({ \ struct inet6_dev *_idev = (idev); \ - if (likely(_idev != NULL)) \ + if (likely(_idev != NULL && (_idev)->stats.statname##dev != NULL)) \ SNMP_INC_STATS_ATOMIC_LONG((_idev)->stats.statname##dev, (field)); \ mod##SNMP_INC_STATS((net)->mib.statname##_statistics, (field));\ })
@@ -184,7 +188,7 @@ extern int sysctl_mld_qrv;#define _DEVINC_ATOMIC_ATOMIC (net, statname, idev, field) \ ({ \ struct inet6_dev *_idev = (idev); \ - if (likely(_idev != NULL)) \ + if (likely(_idev != NULL && (_idev)->stats.statname##dev != NULL)) \ SNMP_INC_STATS_ATOMIC_LONG((_idev)->stats.statname##dev, (field)); \ SNMP_INC_STATS_ATOMIC_LONG((net)->mib.statname##_statistics, (field));\ })
@@ -192,16 +196,24 @@ extern int sysctl_mld_qrv;#define _DEVADD (net, statname, mod, idev, field, val) \ ({ \ struct inet6_dev *_idev = (idev); \ - if (likely(_idev != NULL)) \ - mod##SNMP_ADD_STATS((_idev)->stats.statname, (field), (val)); \ + if (likely(_idev != NULL)) { \ + if (field < __IPSTATS_MIB_FAST_MAX) \ + mod##SNMP_ADD_STATS((_idev)->stats.statname##dev_fast, (field), (val)); \ + else if (likely((_idev)->stats.statname##dev != NULL)) \ + SNMP_ADD_STATS_ATOMIC_LONG((_idev)->stats.statname##dev, (field), (val)); \ + } \ mod##SNMP_ADD_STATS((net)->mib.statname##_statistics, (field), (val));\ })#define _DEVUPD (net, statname, mod, idev, field, val) \ ({ \ struct inet6_dev *_idev = (idev); \ - if (likely(_idev != NULL)) \ - mod##SNMP_UPD_PO_STATS((_idev)->stats.statname, field, (val)); \ + if (likely(_idev != NULL)) { \ + if (field##PKTS < __IPSTATS_MIB_FAST_MAX) \ + mod##SNMP_UPD_PO_STATS((_idev)->stats.statname##dev_fast, field, (val)); \ + else if (likely((_idev)->stats.statname##dev != NULL)) \ + SNMP_UPD_PO_STATS_ATOMIC_LONG((_idev)->stats.statname##dev, field, (val)); \ + } \ mod##SNMP_UPD_PO_STATS((net)->mib.statname##_statistics, field, (val));\ })
include/net/snmp.h +22 -0
@@ -53,12 +53,25 @@ struct snmp_mib {/* IPstats */ #define IPSTATS_MIB_MAX __IPSTATS_MIB_MAX +#define IPSTATS_MIB_FAST_MAX __IPSTATS_MIB_FAST_MAXstruct ipstats_mib {/* mibs[] must be first field of struct ipstats_mib */ u64 mibs[IPSTATS_MIB_MAX ];struct u64_stats_sync syncp; }; +/* Fast per device IPstats */ +struct ipstats_mib_device_fast { +/* mibs[] must be first field of struct ipstats_mib_device_fast */ +u64 mibs[IPSTATS_MIB_FAST_MAX ]; +struct u64_stats_sync syncp; +}; + +/* Slow per device IPstats */ +struct ipstats_mib_device { +atomic_long_t mibs[IPSTATS_MIB_MAX ]; +}; +/* ICMP */ #define ICMP_MIB_MAX __ICMP_MIB_MAXstruct icmp_mib {
@@ -140,6 +153,10 @@ struct linux_xfrm_mib {#define SNMP_ADD_STATS (mib, field, addend) \ this_cpu_add(mib->mibs[field], addend) + +#define SNMP_ADD_STATS_ATOMIC_LONG (mib, field, addend) \ + atomic_long_add(addend, &mib->mibs[field]) +#define SNMP_UPD_PO_STATS (mib, basefield, addend) \ do { \ __typeof__((mib->mibs) + 0) ptr = mib->mibs; \
@@ -152,6 +169,11 @@ struct linux_xfrm_mib { __this_cpu_inc(ptr[basefield##PKTS]); \ __this_cpu_add(ptr[basefield##OCTETS], addend); \ } while (0) +#define SNMP_UPD_PO_STATS_ATOMIC_LONG (mib, basefield, addend) \ + do { \ + atomic_long_inc(&mib->mibs[basefield##PKTS]); \ + atomic_long_add(addend, &mib->mibs[basefield##OCTETS]); \ + } while (0)#if BITS_PER_LONG ==32
include/uapi/linux/ipv6.h +3 -0
@@ -187,6 +187,9 @@ enum {DEVCONF_DISABLE_POLICY ,DEVCONF_ACCEPT_RA_RT_INFO_MIN_PLEN ,DEVCONF_NDISC_TCLASS , +DEVCONF_EXTENDED_IPSTATS , +DEVCONF_ICMPSTATS , +DEVCONF_ICMPMSGSTATS , DEVCONF_MAX };
include/uapi/linux/snmp.h +2 -1
@@ -26,8 +26,9 @@ enumIPSTATS_MIB_OUTFORWDATAGRAMS ,/* OutForwDatagrams */ IPSTATS_MIB_OUTPKTS ,/* OutRequests */ IPSTATS_MIB_OUTOCTETS ,/* OutOctets */ + __IPSTATS_MIB_FAST_MAX,/* other fields */ -IPSTATS_MIB_INHDRERRORS ,/* InHdrErrors */ +IPSTATS_MIB_INHDRERRORS = __IPSTATS_MIB_FAST_MAX,/* InHdrErrors */ IPSTATS_MIB_INTOOBIGERRORS ,/* InTooBigErrors */ IPSTATS_MIB_INNOROUTES ,/* InNoRoutes */ IPSTATS_MIB_INADDRERRORS, /* InAddrErrors */
net/ipv6/addrconf.c +356 -24
454 lines not on this page · raw message
net/ipv6/addrconf_core.c +2 -1
@@ -198,7 +198,8 @@ static void snmp6_free_dev(struct inet6_dev *idev) {kfree (idev->stats.icmpv6msgdev);kfree (idev->stats.icmpv6dev); -free_percpu (idev->stats.ipv6); +kfree (idev->stats.ipv6dev); +free_percpu (idev->stats.ipv6dev_fast); }static void in6_dev_finish_destroy_rcu (struct rcu_head *head)
net/ipv6/proc.c +52 -5
69 lines not on this page · raw message
Signature · 1 line
2.17.1
Quoted 8 lines · Stephen Suryaputra
Separate IPv6 ifstats into the ones that are hit on fast path and the ones that aren't. The ones that are not can be removed as needed using sysctls.
The separation minimizes percpu data. It reduces from 288 bytes per cpu to 56 bytes. When the other stats are disabled, then it adds 4144 bytes saving per netdev.
Quoted 9 lines
The stats that are hit on fast path are percpu and the existing macro that increments them is added with a constant check that should be optimized by the preprocessor.
This is done based on some feedbacks in the past to make IPv6 stats optional. It doesn't fully accomplish that but it enables all but 6 counters be optional. Those optional counters however are still enabled by default to preserve the current behavior.
How have you decided some counters can be 'slow' and other 'fast' ?
I can tell you I see many ultra-fast candidates in your 'slow' list :/
Also think about DDOS.
After your patch, all these 'wrong packets' will incur an expensive operation on a shared and highly contented cache line, effectively making the attack easier to conduct.
+static const struct snmp_mib snmp6_ipstats_device_list[] = {
+ SNMP_MIB_ITEM("Ip6InHdrErrors", IPSTATS_MIB_INHDRERRORS),
+ SNMP_MIB_ITEM("Ip6InTooBigErrors", IPSTATS_MIB_INTOOBIGERRORS),
+ SNMP_MIB_ITEM("Ip6InNoRoutes", IPSTATS_MIB_INNOROUTES),
+ SNMP_MIB_ITEM("Ip6InAddrErrors", IPSTATS_MIB_INADDRERRORS),
+ SNMP_MIB_ITEM("Ip6InUnknownProtos", IPSTATS_MIB_INUNKNOWNPROTOS),
+ SNMP_MIB_ITEM("Ip6InTruncatedPkts", IPSTATS_MIB_INTRUNCATEDPKTS),
+ SNMP_MIB_ITEM("Ip6InDiscards", IPSTATS_MIB_INDISCARDS),
+ SNMP_MIB_ITEM("Ip6OutDiscards", IPSTATS_MIB_OUTDISCARDS),
+ SNMP_MIB_ITEM("Ip6OutNoRoutes", IPSTATS_MIB_OUTNOROUTES),
+ SNMP_MIB_ITEM("Ip6ReasmTimeout", IPSTATS_MIB_REASMTIMEOUT),
+ SNMP_MIB_ITEM("Ip6ReasmReqds", IPSTATS_MIB_REASMREQDS),
+ SNMP_MIB_ITEM("Ip6ReasmOKs", IPSTATS_MIB_REASMOKS),
+ SNMP_MIB_ITEM("Ip6ReasmFails", IPSTATS_MIB_REASMFAILS),
+ SNMP_MIB_ITEM("Ip6FragOKs", IPSTATS_MIB_FRAGOKS),
+ SNMP_MIB_ITEM("Ip6FragFails", IPSTATS_MIB_FRAGFAILS),
+ SNMP_MIB_ITEM("Ip6FragCreates", IPSTATS_MIB_FRAGCREATES),
+ SNMP_MIB_ITEM("Ip6InMcastPkts", IPSTATS_MIB_INMCASTPKTS),
+ SNMP_MIB_ITEM("Ip6OutMcastPkts", IPSTATS_MIB_OUTMCASTPKTS),
+ SNMP_MIB_ITEM("Ip6InMcastOctets", IPSTATS_MIB_INMCASTOCTETS),
+ SNMP_MIB_ITEM("Ip6OutMcastOctets", IPSTATS_MIB_OUTMCASTOCTETS),
+ SNMP_MIB_ITEM("Ip6InBcastOctets", IPSTATS_MIB_INBCASTOCTETS),
+ SNMP_MIB_ITEM("Ip6OutBcastOctets", IPSTATS_MIB_OUTBCASTOCTETS),
+ /* IPSTATS_MIB_CSUMERRORS is not relevant in IPv6 (no checksum) */
+ SNMP_MIB_ITEM("Ip6InNoECTPkts", IPSTATS_MIB_NOECTPKTS),
+ SNMP_MIB_ITEM("Ip6InECT1Pkts", IPSTATS_MIB_ECT1PKTS),
+ SNMP_MIB_ITEM("Ip6InECT0Pkts", IPSTATS_MIB_ECT0PKTS),
+ SNMP_MIB_ITEM("Ip6InCEPkts", IPSTATS_MIB_CEPKTS),
+ SNMP_MIB_SENTINEL
+};
+
Quoted 4 lines · Eric Dumazet
How have you decided some counters can be 'slow' and other 'fast' ?
I can tell you I see many ultra-fast candidates in your 'slow' list :/
Based on what others have categorized based on what's in the code and IMHO they make sense:
enum
{
IPSTATS_MIB_NUM = 0,
/* frequently written fields in fast path, kept in same cache line */ IPSTATS_MIB_INPKTS, /* InReceives */ IPSTATS_MIB_INOCTETS, /* InOctets */ IPSTATS_MIB_INDELIVERS, /* InDelivers */ IPSTATS_MIB_OUTFORWDATAGRAMS, /* OutForwDatagrams */ IPSTATS_MIB_OUTPKTS, /* OutRequests */ IPSTATS_MIB_OUTOCTETS, /* OutOctets */
/* other fields */
IPSTATS_MIB_INHDRERRORS, /* InHdrErrors */
...
__IPSTATS_MIB_MAX
};
Quoted 7 lines
Also think about DDOS.
After your patch, all these 'wrong packets' will incur an expensive operation on a shared and highly contented cache line, effectively making the attack easier to conduct.
I agree about it is becoming more expensive to hit the slow counters due to the check whether they are enabled or not.
Do you think it should just be enabled always? Then the cost to hit the slow counters is the same as what it is right now.
Stephen.
Quoted 8 lines · Stephen Suryaputra
Quoted 4 lines · Eric Dumazet
How have you decided some counters can be 'slow' and other 'fast' ?
I can tell you I see many ultra-fast candidates in your 'slow' list :/
Based on what others have categorized based on what's in the code and IMHO they make sense:
Well, you better test, because you missed a few counters that are hit hard in the fast path for normal (non DDOS) packets.
Quoted 20 lines
enum
{
IPSTATS_MIB_NUM= 0,
/* frequently written fields in fast path, kept in same cache line */IPSTATS_MIB_INPKTS, /* InReceives */IPSTATS_MIB_INOCTETS, /* InOctets */IPSTATS_MIB_INDELIVERS, /* InDelivers */IPSTATS_MIB_OUTFORWDATAGRAMS, /* OutForwDatagrams */IPSTATS_MIB_OUTPKTS, /* OutRequests */IPSTATS_MIB_OUTOCTETS, /* OutOctets */
/* other fields */
IPSTATS_MIB_INHDRERRORS, /* InHdrErrors */
...
__IPSTATS_MIB_MAX
};Quoted 3 lines
Also think about DDOS.
Quoted 7 lines
Quoted 4 lines
After your patch, all these 'wrong packets' will incur an expensive operation on a shared and highly contented cache line, effectively making the attack easier to conduct.
I agree about it is becoming more expensive to hit the slow counters due to the check whether they are enabled or not.
What do you mean ?
The real cost is having dozens of cpus updating the same cache lines if the SNMP counter is an atomic instead of per-cpu counters.
Make sure to test this on a configuration with 16 (or more) RX queues, and cpus handling NIC IRQS spread on multiple NUMA nodes.