RFS hardware acceleration (v2)
Subsystems (4)
cover RFS hardware acceleration (v2)
This patch series extends RFS to use hardware RX filters where available. Depending on the number of hardware RX queues and their IRQs' affinity, this should reduce the need for IPIs or at least get packets delivered to the right NUMA node.
The first patch implements IRQ affinity notifiers, based on the outline that Thomas Gleixner wrote in response to the previous version of this patch series.
The second patch is a generalisation of the CPU affinity reverse- mapping, plus functions to maintain such a mapping based on the new IRQ affinity notifiers.
The remaining patches add the RFS acceleration hooks and an implementation in the sfc driver. These are largely unchanged from the previous version of this patch series; I haven't yet had the time to do further tuning yet. However I have done some more benchmarking and will send some results in a separate follow-up to this.
Ben.
Ben Hutchings (5):
genirq: Add IRQ affinity notifiers
lib: cpu_rmap: CPU affinity reverse-mapping
net: RPS: Enable hardware acceleration
sfc: Limit filter search depth further for performance hints (i.e.
RFS)
sfc: Implement RFS acceleration
drivers/net/sfc/Kconfig | 4 + drivers/net/sfc/efx.c | 66 ++++++++++-- drivers/net/sfc/efx.h | 9 ++ drivers/net/sfc/filter.c | 113 ++++++++++++++++++- include/linux/cpu_rmap.h | 73 ++++++++++++ include/linux/interrupt.h | 41 +++++++ include/linux/irqdesc.h | 3 + include/linux/netdevice.h | 31 +++++- kernel/irq/manage.c | 81 +++++++++++++ lib/Kconfig | 4 + lib/Makefile | 2 + lib/cpu_rmap.c | 272 +++++++++++++++++++++++++++++++++++++++++++++ net/Kconfig | 1 + net/core/dev.c | 89 ++++++++++++++- 14 files changed, 767 insertions(+), 22 deletions(-) create mode 100644 include/linux/cpu_rmap.h create mode 100644 lib/cpu_rmap.c
Signature · 7 lines
1.7.3.2 -- Ben Hutchings, Senior Software Engineer, Solarflare Communications Not speaking for my employer; that's the marketing department's job. They asked us to note that Solarflare product names are trademarked.
RFC PATCH 1/5 genirq: Add IRQ affinity notifiers
When initiating I/O on a multiqueue and multi-IRQ device, we may want to select a queue for which the response will be handled on the same or a nearby CPU. This requires a reverse-map of IRQ affinity. Add a notification mechanism to support this.
This is based closely on work by Thomas Gleixner <[email protected]>.
--- include/linux/interrupt.h | 41 +++++++++++++++++++++++ include/linux/irqdesc.h | 3 ++ kernel/irq/manage.c | 81 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 0 deletions(-)
include/linux/interrupt.h +41 -0
@@ -14,6 +14,8 @@#include <linux/smp.h> #include <linux/percpu.h> #include <linux/hrtimer.h> +#include <linux/kref.h> +#include <linux/workqueue.h> #include <asm/atomic.h> #include <asm/ptrace.h>
@@ -231,6 +233,28 @@ static inline void resume_device_irqs(void) { };static inline int check_wakeup_irqs (void ) {return 0 ; }#endif +/** +* struct irq_affinity_notify - context for notification of IRQ affinity changes +* @irq: Interrupt to which notification applies +* @kref: Reference count, for internal use +* @work: Work item, for internal use +* @notify: Function to be called on change. This will be +* called in process context. +* @release: Function to be called on release. This will be +* called in process context. Once registered, the +* structure must only be freed when this function is +* called or later. +*/ +struct irq_affinity_notify { +unsigned int irq; +struct kref kref; +#if defined(CONFIG_SMP ) && defined(CONFIG_GENERIC_HARDIRQS ) +struct work_struct work; +#endif +void (*notify)(struct irq_affinity_notify *,const cpumask_t *mask); +void (*release)(struct kref *ref); +}; +#if defined(CONFIG_SMP ) && defined(CONFIG_GENERIC_HARDIRQS )extern cpumask_var_t irq_default_affinity;
@@ -240,6 +264,13 @@ extern int irq_can_set_affinity(unsigned int irq);extern int irq_select_affinity (unsigned int irq);extern int irq_set_affinity_hint (unsigned int irq,const struct cpumask *m); +extern int +irq_set_affinity_notifier (unsigned int irq,struct irq_affinity_notify *notify); + +static inline void irq_run_affinity_notifiers (void ) +{ +flush_scheduled_work (); +}#else /* CONFIG_SMP */ static inline int irq_set_affinity (unsigned int irq,const struct cpumask *m)
@@ -259,6 +290,16 @@ static inline int irq_set_affinity_hint(unsigned int irq, {return -EINVAL ; } + +static inline int +irq_set_affinity_notifier (unsigned int irq,struct irq_affinity_notify *notify) +{ +return 0 ; +} + +static inline void irq_run_affinity_notifiers (void ) +{ +}#endif /* CONFIG_SMP && CONFIG_GENERIC_HARDIRQS */ #ifdef CONFIG_GENERIC_HARDIRQS
include/linux/irqdesc.h +3 -0
@@ -8,6 +8,7 @@* For now it's included from <linux/irq.h> */ +struct irq_affinity_notify ;struct proc_dir_entry ;struct timer_rand_state ;/**
@@ -24,6 +25,7 @@ struct timer_rand_state;* @last_unhandled: aging timer for unhandled count * @irqs_unhandled: stats field for spurious unhandled interrupts * @lock: locking for SMP +* @affinity_notify: context for notification of affinity changes * @pending_mask: pending rebalanced interrupts * @threads_active: number of irqaction threads currently running * @wait_for_threads: wait queue for sync_irq to wait for threaded handlers
@@ -70,6 +72,7 @@ struct irq_desc {raw_spinlock_t lock;#ifdef CONFIG_SMP const struct cpumask *affinity_hint; +struct irq_affinity_notify *affinity_notify;#ifdef CONFIG_GENERIC_PENDING_IRQ cpumask_var_t pending_mask;#endif
kernel/irq/manage.c +81 -0
99 lines not on this page · raw message
Signature · 8 lines
1.7.3.2 -- Ben Hutchings, Senior Software Engineer, Solarflare Communications Not speaking for my employer; that's the marketing department's job. They asked us to note that Solarflare product names are trademarked.
RFC PATCH 2/5 lib: cpu_rmap: CPU affinity reverse-mapping
In reply to Ben HutchingsWhen initiating I/O on a multiqueue and multi-IRQ device, we may want to select a queue for which the response will be handled on the same or a nearby CPU. This requires a reverse-map of IRQ affinity. Add library functions to support a generic reverse-mapping from CPUs to objects with affinity and the specific case where the objects are IRQs.
--- include/linux/cpu_rmap.h | 73 ++++++++++++ lib/Kconfig | 4 + lib/Makefile | 2 + lib/cpu_rmap.c | 272 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 351 insertions(+), 0 deletions(-) create mode 100644 include/linux/cpu_rmap.h create mode 100644 lib/cpu_rmap.c
include/linux/cpu_rmap.h new file +73 -0
@@ -0,0 +1,73 @@ +/* +* cpu_rmap.c: CPU affinity reverse-map support +* Copyright 2010 Solarflare Communications Inc. +* +* This program is free software; you can redistribute it and/or modify it +* under the terms of the GNU General Public License version 2 as published +* by the Free Software Foundation, incorporated herein by reference. +*/ + +#include <linux/cpumask.h> +#include <linux/gfp.h> +#include <linux/slab.h> + +/** +* struct cpu_rmap - CPU affinity reverse-map +* @near: For each CPU, the index and distance to the nearest object, +* based on affinity masks +* @size: Number of objects to be reverse-mapped +* @used: Number of objects added +* @obj: Array of object pointers +*/ +struct cpu_rmap { +struct { +u16 index; +u16 dist; + } near[NR_CPUS ]; +u16 size, used; +void *obj[0 ]; +}; +#define CPU_RMAP_DIST_INF 0xffff + +extern struct cpu_rmap *alloc_cpu_rmap (unsigned int size,gfp_t flags); + +/** +* free_cpu_rmap - free CPU affinity reverse-map +* @rmap: Reverse-map allocated with alloc_cpu_rmap(), or %NULL +*/ +static inline void free_cpu_rmap (struct cpu_rmap *rmap) +{ +kfree (rmap); +} + +extern int cpu_rmap_add (struct cpu_rmap *rmap,void *obj); +extern void cpu_rmap_update (struct cpu_rmap *rmap,u16 index, +const struct cpumask *affinity); + +static inline u16 cpu_rmap_lookup_index (struct cpu_rmap *rmap,unsigned int cpu) +{ +return rmap->near[cpu].index; +} + +static inline void *cpu_rmap_lookup_obj (struct cpu_rmap *rmap,unsigned int cpu) +{ +return rmap->obj[rmap->near[cpu].index]; +} + +#ifdef CONFIG_GENERIC_HARDIRQS + +/** +* alloc_irq_cpu_rmap - allocate CPU affinity reverse-map for IRQs +* @size: Number of objects to be mapped +* +* Must be called in process context. +*/ +static inline struct cpu_rmap *alloc_irq_cpu_rmap (unsigned int size) +{ +return alloc_cpu_rmap (size,GFP_KERNEL ); +} +extern void free_irq_cpu_rmap (struct cpu_rmap *rmap); + +extern int irq_cpu_rmap_add (struct cpu_rmap *rmap,int irq); + +#endif
lib/Kconfig +4 -0
@@ -195,6 +195,10 @@ config DISABLE_OBSOLETE_CPUMASK_FUNCTIONS bool "Disable obsolete cpumask functions" if DEBUG_PER_CPU_MAPS depends on EXPERIMENTAL && BROKEN +config CPU_RMAP + bool + depends on SMP + # # Netlink attribute parsing support is select'ed if needed #
lib/Makefile +2 -0
@@ -106,6 +106,8 @@ obj-$(CONFIG_GENERIC_ATOMIC64) += atomic64.o obj-$(CONFIG_ATOMIC64_SELFTEST) += atomic64_test.o +obj-$(CONFIG_CPU_RMAP) += cpu_rmap.o + hostprogs-y := gen_crc32table clean-files := crc32table.h
lib/cpu_rmap.c new file +272 -0
272 lines not on this page · raw message
Signature · 8 lines
1.7.3.2 -- Ben Hutchings, Senior Software Engineer, Solarflare Communications Not speaking for my employer; that's the marketing department's job. They asked us to note that Solarflare product names are trademarked.
RFC PATCH 3/5 net: RPS: Enable hardware acceleration
In reply to Ben HutchingsAllow drivers for multiqueue hardware with flow filter tables to accelerate RFS. The driver must:
1. Set net_device::rx_cpu_rmap to a cpu_rmap of the RX completion IRQs (in queue order). This will provide a mapping from CPUs to the queues for which completions are handled nearest to them.
2. Implement net_device_ops::ndo_rx_flow_steer. This operation adds or replaces a filter steering the given flow to the given RX queue, if possible.
3. Periodically remove filters for which rps_may_expire_flow() returns true.
---
At Netconf, Tom Herbert suggested making ndo_start_xmit() set up filters using information attached to the skbs it receives, rather than adding a callback from the RX path. This would be fine for TCP but not for all UDP applications and in particular it seems like it would be useless for multicast receivers.
Ben.
include/linux/netdevice.h | 31 ++++++++++++++-- net/Kconfig | 1 + net/core/dev.c | 89 ++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 112 insertions(+), 9 deletions(-)
include/linux/netdevice.h +28 -3
@@ -530,14 +530,16 @@ struct rps_map {#define RPS_MAP_SIZE (_num) (sizeof(struct rps_map) + (_num * sizeof(u16)))/* -* The rps_dev_flow structure contains the mapping of a flow to a CPU and the -* tail pointer for that CPU's input queue at the time of last enqueue. +* The rps_dev_flow structure contains the mapping of a flow to a CPU, the +* tail pointer for that CPU's input queue at the time of last enqueue, and +* a hardware filter index. */ struct rps_dev_flow {u16 cpu; -u16 fill; +u16 filter;unsigned int last_qtail; }; +#define RPS_NO_FILTER 0xffff/* * The rps_dev_flow_table structure contains a table of flow mappings.
@@ -587,6 +589,9 @@ static inline void rps_reset_sock_flow(struct rps_sock_flow_table *table,extern struct rps_sock_flow_table __rcu *rps_sock_flow_table; +extern bool rps_may_expire_flow (struct net_device *dev,u16 rxq_index, +u32 flow_id,u16 filter_id); +/* This structure contains an instance of an RX queue. */ struct netdev_rx_queue {struct rps_map __rcu *rps_map;
@@ -706,6 +711,13 @@ struct netdev_rx_queue {* int (*ndo_set_vf_port)(struct net_device *dev, int vf, * struct nlattr *port[]); * int (*ndo_get_vf_port)(struct net_device *dev, int vf, struct sk_buff *skb); +* +* RFS acceleration. +* int (*ndo_rx_flow_steer)(struct net_device *dev, const struct sk_buff *skb, +* u16 rxq_index, u32 flow_id); +* Set hardware filter for RFS. rxq_index is the target queue index; +* flow_id is a flow ID to be passed to rps_may_expire_flow() later. +* Return the filter ID on success, or a negative error code. */ #define HAVE_NET_DEVICE_OPS struct net_device_ops {
@@ -778,6 +790,12 @@ struct net_device_ops {int (*ndo_fcoe_get_wwn)(struct net_device *dev,u64 *wwn,int type);#endif +#ifdef CONFIG_RPS +int (*ndo_rx_flow_steer)(struct net_device *dev, +const struct sk_buff *skb, +u16 rxq_index, +u32 flow_id); +#endif };/*
@@ -992,6 +1010,13 @@ struct net_device {/* Number of RX queues currently active in device */ unsigned int real_num_rx_queues; + +#ifdef CONFIG_CPU_RMAP +/* CPU reverse-mapping for RX completion interrupts, indexed +* by RX queue number. Assigned by driver. This must only be +* set if the ndo_rx_flow_steer operation is defined. */ +struct cpu_rmap *rx_cpu_rmap; +#endif #endif rx_handler_func_t __rcu *rx_handler;
net/Kconfig +1 -0
@@ -218,6 +218,7 @@ source "net/dns_resolver/Kconfig" config RPS boolean depends on SMP && SYSFS && USE_GENERIC_SMP_HELPERS + select CPU_RMAP default y menu "Network testing"
net/core/dev.c +83 -6
113 lines not on this page · raw message
Signature · 8 lines
1.7.3.2 -- Ben Hutchings, Senior Software Engineer, Solarflare Communications Not speaking for my employer; that's the marketing department's job. They asked us to note that Solarflare product names are trademarked.
RFC PATCH 4/5 sfc: Limit filter search depth further for performance hints (i.e. RFS)
In reply to Ben Hutchings--- drivers/net/sfc/filter.c | 13 +++++++++---- 1 files changed, 9 insertions(+), 4 deletions(-)
drivers/net/sfc/filter.c +9 -4
@@ -26,6 +26,10 @@*/ #define FILTER_CTL_SRCH_MAX 200 +/* Don't try very hard to find space for performance hints, as this is +* counter-productive. */ +#define FILTER_CTL_SRCH_HINT_MAX 5 +struct efx_filter_table {u32 offset;/* address of table relative to BAR */ unsigned size;/* number of entries */
@@ -182,15 +186,16 @@ static int efx_filter_search(struct efx_filter_table *table, struct efx_filter_spec *spec, u32 key, bool for_insert, int *depth_required) { - unsigned hash, incr, filter_idx, depth; + unsigned hash, incr, filter_idx, depth, depth_max;struct efx_filter_spec *cmp; hash =efx_filter_hash (key); incr =efx_filter_increment (key); + depth_max = (spec->priority <=EFX_FILTER_PRI_HINT ? +FILTER_CTL_SRCH_HINT_MAX :FILTER_CTL_SRCH_MAX );for (depth =1 , filter_idx = hash & (table->size -1 ); - depth <=FILTER_CTL_SRCH_MAX && -test_bit (filter_idx, table->used_bitmap); + depth <= depth_max &&test_bit (filter_idx, table->used_bitmap); ++depth) { cmp = &table->spec[filter_idx];if (efx_filter_equal (spec, cmp))
@@ -199,7 +204,7 @@ static int efx_filter_search(struct efx_filter_table *table, }if (!for_insert)return -ENOENT ; -if (depth >FILTER_CTL_SRCH_MAX ) +if (depth > depth_max)return -EBUSY ; found: *depth_required = depth;
Signature · 8 lines
1.7.3.2 -- Ben Hutchings, Senior Software Engineer, Solarflare Communications Not speaking for my employer; that's the marketing department's job. They asked us to note that Solarflare product names are trademarked.
RFC PATCH 5/5 sfc: Implement RFS acceleration
In reply to Ben Hutchings--- drivers/net/sfc/Kconfig | 4 ++ drivers/net/sfc/efx.c | 66 ++++++++++++++++++++++++++---- drivers/net/sfc/efx.h | 9 ++++ drivers/net/sfc/filter.c | 100 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 170 insertions(+), 9 deletions(-)
drivers/net/sfc/Kconfig +4 -0
@@ -20,3 +20,7 @@ config SFC_MTD This exposes the on-board flash memory as MTD devices (e.g. /dev/mtd1). This makes it possible to upload new firmware to the NIC. +config SFC_RFS_ACCEL + bool + depends on SFC && RPS && GENERIC_HARDIRQS + default y
drivers/net/sfc/efx.c +57 -9
146 lines not on this page · raw message
drivers/net/sfc/efx.h +9 -0
@@ -77,6 +77,15 @@ extern int efx_filter_remove_filter(struct efx_nic *efx,extern void efx_filter_table_clear (struct efx_nic *efx,enum efx_filter_table_id table_id,enum efx_filter_priority priority); +#ifdef CONFIG_SFC_RFS_ACCEL +extern int efx_filter_rfs (struct net_device *net_dev,const struct sk_buff *skb, +u16 rxq_index,u32 flow_id); +extern void efx_filter_rfs_expire (struct efx_nic *efx); +#define efx_filter_rfs_enabled () 1 +#else +static inline void efx_filter_rfs_expire (struct efx_nic *efx) {} +#define efx_filter_rfs_enabled () 0 +#endif /* Channels */ extern void efx_process_channel_now (struct efx_channel *channel);
drivers/net/sfc/filter.c +100 -0
123 lines not on this page · raw message
Signature · 7 lines
1.7.3.2 -- Ben Hutchings, Senior Software Engineer, Solarflare Communications Not speaking for my employer; that's the marketing department's job. They asked us to note that Solarflare product names are trademarked.
So, some preliminary benchmark results.
Tom said he was using 200 concurrent netperf TCP_RR tests, so I've done the same, using netperf 2.4.1 (a bit out of date, I know).
The test machines were a Dell R810 with 4 * Xeon E7520 1.87 GHz and a Dell R900 with 4 * Xeon X7350 2.92 GHz (both quad-core processors, with HT disabled, for a total of 16 cores).
The kernel was an x86-64 build of net-next-2.6 with NUMA and PREEMPT_VOLUNTARY enabled.
The NICs were Solarstorm SFN5122F (dual-port SFP+) adapters connected with a Direct Attach cable.
The sfc driver allocates 4 IRQs per port (and it doesn't seem to be possible to allocate more on this hardware), which I pinned to the first core of each package.
I tested with and without pinning of processes. When pinning, I assigned netperf and netserver processes to all 16 cores in rotation.
Unpinned Pinned
No RFS Soft Accel No RFS Soft Accel
RFS RFS RFS RFSRequest size = 1, response size = 1, moderation = 60 us adaptive (default)
avg(Hz) 1759 3213 3633 2222 3523 3848 std.dev 189 76 265 703 2136 2120 lat(us) 568 311 275 450 284 260 scaled 0.55 0.88 0.63 0.92
Request size = 1, response size = 1, moderation = 20 us adaptive
avg(Hz) 1797 3616 3917 2458 3706 4125 std.dev 260 101 295 1098 1987 2186 lat(us) 556 277 255 407 270 242 scaled 0.50 0.92 0.66 0.90
Request size = 100, response size = 10000, moderation = 60 us adaptive
avg(Hz) 1658 2909 3230 2338 3003 3437 std.dev 149 144 221 993 856 1615 lat(us) 603 344 310 428 333 291 scaled 0.57 0.90 0.78 0.87
Request size = 100, response size = 10000, moderation = 20 us adaptive
avg(Hz) 3348 3110 3331 2470 3271 3487 std.dev 406 176 364 1110 1693 1817 lat(us) 299 322 300 405 306 287 scaled 1.08 0.93 0.76 0.94
So accelerated RFS gave a 6-13% improvement over software RFS in transaction rate for these various cases.
Ben.
Signature · 3 lines
Ben Hutchings, Senior Software Engineer, Solarflare Communications Not speaking for my employer; that's the marketing department's job. They asked us to note that Solarflare product names are trademarked.
Quoted 3 lines
So accelerated RFS gave a 6-13% improvement over software RFS in transaction rate for these various cases.
Very nice preliminary results! Are you seeing any OOO packets in these tests, I am still holding on to hope that these can be avoided with such mechanisms.
Tom
Quoted 4 lines · Ben Hutchings
So, some preliminary benchmark results.
Tom said he was using 200 concurrent netperf
TCP_RRtests, so I've done the same, using netperf 2.4.1 (a bit out of date, I know).
Not a huge deal for a basic TCP_RR test though.
<data snipped>
Quoted 2 lines
So accelerated RFS gave a 6-13% improvement over software RFS in transaction rate for these various cases.
Do you have any data on frequency with which unpinned netperf processes migrated from one core to another? Or PCIe utilization?
rick jones