Skip to contentSkip to conversation

RFS hardware acceleration (v2)

RFC PATCH In review Ben Hutchings 3 participants 9 messages Nov 19, 2010

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

cover RFS hardware acceleration (v2)

This patch series extends RFS to use hardware RX filters where available. Depending on the number of hardware RX queues and their IRQs' affinity, this should reduce the need for IPIs or at least get packets delivered to the right NUMA node.

The first patch implements IRQ affinity notifiers, based on the outline that Thomas Gleixner wrote in response to the previous version of this patch series.

The second patch is a generalisation of the CPU affinity reverse- mapping, plus functions to maintain such a mapping based on the new IRQ affinity notifiers.

The remaining patches add the RFS acceleration hooks and an implementation in the sfc driver. These are largely unchanged from the previous version of this patch series; I haven't yet had the time to do further tuning yet. However I have done some more benchmarking and will send some results in a separate follow-up to this.

Ben.

Ben Hutchings (5):
genirq: Add IRQ affinity notifiers
lib: cpu_rmap: CPU affinity reverse-mapping
net: RPS: Enable hardware acceleration
sfc: Limit filter search depth further for performance hints (i.e.
RFS)
sfc: Implement RFS acceleration

 drivers/net/sfc/Kconfig   |    4 +
 drivers/net/sfc/efx.c     |   66 ++++++++++--
 drivers/net/sfc/efx.h     |    9 ++
 drivers/net/sfc/filter.c  |  113 ++++++++++++++++++-
 include/linux/cpu_rmap.h  |   73 ++++++++++++
 include/linux/interrupt.h |   41 +++++++
 include/linux/irqdesc.h   |    3 +
 include/linux/netdevice.h |   31 +++++-
 kernel/irq/manage.c       |   81 +++++++++++++
 lib/Kconfig               |    4 +
 lib/Makefile              |    2 +
 lib/cpu_rmap.c            |  272 +++++++++++++++++++++++++++++++++++++++++++++
 net/Kconfig               |    1 +
 net/core/dev.c            |   89 ++++++++++++++-
 14 files changed, 767 insertions(+), 22 deletions(-)
 create mode 100644 include/linux/cpu_rmap.h
 create mode 100644 lib/cpu_rmap.c
Signature · 7 lines
1.7.3.2


-- 
Ben Hutchings, Senior Software Engineer, Solarflare Communications
Not speaking for my employer; that's the marketing department's job.
They asked us to note that Solarflare product names are trademarked.
#1 lore raw

RFC PATCH 1/5 genirq: Add IRQ affinity notifiers

When initiating I/O on a multiqueue and multi-IRQ device, we may want to select a queue for which the response will be handled on the same or a nearby CPU. This requires a reverse-map of IRQ affinity. Add a notification mechanism to support this.

This is based closely on work by Thomas Gleixner <[email protected]>.

---
 include/linux/interrupt.h |   41 +++++++++++++++++++++++
 include/linux/irqdesc.h   |    3 ++
 kernel/irq/manage.c       |   81 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 125 insertions(+), 0 deletions(-)
include/linux/interrupt.h +41 -0
index 79d0c4f..1649b30 100644
@@ -14,6 +14,8 @@
 #include <linux/smp.h>
 #include <linux/percpu.h>
 #include <linux/hrtimer.h>
+#include <linux/kref.h>
+#include <linux/workqueue.h>
 
 #include <asm/atomic.h>
 #include <asm/ptrace.h>
@@ -231,6 +233,28 @@ static inline void resume_device_irqs(void) { };
 static inline int check_wakeup_irqs(void) { return 0; }
 #endif
 
+/**
+ * struct irq_affinity_notify - context for notification of IRQ affinity changes
+ * @irq:		Interrupt to which notification applies
+ * @kref:		Reference count, for internal use
+ * @work:		Work item, for internal use
+ * @notify:		Function to be called on change.  This will be
+ *			called in process context.
+ * @release:		Function to be called on release.  This will be
+ *			called in process context.  Once registered, the
+ *			structure must only be freed when this function is
+ *			called or later.
+ */
+struct irq_affinity_notify {
+        unsigned int irq;
+        struct kref kref;
+#if defined(CONFIG_SMP) && defined(CONFIG_GENERIC_HARDIRQS)
+        struct work_struct work;
+#endif
+        void (*notify)(struct irq_affinity_notify *, const cpumask_t *mask);
+        void (*release)(struct kref *ref);
+};
+
 #if defined(CONFIG_SMP) && defined(CONFIG_GENERIC_HARDIRQS)
 
 extern cpumask_var_t irq_default_affinity;
@@ -240,6 +264,13 @@ extern int irq_can_set_affinity(unsigned int irq);
 extern int irq_select_affinity(unsigned int irq);
 
 extern int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m);
+extern int
+irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify);
+
+static inline void irq_run_affinity_notifiers(void)
+{
+	flush_scheduled_work();
+}
 #else /* CONFIG_SMP */
 
 static inline int irq_set_affinity(unsigned int irq, const struct cpumask *m)
@@ -259,6 +290,16 @@ static inline int irq_set_affinity_hint(unsigned int irq,
 {
 	return -EINVAL;
 }
+
+static inline int
+irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify)
+{
+	return 0;
+}
+
+static inline void irq_run_affinity_notifiers(void)
+{
+}
 #endif /* CONFIG_SMP && CONFIG_GENERIC_HARDIRQS */
 
 #ifdef CONFIG_GENERIC_HARDIRQS
include/linux/irqdesc.h +3 -0
index 979c68c..5e0d2e4 100644
@@ -8,6 +8,7 @@
  * For now it's included from <linux/irq.h>
  */
 
+struct irq_affinity_notify;
 struct proc_dir_entry;
 struct timer_rand_state;
 /**
@@ -24,6 +25,7 @@ struct timer_rand_state;
  * @last_unhandled:	aging timer for unhandled count
  * @irqs_unhandled:	stats field for spurious unhandled interrupts
  * @lock:		locking for SMP
+ * @affinity_notify:	context for notification of affinity changes
  * @pending_mask:	pending rebalanced interrupts
  * @threads_active:	number of irqaction threads currently running
  * @wait_for_threads:	wait queue for sync_irq to wait for threaded handlers
@@ -70,6 +72,7 @@ struct irq_desc {
 	raw_spinlock_t		lock;
 #ifdef CONFIG_SMP
 	const struct cpumask	*affinity_hint;
+	struct irq_affinity_notify *affinity_notify;
 #ifdef CONFIG_GENERIC_PENDING_IRQ
 	cpumask_var_t		pending_mask;
 #endif
kernel/irq/manage.c +81 -0

99 lines not on this page · raw message

Signature · 8 lines
1.7.3.2



-- 
Ben Hutchings, Senior Software Engineer, Solarflare Communications
Not speaking for my employer; that's the marketing department's job.
They asked us to note that Solarflare product names are trademarked.
#2 lore raw

RFC PATCH 2/5 lib: cpu_rmap: CPU affinity reverse-mapping

In reply to Ben Hutchings

When initiating I/O on a multiqueue and multi-IRQ device, we may want to select a queue for which the response will be handled on the same or a nearby CPU. This requires a reverse-map of IRQ affinity. Add library functions to support a generic reverse-mapping from CPUs to objects with affinity and the specific case where the objects are IRQs.

---
 include/linux/cpu_rmap.h |   73 ++++++++++++
 lib/Kconfig              |    4 +
 lib/Makefile             |    2 +
 lib/cpu_rmap.c           |  272 ++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 351 insertions(+), 0 deletions(-)
 create mode 100644 include/linux/cpu_rmap.h
 create mode 100644 lib/cpu_rmap.c
include/linux/cpu_rmap.h new file +73 -0
new file mode 100644
index 0000000..4771e95
@@ -0,0 +1,73 @@
+/*
+ * cpu_rmap.c: CPU affinity reverse-map support
+ * Copyright 2010 Solarflare Communications Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, incorporated herein by reference.
+ */
+
+#include <linux/cpumask.h>
+#include <linux/gfp.h>
+#include <linux/slab.h>
+
+/**
+ * struct cpu_rmap - CPU affinity reverse-map
+ * @near: For each CPU, the index and distance to the nearest object,
+ *      based on affinity masks
+ * @size: Number of objects to be reverse-mapped
+ * @used: Number of objects added
+ * @obj: Array of object pointers
+ */
+struct cpu_rmap {
+	struct {
+		u16     index;
+		u16     dist;
+	} near[NR_CPUS];
+	u16		size, used;
+	void		*obj[0];
+};
+#define CPU_RMAP_DIST_INF 0xffff
+
+extern struct cpu_rmap *alloc_cpu_rmap(unsigned int size, gfp_t flags);
+
+/**
+ * free_cpu_rmap - free CPU affinity reverse-map
+ * @rmap: Reverse-map allocated with alloc_cpu_rmap(), or %NULL
+ */
+static inline void free_cpu_rmap(struct cpu_rmap *rmap)
+{
+	kfree(rmap);
+}
+
+extern int cpu_rmap_add(struct cpu_rmap *rmap, void *obj);
+extern void cpu_rmap_update(struct cpu_rmap *rmap, u16 index,
+			    const struct cpumask *affinity);
+
+static inline u16 cpu_rmap_lookup_index(struct cpu_rmap *rmap, unsigned int cpu)
+{
+	return rmap->near[cpu].index;
+}
+
+static inline void *cpu_rmap_lookup_obj(struct cpu_rmap *rmap, unsigned int cpu)
+{
+	return rmap->obj[rmap->near[cpu].index];
+}
+
+#ifdef CONFIG_GENERIC_HARDIRQS
+
+/**
+ * alloc_irq_cpu_rmap - allocate CPU affinity reverse-map for IRQs
+ * @size: Number of objects to be mapped
+ *
+ * Must be called in process context.
+ */
+static inline struct cpu_rmap *alloc_irq_cpu_rmap(unsigned int size)
+{
+	return alloc_cpu_rmap(size, GFP_KERNEL);
+}
+extern void free_irq_cpu_rmap(struct cpu_rmap *rmap);
+
+extern int irq_cpu_rmap_add(struct cpu_rmap *rmap, int irq);
+
+#endif
lib/Kconfig +4 -0
index fa9bf2c..a66c76d 100644
@@ -195,6 +195,10 @@ config DISABLE_OBSOLETE_CPUMASK_FUNCTIONS
        bool "Disable obsolete cpumask functions" if DEBUG_PER_CPU_MAPS
        depends on EXPERIMENTAL && BROKEN
 
+config CPU_RMAP
+	bool
+	depends on SMP
+
 #
 # Netlink attribute parsing support is select'ed if needed
 #
lib/Makefile +2 -0
index e6a3763..ded1a81 100644
@@ -106,6 +106,8 @@ obj-$(CONFIG_GENERIC_ATOMIC64) += atomic64.o
 
 obj-$(CONFIG_ATOMIC64_SELFTEST) += atomic64_test.o
 
+obj-$(CONFIG_CPU_RMAP) += cpu_rmap.o
+
 hostprogs-y	:= gen_crc32table
 clean-files	:= crc32table.h
 
lib/cpu_rmap.c new file +272 -0

272 lines not on this page · raw message

Signature · 8 lines
1.7.3.2



-- 
Ben Hutchings, Senior Software Engineer, Solarflare Communications
Not speaking for my employer; that's the marketing department's job.
They asked us to note that Solarflare product names are trademarked.
#3 lore raw

RFC PATCH 3/5 net: RPS: Enable hardware acceleration

In reply to Ben Hutchings

Allow drivers for multiqueue hardware with flow filter tables to accelerate RFS. The driver must:

1. Set net_device::rx_cpu_rmap to a cpu_rmap of the RX completion IRQs (in queue order). This will provide a mapping from CPUs to the queues for which completions are handled nearest to them.

2. Implement net_device_ops::ndo_rx_flow_steer. This operation adds or replaces a filter steering the given flow to the given RX queue, if possible.

3. Periodically remove filters for which rps_may_expire_flow() returns true.
---
At Netconf, Tom Herbert suggested making ndo_start_xmit() set up filters using information attached to the skbs it receives, rather than adding a callback from the RX path. This would be fine for TCP but not for all UDP applications and in particular it seems like it would be useless for multicast receivers.

Ben.

 include/linux/netdevice.h |   31 ++++++++++++++--
 net/Kconfig               |    1 +
 net/core/dev.c            |   89 ++++++++++++++++++++++++++++++++++++++++++---
 3 files changed, 112 insertions(+), 9 deletions(-)
include/linux/netdevice.h +28 -3
index b45c1b8..7875042 100644
@@ -530,14 +530,16 @@ struct rps_map {
 #define RPS_MAP_SIZE(_num) (sizeof(struct rps_map) + (_num * sizeof(u16)))
 
 /*
- * The rps_dev_flow structure contains the mapping of a flow to a CPU and the
- * tail pointer for that CPU's input queue at the time of last enqueue.
+ * The rps_dev_flow structure contains the mapping of a flow to a CPU, the
+ * tail pointer for that CPU's input queue at the time of last enqueue, and
+ * a hardware filter index.
  */
 struct rps_dev_flow {
 	u16 cpu;
-	u16 fill;
+	u16 filter;
 	unsigned int last_qtail;
 };
+#define RPS_NO_FILTER 0xffff
 
 /*
  * The rps_dev_flow_table structure contains a table of flow mappings.
@@ -587,6 +589,9 @@ static inline void rps_reset_sock_flow(struct rps_sock_flow_table *table,
 
 extern struct rps_sock_flow_table __rcu *rps_sock_flow_table;
 
+extern bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
+				u32 flow_id, u16 filter_id);
+
 /* This structure contains an instance of an RX queue. */
 struct netdev_rx_queue {
 	struct rps_map __rcu		*rps_map;
@@ -706,6 +711,13 @@ struct netdev_rx_queue {
  * int (*ndo_set_vf_port)(struct net_device *dev, int vf,
  *			  struct nlattr *port[]);
  * int (*ndo_get_vf_port)(struct net_device *dev, int vf, struct sk_buff *skb);
+ *
+ *	RFS acceleration.
+ * int (*ndo_rx_flow_steer)(struct net_device *dev, const struct sk_buff *skb,
+ *			    u16 rxq_index, u32 flow_id);
+ *	Set hardware filter for RFS.  rxq_index is the target queue index;
+ *	flow_id is a flow ID to be passed to rps_may_expire_flow() later.
+ *	Return the filter ID on success, or a negative error code.
  */
 #define HAVE_NET_DEVICE_OPS
 struct net_device_ops {
@@ -778,6 +790,12 @@ struct net_device_ops {
 	int			(*ndo_fcoe_get_wwn)(struct net_device *dev,
 						    u64 *wwn, int type);
 #endif
+#ifdef CONFIG_RPS
+	int			(*ndo_rx_flow_steer)(struct net_device *dev,
+						     const struct sk_buff *skb,
+						     u16 rxq_index,
+						     u32 flow_id);
+#endif
 };
 
 /*
@@ -992,6 +1010,13 @@ struct net_device {
 
 	/* Number of RX queues currently active in device */
 	unsigned int		real_num_rx_queues;
+
+#ifdef CONFIG_CPU_RMAP
+	/* CPU reverse-mapping for RX completion interrupts, indexed
+	 * by RX queue number.  Assigned by driver.  This must only be
+	 * set if the ndo_rx_flow_steer operation is defined. */
+	struct cpu_rmap		*rx_cpu_rmap;
+#endif
 #endif
 
 	rx_handler_func_t __rcu	*rx_handler;
net/Kconfig +1 -0
index 55fd82e..f330a45 100644
@@ -218,6 +218,7 @@ source "net/dns_resolver/Kconfig"
 config RPS
 	boolean
 	depends on SMP && SYSFS && USE_GENERIC_SMP_HELPERS
+	select CPU_RMAP
 	default y
 
 menu "Network testing"
net/core/dev.c +83 -6

113 lines not on this page · raw message

Signature · 8 lines
1.7.3.2



-- 
Ben Hutchings, Senior Software Engineer, Solarflare Communications
Not speaking for my employer; that's the marketing department's job.
They asked us to note that Solarflare product names are trademarked.
#4 lore raw

RFC PATCH 4/5 sfc: Limit filter search depth further for performance hints (i.e. RFS)

In reply to Ben Hutchings
---
 drivers/net/sfc/filter.c |   13 +++++++++----
 1 files changed, 9 insertions(+), 4 deletions(-)
drivers/net/sfc/filter.c +9 -4
index 52cb608..e0ad1b8 100644
@@ -26,6 +26,10 @@
  */
 #define FILTER_CTL_SRCH_MAX 200
 
+/* Don't try very hard to find space for performance hints, as this is
+ * counter-productive. */
+#define FILTER_CTL_SRCH_HINT_MAX 5
+
 struct efx_filter_table {
 	u32		offset;		/* address of table relative to BAR */
 	unsigned	size;		/* number of entries */
@@ -182,15 +186,16 @@ static int efx_filter_search(struct efx_filter_table *table,
 			     struct efx_filter_spec *spec, u32 key,
 			     bool for_insert, int *depth_required)
 {
-	unsigned hash, incr, filter_idx, depth;
+	unsigned hash, incr, filter_idx, depth, depth_max;
 	struct efx_filter_spec *cmp;
 
 	hash = efx_filter_hash(key);
 	incr = efx_filter_increment(key);
+	depth_max = (spec->priority <= EFX_FILTER_PRI_HINT ?
+		     FILTER_CTL_SRCH_HINT_MAX : FILTER_CTL_SRCH_MAX);
 
 	for (depth = 1, filter_idx = hash & (table->size - 1);
-	     depth <= FILTER_CTL_SRCH_MAX &&
-		     test_bit(filter_idx, table->used_bitmap);
+	     depth <= depth_max && test_bit(filter_idx, table->used_bitmap);
 	     ++depth) {
 		cmp = &table->spec[filter_idx];
 		if (efx_filter_equal(spec, cmp))
@@ -199,7 +204,7 @@ static int efx_filter_search(struct efx_filter_table *table,
 	}
 	if (!for_insert)
 		return -ENOENT;
-	if (depth > FILTER_CTL_SRCH_MAX)
+	if (depth > depth_max)
 		return -EBUSY;
 found:
 	*depth_required = depth;
Signature · 8 lines
1.7.3.2



-- 
Ben Hutchings, Senior Software Engineer, Solarflare Communications
Not speaking for my employer; that's the marketing department's job.
They asked us to note that Solarflare product names are trademarked.
#5 lore raw

RFC PATCH 5/5 sfc: Implement RFS acceleration

In reply to Ben Hutchings
---
 drivers/net/sfc/Kconfig  |    4 ++
 drivers/net/sfc/efx.c    |   66 ++++++++++++++++++++++++++----
 drivers/net/sfc/efx.h    |    9 ++++
 drivers/net/sfc/filter.c |  100 ++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 170 insertions(+), 9 deletions(-)
drivers/net/sfc/Kconfig +4 -0
index a65c986..8d286c3 100644
@@ -20,3 +20,7 @@ config SFC_MTD
 	  This exposes the on-board flash memory as MTD devices (e.g.
 	  /dev/mtd1).  This makes it possible to upload new firmware
 	  to the NIC.
+config SFC_RFS_ACCEL
+	bool
+	depends on SFC && RPS && GENERIC_HARDIRQS
+	default y
drivers/net/sfc/efx.c +57 -9

146 lines not on this page · raw message

drivers/net/sfc/efx.h +9 -0
index 10a1bf4..8b8cf63 100644
@@ -77,6 +77,15 @@ extern int efx_filter_remove_filter(struct efx_nic *efx,
 extern void efx_filter_table_clear(struct efx_nic *efx,
 				   enum efx_filter_table_id table_id,
 				   enum efx_filter_priority priority);
+#ifdef CONFIG_SFC_RFS_ACCEL
+extern int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,
+			  u16 rxq_index, u32 flow_id);
+extern void efx_filter_rfs_expire(struct efx_nic *efx);
+#define efx_filter_rfs_enabled() 1
+#else
+static inline void efx_filter_rfs_expire(struct efx_nic *efx) {}
+#define efx_filter_rfs_enabled() 0
+#endif
 
 /* Channels */
 extern void efx_process_channel_now(struct efx_channel *channel);
drivers/net/sfc/filter.c +100 -0

123 lines not on this page · raw message

Signature · 7 lines
1.7.3.2


-- 
Ben Hutchings, Senior Software Engineer, Solarflare Communications
Not speaking for my employer; that's the marketing department's job.
They asked us to note that Solarflare product names are trademarked.
#6 lore raw
In reply to Ben Hutchings

So, some preliminary benchmark results.

Tom said he was using 200 concurrent netperf TCP_RR tests, so I've done the same, using netperf 2.4.1 (a bit out of date, I know).

The test machines were a Dell R810 with 4 * Xeon E7520 1.87 GHz and a Dell R900 with 4 * Xeon X7350 2.92 GHz (both quad-core processors, with HT disabled, for a total of 16 cores).

The kernel was an x86-64 build of net-next-2.6 with NUMA and PREEMPT_VOLUNTARY enabled.

The NICs were Solarstorm SFN5122F (dual-port SFP+) adapters connected with a Direct Attach cable.

The sfc driver allocates 4 IRQs per port (and it doesn't seem to be possible to allocate more on this hardware), which I pinned to the first core of each package.

I tested with and without pinning of processes. When pinning, I assigned netperf and netserver processes to all 16 cores in rotation.

               Unpinned                 Pinned
        No RFS  Soft    Accel   No RFS  Soft    Accel
                RFS     RFS             RFS     RFS

Request size = 1, response size = 1, moderation = 60 us adaptive (default)

avg(Hz) 1759    3213    3633    2222    3523    3848
std.dev 189     76      265     703     2136    2120
lat(us) 568     311     275     450     284     260
scaled          0.55    0.88            0.63    0.92

Request size = 1, response size = 1, moderation = 20 us adaptive

avg(Hz) 1797    3616    3917    2458    3706    4125
std.dev 260     101     295     1098    1987    2186
lat(us) 556     277     255     407     270     242
scaled          0.50    0.92            0.66    0.90

Request size = 100, response size = 10000, moderation = 60 us adaptive

avg(Hz) 1658    2909    3230    2338    3003    3437
std.dev 149     144     221     993     856     1615
lat(us) 603     344     310     428     333     291
scaled          0.57    0.90            0.78    0.87

Request size = 100, response size = 10000, moderation = 20 us adaptive

avg(Hz) 3348    3110    3331    2470    3271    3487
std.dev 406     176     364     1110    1693    1817
lat(us) 299     322     300     405     306     287
scaled          1.08    0.93            0.76    0.94

So accelerated RFS gave a 6-13% improvement over software RFS in transaction rate for these various cases.

Ben.

Signature · 3 lines
Ben Hutchings, Senior Software Engineer, Solarflare Communications
Not speaking for my employer; that's the marketing department's job.
They asked us to note that Solarflare product names are trademarked.
#7 lore raw
Quoted 3 lines

So accelerated RFS gave a 6-13% improvement over software RFS in transaction rate for these various cases.

Very nice preliminary results! Are you seeing any OOO packets in these tests, I am still holding on to hope that these can be avoided with such mechanisms.

Tom

#8 lore raw
In reply to Ben Hutchings
Quoted 4 lines · Ben Hutchings

So, some preliminary benchmark results.

Tom said he was using 200 concurrent netperf TCP_RR tests, so I've done the same, using netperf 2.4.1 (a bit out of date, I know).

Not a huge deal for a basic TCP_RR test though.

<data snipped>

Quoted 2 lines

So accelerated RFS gave a 6-13% improvement over software RFS in transaction rate for these various cases.

Do you have any data on frequency with which unpinned netperf processes migrated from one core to another? Or PCIe utilization?

rick jones