netwrok tree allocator. Sending and receiving zero-copy support.

PATCH New Evgeniy Polyakov 1 participant 1 message Aug 26, 2006

Subsystems (2)

Networking Network drivers

lore

Share & tools

lore

Evgeniy Polyakov

author

2006-08-26 20:41 UTC #0 lore raw

PATCH 1/1 netwrok tree allocator. Sending and receiving zero-copy support.

Hello.

This patch introduce network-based receiving and sending zero-copy support. Receiving side is used for zero-copy sniffer (patch introduce full receiving support for all network allocations including netlink, unix socket and so on), sending side zero-copy is used for userspace network stack.

Receiving zero-copy support.
Information about each network allocation is stored into special buffer accessible from userspace through read/write for special char device. This information is added when skb is freed, so data placed there is valid. Such approach does not allow to store transient state of the data, for example encrypted IPsec packets if decryption happens in-place, to store such states it is required to copy information into allocated buffer (which then can be immediately freed, or decryption can happen from source do destination buffers).
Userspace can mmap whole allocation pool (using special commands via ioctl() to get appropriate information) and thus access data without any copy. There is _no_ overhead in such sniffer except one atomic operation (and delayed freeing which can lead to increased memory usage, or (like implemented) if introduced maximum amount of "locked" by sniffer data, there is only extremely small overhead introduced by reading/writing meta-information assotiated for each allocation (very small amount of copy).

Sending zero-copy support.
Sending is performaed in two steps. First one is allocation, new object can be accessed without any copy like described above (and actually can be used with things like kevents to implement proposed by Ulrich Drepper "DMA" allocation (allocation of data, which can be used both by kernel and userspace without additional copies)). Second step is data commit, where allocated packet is attached to special skb and pushed into the stack using dst_output().

Above extensions are possible due to design of network allocator. Performance test for NTA shows very noticeble performance improvement over existing SLAB one (details can be found on project homepage [1]).

I have not run promised performance test for zero-copy sniffer yet (I'm going to vacations to Finland and Sweden until 31 Aug).

Interested reader can find all userspace applications (zero-copy sniffer and sending program) on project homepage [1] (server will be turned on on Monday 28).

Sniffer development uncovered very strange (and I would not call it correct) behaviour of Linux startup process - there is extremely enormous amount of continuous allocations of (likely netlink) network messages (several tens of thousands) during device enumerations and initial system startup on my test system (FC2 or 3, small number of external devices). Initial suspicious is kobject_uevent, second one is probably HAL. That flow of allocations eventually stops, but number of allocated messages can cover any set of devices in system like Blue Gene.

1. Network tree allocator. Sending and receiving zero-copy networking.
http://tservice.net.ru/~s0mbre/old/?section=projects&item=nta

Signed-off-by Evgeniy Polyakov <[email protected]>

include/linux/skbuff.h +13 -0

index 19c96d4..cf3ae3b 100644
@@ -327,16 +327,29 @@ #include <linux/slab.h>
 
 #include <asm/system.h>
 
+extern void *avl_alloc(unsigned int size, gfp_t gfp_mask);
+extern void avl_free(void *ptr, unsigned int size);
+extern int avl_init(void);
+
 extern void kfree_skb(struct sk_buff *skb);
 extern void	       __kfree_skb(struct sk_buff *skb);
 extern struct sk_buff *__alloc_skb(unsigned int size,
 				   gfp_t priority, int fclone);
+extern struct sk_buff *__alloc_skb_emtpy(unsigned int size,
+				   gfp_t priority);
+
 static inline struct sk_buff *alloc_skb(unsigned int size,
 					gfp_t priority)
 {
 	return __alloc_skb(size, priority, 0);
 }
 
+static inline struct sk_buff *alloc_skb_empty(unsigned int size,
+					gfp_t priority)
+{
+	return __alloc_skb_emtpy(size, priority);
+}
+
 static inline struct sk_buff *alloc_skb_fclone(unsigned int size,
 					       gfp_t priority)
 {

net/core/Makefile +2 -0

index 2645ba4..d86d468 100644
@@ -10,6 +10,8 @@ obj-$(CONFIG_SYSCTL) += sysctl_net_core.
 obj-y		     += dev.o ethtool.o dev_mcast.o dst.o netevent.o \
 			neighbour.o rtnetlink.o utils.o link_watch.o filter.o
 
+obj-y += alloc/
+
 obj-$(CONFIG_XFRM) += flow.o
 obj-$(CONFIG_SYSFS) += net-sysfs.o
 obj-$(CONFIG_NET_DIVERT) += dv.o

net/core/alloc/Makefile new file +3 -0

new file mode 100644
index 0000000..779eba2
@@ -0,0 +1,3 @@
+obj-y		:= allocator.o
+
+allocator-y	:= avl.o zc.o

net/core/alloc/avl.c new file +769 -0

769 lines not on this page · raw message

net/core/alloc/avl.h new file +223 -0

223 lines not on this page · raw message

net/core/alloc/zc.c new file +483 -0

483 lines not on this page · raw message

net/core/skbuff.c +34 -4

index 022d889..7eec140 100644
@@ -125,6 +125,33 @@ EXPORT_SYMBOL(skb_truesize_bug);
  *
  */
 
+
+/**
+ * 	__alloc_skb_empty - allocate an empty network buffer
+ *	@size: size to allocate
+ *	@gfp_mask: allocation mask
+ */
+
+struct sk_buff *__alloc_skb_emtpy(unsigned int size, gfp_t gfp_mask)
+{
+	struct skb_shared_info *shinfo;
+	struct sk_buff *skb;
+
+	/* Get the HEAD */
+	skb = kmem_cache_alloc(skbuff_head_cache, gfp_mask & ~__GFP_DMA);
+	if (!skb)
+		goto out;
+
+	memset(skb, 0, offsetof(struct sk_buff, truesize));
+	
+	size = SKB_DATA_ALIGN(size);
+	skb->truesize = size + sizeof(struct sk_buff);
+	atomic_set(&skb->users, 1);
+
+out:
+	return skb;
+}
+
 /**
  *	__alloc_skb	-	allocate a network buffer
  *	@size: size to allocate

@@ -156,7 +183,7 @@ struct sk_buff *__alloc_skb(unsigned int
 
 	/* Get the DATA. Size must match skb_add_mtu(). */
 	size = SKB_DATA_ALIGN(size);
-	data = ____kmalloc(size + sizeof(struct skb_shared_info), gfp_mask);
+	data = avl_alloc(size + sizeof(struct skb_shared_info), gfp_mask);
 	if (!data)
 		goto nodata;

@@ -223,7 +250,7 @@ struct sk_buff *alloc_skb_from_cache(kme
 
 	/* Get the DATA. */
 	size = SKB_DATA_ALIGN(size);
-	data = kmem_cache_alloc(cp, gfp_mask);
+	data = avl_alloc(size, gfp_mask);
 	if (!data)
 		goto nodata;

@@ -313,7 +340,7 @@ static void skb_release_data(struct sk_b
 		if (skb_shinfo(skb)->frag_list)
 			skb_drop_fraglist(skb);
 
-		kfree(skb->head);
+		avl_free(skb->head, skb->end - skb->head + sizeof(struct skb_shared_info));
 	}
 }

@@ -688,7 +715,7 @@ int pskb_expand_head(struct sk_buff *skb
 
 	size = SKB_DATA_ALIGN(size);
 
-	data = kmalloc(size + sizeof(struct skb_shared_info), gfp_mask);
+	data = avl_alloc(size + sizeof(struct skb_shared_info), gfp_mask);
 	if (!data)
 		goto nodata;

@@ -2057,6 +2084,9 @@ void __init skb_init(void)
 						NULL, NULL);
 	if (!skbuff_fclone_cache)
 		panic("cannot create skbuff cache");
+
+	if (avl_init())
+		panic("Failed to initialize network tree allocator.\n");
 }
 
 EXPORT_SYMBOL(___pskb_trim);

Signature · 1 line

	Evgeniy Polyakov