Skip to contentSkip to conversation

improve rq buff allocation and reduce dma mapping

PATCH In review Govindarajulu Varadarajan 2 participants 4 messages Feb 11, 2015 – Feb 19, 2015

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

v2 cover improve rq buff allocation and reduce dma mapping

The following series tries to address these two problem in rq buff allocation.

* Memory wastage because of large 9k allocation using kmalloc:
For 9k mtu buffer, netdev_alloc_skb_ip_align internally calls kmalloc for size > 4096. In case of 9k buff, kmalloc returns pages for order 2, 16k. And we use only ~9k of 16k. 7k memory wasted. Using the frag the frag allocator in patch 1/2, we can allocate three 9k buffs in a 32k page size. Typical enic configuration has 8 rq, and desc ring of size 4096. Thats 8 * 4096 * (16*1024) = 512 MB. Using this frag allocator:
8 * 4096 * (32*1024/3) = 341 MB. Thats 171 MB of memory save.

* frequent dma_map() calls:
we call dma_map() for every buff we allocate. When iommu is on, This is very cpu time consuming. From my testing, most of the cpu cycles are wasted spinning on spin_lock_irqsave(&iovad->iova_rbtree_lock, flags) in intel_map_page() .. -> ..__alloc_and_insert_iova_range()

With this patch, we call dma_map() once for 32k page. i.e once for every three 9k desc, and once every twenty 1500 bytes desc.

Here are testing result with 8 rq, 4096 ring size and 9k mtu. irq of each rq is affinitized with different CPU. Ran iperf with 32 threads. Link is 10G. iommu is on.

CPU utilization throughput
without patch 100% 1.8 Gbps
with patch 13% 9.8 Gbps

v2:
Remove changing order facility

Govindarajulu Varadarajan (2):
enic: implement frag allocator
enic: Add rq allocation failure stats

 drivers/net/ethernet/cisco/enic/enic.h         |  15 +++
 drivers/net/ethernet/cisco/enic/enic_ethtool.c |   2 +
 drivers/net/ethernet/cisco/enic/enic_main.c    | 158 +++++++++++++++++++++----
 drivers/net/ethernet/cisco/enic/vnic_rq.c      |  13 ++
 drivers/net/ethernet/cisco/enic/vnic_rq.h      |   2 +
 drivers/net/ethernet/cisco/enic/vnic_stats.h   |   2 +
 6 files changed, 168 insertions(+), 24 deletions(-)
Signature · 1 line
2.3.0
#1 lore raw

PATCH v2 1/2 enic: implement frag allocator

This patch implements frag allocator for rq buffer. This is based on __alloc_page_frag & __page_frag_refill implementation in net/core/skbuff.c

In addition to frag allocation from order(3) page in __alloc_page_frag, we also maintain dma address of the page. While allocating a frag for rx buffer we return va + offset for virtual address of the frag, and pa + offset for dma address of the frag. This reduces the number of calls to dma_map() by 1/3 for 9k mtu and by 1/20 for 1500 mtu.

__alloc_page_frag is limited to max buffer size of PAGE_SIZE, i.e 4096 in most of the cases. So 9k buffer allocation goes through kmalloc which return page of order 2, 16k. We waste 7k bytes for every 9k buffer.

we maintain dma_count variable which is incremented when we allocate a frag. enic_unmap_dma will decrement the dma_count and unmap it when there is no user of that page in rx ring.

This reduces the memory utilization for 9k mtu by 33%.

---
 drivers/net/ethernet/cisco/enic/enic.h      |  15 +++
 drivers/net/ethernet/cisco/enic/enic_main.c | 155 +++++++++++++++++++++++-----
 drivers/net/ethernet/cisco/enic/vnic_rq.c   |  13 +++
 drivers/net/ethernet/cisco/enic/vnic_rq.h   |   2 +
 4 files changed, 161 insertions(+), 24 deletions(-)
drivers/net/ethernet/cisco/enic/enic.h +15 -0
index 84b6a2b..0f3b6327 100644
@@ -20,6 +20,11 @@
 #ifndef _ENIC_H_
 #define _ENIC_H_
 
+#include <linux/if.h>
+#include <linux/if_link.h>
+#include <linux/if_ether.h>
+#include <linux/netdevice.h>
+
 #include "vnic_enet.h"
 #include "vnic_dev.h"
 #include "vnic_wq.h"
@@ -191,6 +196,16 @@ struct enic {
 	struct vnic_gen_stats gen_stats;
 };
 
+#define ENIC_ALLOC_ORDER	PAGE_ALLOC_COSTLY_ORDER
+
+struct enic_alloc_cache {
+	struct page_frag	frag;
+	unsigned int		pagecnt_bias;
+	int			dma_count;
+	void			*va;
+	dma_addr_t		pa;
+};
+
 static inline struct device *enic_get_dev(struct enic *enic)
 {
 	return &(enic->pdev->dev);
drivers/net/ethernet/cisco/enic/enic_main.c +131 -24

212 lines not on this page · raw message

drivers/net/ethernet/cisco/enic/vnic_rq.c +13 -0
index 36a2ed6..c31669f 100644
@@ -26,6 +26,7 @@
 
 #include "vnic_dev.h"
 #include "vnic_rq.h"
+#include "enic.h"
 
 static int vnic_rq_alloc_bufs(struct vnic_rq *rq)
 {
@@ -199,6 +200,18 @@ void vnic_rq_clean(struct vnic_rq *rq,
 		rq->ring.desc_avail++;
 	}
 
+	if (rq->ec) {
+		struct enic *enic = vnic_dev_priv(rq->vdev);
+		struct enic_alloc_cache *ec = rq->ec;
+
+		WARN_ON(ec->dma_count);
+		pci_unmap_single(enic->pdev, ec->pa, ec->frag.size,
+				 PCI_DMA_FROMDEVICE);
+		atomic_sub(ec->pagecnt_bias - 1, &ec->frag.page->_count);
+		__free_pages(ec->frag.page, get_order(ec->frag.size));
+		kfree(ec);
+		rq->ec = NULL;
+	}
 	/* Use current fetch_index as the ring starting point */
 	fetch_index = ioread32(&rq->ctrl->fetch_index);
 
drivers/net/ethernet/cisco/enic/vnic_rq.h +2 -0
index 8111d52..2e4815a 100644
@@ -73,6 +73,7 @@ struct vnic_rq_buf {
 	unsigned int index;
 	void *desc;
 	uint64_t wr_id;
+	struct enic_alloc_cache	*ec;
 };
 
 struct vnic_rq {
@@ -100,6 +101,7 @@ struct vnic_rq {
 	unsigned int bpoll_state;
 	spinlock_t bpoll_lock;
 #endif /* CONFIG_NET_RX_BUSY_POLL */
+	struct enic_alloc_cache	*ec;
 };
 
 static inline unsigned int vnic_rq_desc_avail(struct vnic_rq *rq)
Signature · 1 line
2.3.0
maintainer
#2 lore raw

Re: [PATCH net-next v2 1/2] enic: implement frag allocator

From: Govindarajulu Varadarajan <[email protected]>
Date: Wed, 11 Feb 2015 18:29:17 +0530

Quoted 13 lines

This patch implements frag allocator for rq buffer. This is based on __alloc_page_frag & __page_frag_refill implementation in net/core/skbuff.c

In addition to frag allocation from order(3) page in __alloc_page_frag, we also maintain dma address of the page. While allocating a frag for rx buffer we return va + offset for virtual address of the frag, and pa + offset for dma address of the frag. This reduces the number of calls to dma_map() by 1/3 for 9k mtu and by 1/20 for 1500 mtu.

__alloc_page_frag is limited to max buffer size of PAGE_SIZE, i.e 4096 in most of the cases. So 9k buffer allocation goes through kmalloc which return page of order 2, 16k. We waste 7k bytes for every 9k buffer.

Quoted 7 lines

we maintain dma_count variable which is incremented when we allocate a frag. enic_unmap_dma will decrement the dma_count and unmap it when there is no user of that page in rx ring.

This reduces the memory utilization for 9k mtu by 33%.

This is a nice optimization, but this is definitely useful for other drivers rather than just your's. And there isn't anything that really keeps this from being put somewhere generically.

Quoted 1 line

+#define ENIC_ALLOC_ORDER PAGE_ALLOC_COSTLY_ORDER

You talk about order(3) but then use PAGE_ALLOC_COSTLY_ORDER, which in theory could change in the future.

But, in any event, there is no reason not to use NETDEV_FRAG_PAGE_MAX_ORDER, just like __alloc_page_frag() does.

drivers/net/ethernet/cisco/enic/enic.h struct enic { · line 207

Quoted 7 lines
drivers/net/ethernet/cisco/enic/enic.h +7 -0
+struct enic_alloc_cache {
+	struct page_frag	frag;
+	unsigned int		pagecnt_bias;
+	int			dma_count;
+	void			*va;
+	dma_addr_t		pa;
+};

Make this a generic structure, perhaps named something like "netdev_dma_alloc_cache".

'pa' is not a good name for a DMA address, because it is not (necessarily) a physical address. It could be a virtual address translated by an IOMMU. "dma_addr" is probably therefore a better member name.

In the generic version the driver will have to pass in a pointer to the "netdev_dma_alloc_cache". I would suggest having this embedded in the driver per-queue structure rather than being allocated dynamically.

Then you can provide a netdev_dma_alloc_cache_init() the driver can call which initializes this embedded object.

Review

Quoted 2 lines
(quoted diff) +2 -0
+	ec->pa = pci_map_single(enic->pdev, ec->va, ec->frag.size,
+				PCI_DMA_FROMDEVICE);

Next, these need to be converted to dma_*() calls, and the interface for netdev_dma_alloc_cache() will need to have a "struct device *" argument for these calls.

Quoted 9 lines
drivers/net/ethernet/cisco/enic/vnic_rq.c +5 -0
@@ -199,6 +200,18 @@ void vnic_rq_clean(struct vnic_rq *rq,
 		rq->ring.desc_avail++;
 	}
 
+	if (rq->ec) {
+		struct enic *enic = vnic_dev_priv(rq->vdev);
+		struct enic_alloc_cache *ec = rq->ec;
+
+		WARN_ON(ec->dma_count);

drivers/net/ethernet/cisco/enic/vnic_rq.c void vnic_rq_clean(struct vnic_rq *rq, · line 217

Quoted 10 lines
drivers/net/ethernet/cisco/enic/vnic_rq.c +7 -0
+		pci_unmap_single(enic->pdev, ec->pa, ec->frag.size,
+				 PCI_DMA_FROMDEVICE);
+		atomic_sub(ec->pagecnt_bias - 1, &ec->frag.page->_count);
+		__free_pages(ec->frag.page, get_order(ec->frag.size));
+		kfree(ec);
+		rq->ec = NULL;
+	}
 	/* Use current fetch_index as the ring starting point */
 	fetch_index = ioread32(&rq->ctrl->fetch_index);
 

Finally, you'll need to define a "netdev_dma_alloc_cache_destroy()" function which you'll call from here.

#3 lore raw

PATCH v2 2/2 enic: Add rq allocation failure stats

In reply to Govindarajulu Varadarajan

This patch adds rq buff allocation failure stats.

cache_alloc_err: incremented when higher order page allocation fails.

enic_rq_alloc_buf: incremented rq buff fails. Either due to page alloc
failure or build_skb.

---
 drivers/net/ethernet/cisco/enic/enic_ethtool.c | 2 ++
 drivers/net/ethernet/cisco/enic/enic_main.c    | 3 +++
 drivers/net/ethernet/cisco/enic/vnic_stats.h   | 2 ++
 3 files changed, 7 insertions(+)
drivers/net/ethernet/cisco/enic/enic_ethtool.c +2 -0
index 28d9ca6..c804e6c 100644
@@ -86,6 +86,8 @@ static const struct enic_stat enic_rx_stats[] = {
 
 static const struct enic_stat enic_gen_stats[] = {
 	ENIC_GEN_STAT(dma_map_error),
+	ENIC_GEN_STAT(cache_alloc_err),
+	ENIC_GEN_STAT(rq_alloc_err),
 };
 
 static const unsigned int enic_n_tx_stats = ARRAY_SIZE(enic_tx_stats);
drivers/net/ethernet/cisco/enic/enic_main.c +3 -0
index 841571f..b92bfe4 100644
@@ -962,6 +962,7 @@ struct enic_alloc_cache *enic_page_refill(struct enic *enic, size_t sz,
 		goto no_ec;
 	ec->frag.page = alloc_pages_node(NUMA_NO_NODE, gfp_comp, order);
 	if (unlikely(!ec->frag.page)) {
+		enic->gen_stats.cache_alloc_err++;
 		order = get_order(sz);
 		ec->frag.page = alloc_pages_node(NUMA_NO_NODE, gfp, order);
 		if (!ec->frag.page)
@@ -1105,6 +1106,8 @@ static int enic_rq_alloc_buf(struct vnic_rq *rq)
 	return 0;
 
 alloc_fail:
+	enic->gen_stats.rq_alloc_err++;
+
 	return -ENOMEM;
 }
 
drivers/net/ethernet/cisco/enic/vnic_stats.h +2 -0
index 74c81ed..b2a4528 100644
@@ -65,6 +65,8 @@ struct vnic_rx_stats {
 /* Generic statistics */
 struct vnic_gen_stats {
 	u64 dma_map_error;
+	u64 cache_alloc_err;	/* alloc_pages(enic->order) failures */
+	u64 rq_alloc_err;	/* rq buf + skb alloc failures */
 };
 
 struct vnic_stats {
Signature · 1 line
2.3.0