gve: Reduce alloc and copy costs in the GQ rx path
Subsystems (3)
PATCH 1/1 gve: Reduce alloc and copy costs in the GQ rx path
Previously, even if just one of the many fragments of a 9k packet required a copy, we'd copy the whole packet into a freshly-allocated 9k-sized linear SKB, and this led to performance issues.
By having a pool of pages to copy into, each fragment can be independently handled, leading to a reduced incidence of allocation and copy.
--- drivers/net/ethernet/google/gve/gve.h | 26 +- drivers/net/ethernet/google/gve/gve_ethtool.c | 2 + drivers/net/ethernet/google/gve/gve_rx.c | 542 ++++++++++-------- drivers/net/ethernet/google/gve/gve_rx_dqo.c | 2 +- drivers/net/ethernet/google/gve/gve_utils.c | 30 +- drivers/net/ethernet/google/gve/gve_utils.h | 2 +- 6 files changed, 338 insertions(+), 266 deletions(-)
drivers/net/ethernet/google/gve/gve.h +20 -6
@@ -60,7 +60,8 @@ struct gve_rx_slot_page_info {void *page_address;u32 page_offset;/* offset to write to in page */ int pagecnt_bias;/* expected pagecnt if only the driver has a ref */ -u8 can_flip; +u16 pad;/* adjustment for rx padding */ +u8 can_flip;/* tracks if the networking stack is using the page */ };/* A list of pages registered with the device during setup and used by a queue
@@ -149,10 +150,17 @@ struct gve_rx_ctx {/* head and tail of skb chain for the current packet or NULL if none */ struct sk_buff *skb_head;struct sk_buff *skb_tail; -u16 total_expected_size; -u8 expected_frag_cnt; -u8 curr_frag_cnt; -u8 reuse_frags; +u32 total_size; +u8 frag_cnt; +bool drop_pkt; +}; + +struct gve_rx_cnts { +u32 ok_pkt_bytes; +u16 ok_pkt_cnt; +u16 total_pkt_cnt; +u16 cont_pkt_cnt; +u16 desc_err_pkt_cnt; };/* Contains datapath state used to represent an RX queue. */
@@ -167,6 +175,10 @@ struct gve_rx_ring {/* threshold for posting new buffs and descs */ u32 db_threshold;u16 packet_buffer_size; + +u32 qpl_copy_pool_mask; +u32 qpl_copy_pool_head; +struct gve_rx_slot_page_info *qpl_copy_pool; };/* DQO fields. */
@@ -216,7 +228,9 @@ struct gve_rx_ring {u64 rx_desc_err_dropped_pkt;/* free-running count of packets dropped by descriptor error */ u64 rx_cont_packet_cnt;/* free-running multi-fragment packets received */ u64 rx_frag_flip_cnt;/* free-running count of rx segments where page_flip was used */ -u64 rx_frag_copy_cnt;/* free-running count of rx segments copied into skb linear portion */ +u64 rx_frag_copy_cnt;/* free-running count of rx segments copied */ +u64 rx_frag_alloc_cnt;/* free-running count of rx page allocations */ +u32 q_num;/* queue index */ u32 ntfy_id;/* notification block index */ struct gve_queue_resources *q_resources;/* head and tail pointer idx */
drivers/net/ethernet/google/gve/gve_ethtool.c +2 -0
@@ -45,6 +45,7 @@ static const char gve_gstrings_main_stats[][ETH_GSTRING_LEN] = {static const char gve_gstrings_rx_stats[][ETH_GSTRING_LEN ] = {"rx_posted_desc[%u]" ,"rx_completed_desc[%u]" ,"rx_consumed_desc[%u]" ,"rx_bytes[%u]" ,"rx_cont_packet_cnt[%u]" ,"rx_frag_flip_cnt[%u]" ,"rx_frag_copy_cnt[%u]" , +"rx_frag_alloc_cnt[%u]" ,"rx_dropped_pkt[%u]" ,"rx_copybreak_pkt[%u]" ,"rx_copied_pkt[%u]" ,"rx_queue_drop_cnt[%u]" ,"rx_no_buffers_posted[%u]" ,"rx_drops_packet_over_mru[%u]" ,"rx_drops_invalid_checksum[%u]" ,
@@ -271,6 +272,7 @@ gve_get_ethtool_stats(struct net_device *netdev, data[i++] = rx->rx_cont_packet_cnt; data[i++] = rx->rx_frag_flip_cnt; data[i++] = rx->rx_frag_copy_cnt; + data[i++] = rx->rx_frag_alloc_cnt;/* rx dropped packets */ data[i++] = tmp_rx_skb_alloc_fail + tmp_rx_buf_alloc_fail +
drivers/net/ethernet/google/gve/gve_rx.c +307 -235
700 lines not on this page · raw message
drivers/net/ethernet/google/gve/gve_rx_dqo.c +1 -1
@@ -568,7 +568,7 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx,if (eop && buf_len <= priv->rx_copybreak) { rx->ctx.skb_head =gve_rx_copy (priv->dev, napi, - &buf_state->page_info, buf_len,0 ,NULL ); + &buf_state->page_info, buf_len,0 );if (unlikely (!rx->ctx.skb_head)) goto error; rx->ctx.skb_tail = rx->ctx.skb_head;
drivers/net/ethernet/google/gve/gve_utils.c +7 -23
@@ -50,34 +50,18 @@ void gve_rx_add_to_block(struct gve_priv *priv, int queue_idx)struct sk_buff *gve_rx_copy (struct net_device *dev,struct napi_struct *napi,struct gve_rx_slot_page_info *page_info,u16 len, -u16 padding,struct gve_rx_ctx *ctx) +u16 padding) {void *va = page_info->page_address + padding + page_info->page_offset; -int skb_linear_offset =0 ; -bool set_protocol = false;struct sk_buff *skb; -if (ctx) { -if (!ctx->skb_head) - ctx->skb_head =napi_alloc_skb (napi, ctx->total_expected_size); - -if (unlikely (!ctx->skb_head)) -return NULL ; - skb = ctx->skb_head; - skb_linear_offset = skb->len; - set_protocol = ctx->curr_frag_cnt == ctx->expected_frag_cnt -1 ; - }else { - skb =napi_alloc_skb (napi, len); - -if (unlikely (!skb)) -return NULL ; - set_protocol = true; - } -__skb_put (skb, len); -skb_copy_to_linear_data_offset (skb, skb_linear_offset, va, len); + skb =napi_alloc_skb (napi, len); +if (unlikely (!skb)) +return NULL ; -if (set_protocol) - skb->protocol =eth_type_trans (skb, dev); +__skb_put (skb, len); +skb_copy_to_linear_data_offset (skb,0 , va, len); + skb->protocol =eth_type_trans (skb, dev);return skb; }
drivers/net/ethernet/google/gve/gve_utils.h +1 -1
@@ -19,7 +19,7 @@ void gve_rx_add_to_block(struct gve_priv *priv, int queue_idx);struct sk_buff *gve_rx_copy (struct net_device *dev,struct napi_struct *napi,struct gve_rx_slot_page_info *page_info,u16 len, -u16 pad,struct gve_rx_ctx *ctx); +u16 pad);/* Decrement pagecnt_bias. Set it back to INT_MAX if it reached zero. */ void gve_dec_pagecnt_bias (struct gve_rx_slot_page_info *page_info);
Signature · 1 line
2.38.1.273.g43a17bfeac-goog
This patch was applied to netdev/net-next.git (master) by David S. Miller <[email protected]>:
Hello:
This patch was applied to netdev/net-next.git (master) by David S. Miller <[email protected]>:
Quoted 9 lines · On Sat, 29 Oct 2022 09:53:22 -0700 you wrote:
Previously, even if just one of the many fragments of a 9k packet required a copy, we'd copy the whole packet into a freshly-allocated 9k-sized linear SKB, and this led to performance issues.
By having a pool of pages to copy into, each fragment can be independently handled, leading to a reduced incidence of allocation and copy.
[...]
Here is the summary with links:
- [net-next] gve: Reduce alloc and copy costs in the GQ rx path
https://git.kernel.org/netdev/net-next/c/82fd151d38d9
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html