Add QPL mode for DQO descriptor format
Subsystems (3)
cover Add QPL mode for DQO descriptor format
GVE supports QPL ("queue-page-list") mode where all data is communicated through a set of pre-registered pages. Adding this mode to DQO.
Rushil Gupta (4):
gve: Control path for DQO-QPL
gve: Tx path for QPL
gve: RX path for DQO-QPL
gve: update gve.rst
.../device_drivers/ethernet/google/gve.rst | 9 + drivers/net/ethernet/google/gve/gve.h | 112 ++++- drivers/net/ethernet/google/gve/gve_adminq.c | 93 +++- drivers/net/ethernet/google/gve/gve_adminq.h | 10 + drivers/net/ethernet/google/gve/gve_main.c | 20 +- drivers/net/ethernet/google/gve/gve_rx_dqo.c | 126 +++++- drivers/net/ethernet/google/gve/gve_tx_dqo.c | 404 ++++++++++++++---- 7 files changed, 653 insertions(+), 121 deletions(-)
Signature · 1 line
2.41.0.585.gd2178a4bd4-goog
PATCH 1/4 gve: Control path for DQO-QPL
Add checks, abi-changes and device options to support QPL mode for DQO in addition to GQI. Also, use pages-per-qpl supplied by device-option to control the size of the "queue-page-list".
--- drivers/net/ethernet/google/gve/gve.h | 20 ++++- drivers/net/ethernet/google/gve/gve_adminq.c | 93 +++++++++++++++++--- drivers/net/ethernet/google/gve/gve_adminq.h | 10 +++ drivers/net/ethernet/google/gve/gve_main.c | 20 +++-- 4 files changed, 123 insertions(+), 20 deletions(-)
drivers/net/ethernet/google/gve/gve.h +17 -3
@@ -51,6 +51,12 @@#define GVE_GQ_TX_MIN_PKT_DESC_BYTES 182 +#define DQO_QPL_DEFAULT_TX_PAGES 512 +#define DQO_QPL_DEFAULT_RX_PAGES 2048 + +/* Maximum TSO size supported on DQO */ +#define GVE_DQO_TX_MAX 0x3FFFF +/* Each slot in the desc ring has a 1:1 mapping to a slot in the data ring */ struct gve_rx_desc_queue {struct gve_rx_desc *desc_ring;/* the descriptor ring */
@@ -531,6 +537,7 @@ enum gve_queue_format {GVE_GQI_RDA_FORMAT =0x1 ,GVE_GQI_QPL_FORMAT =0x2 ,GVE_DQO_RDA_FORMAT =0x3 , + GVE_DQO_QPL_FORMAT = 0x4, }; struct gve_priv {
@@ -550,7 +557,8 @@ struct gve_priv {u16 num_event_counters;u16 tx_desc_cnt;/* num desc per ring */ u16 rx_desc_cnt;/* num desc per ring */ -u16 tx_pages_per_qpl;/* tx buffer length */ +u16 tx_pages_per_qpl;/* Suggested number of pages per qpl for TX queues by NIC */ +u16 rx_pages_per_qpl;/* Suggested number of pages per qpl for RX queues by NIC */ u16 rx_data_slot_cnt;/* rx buffer length */ u64 max_registered_pages;u64 num_registered_pages;/* num pages registered with NIC */
@@ -808,11 +816,17 @@ static inline u32 gve_rx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx)return (priv->num_ntfy_blks /2 ) + queue_idx; } +static inline bool gve_is_qpl (struct gve_priv *priv) +{ +return priv->queue_format ==GVE_GQI_QPL_FORMAT || + priv->queue_format ==GVE_DQO_QPL_FORMAT ; +} +/* Returns the number of tx queue page lists */ static inline u32 gve_num_tx_qpls (struct gve_priv *priv) { -if (priv->queue_format !=GVE_GQI_QPL_FORMAT ) +if (!gve_is_qpl (priv))return 0 ;return priv->tx_cfg.num_queues + priv->num_xdp_queues;
@@ -832,7 +846,7 @@ static inline u32 gve_num_xdp_qpls(struct gve_priv *priv)*/ static inline u32 gve_num_rx_qpls (struct gve_priv *priv) { -if (priv->queue_format !=GVE_GQI_QPL_FORMAT ) +if (!gve_is_qpl (priv))return 0 ;return priv->rx_cfg.num_queues;
drivers/net/ethernet/google/gve/gve_adminq.c +82 -11
189 lines not on this page · raw message
drivers/net/ethernet/google/gve/gve_adminq.h +10 -0
@@ -109,6 +109,14 @@ struct gve_device_option_dqo_rda {static_assert (sizeof (struct gve_device_option_dqo_rda ) ==8 ); +struct gve_device_option_dqo_qpl { +__be32 supported_features_mask; +__be16 tx_pages_per_qpl; +__be16 rx_pages_per_qpl; +}; + +static_assert (sizeof (struct gve_device_option_dqo_qpl ) ==8 ); +struct gve_device_option_jumbo_frames {__be32 supported_features_mask;__be16 max_mtu;
@@ -130,6 +138,7 @@ enum gve_dev_opt_id {GVE_DEV_OPT_ID_GQI_RDA =0x2 ,GVE_DEV_OPT_ID_GQI_QPL =0x3 ,GVE_DEV_OPT_ID_DQO_RDA =0x4 , +GVE_DEV_OPT_ID_DQO_QPL =0x7 , GVE_DEV_OPT_ID_JUMBO_FRAMES = 0x8, };
@@ -139,6 +148,7 @@ enum gve_dev_opt_req_feat_mask {GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL =0x0 ,GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA =0x0 ,GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES =0x0 , + GVE_DEV_OPT_REQ_FEAT_MASK_DQO_QPL = 0x0, }; enum gve_sup_feature_mask {
drivers/net/ethernet/google/gve/gve_main.c +14 -6
@@ -31,7 +31,6 @@// Minimum amount of time between queue kicks in msec (10 seconds) #define MIN_TX_TIMEOUT_GAP (1000 * 10) -#define DQO_TX_MAX 0x3FFFFchar gve_driver_name[] ="gve" ;const char gve_version_str[] =GVE_VERSION ;
@@ -494,7 +493,7 @@ static int gve_setup_device_resources(struct gve_priv *priv) goto abort_with_stats_report; } -if (priv->queue_format ==GVE_DQO_RDA_FORMAT ) { +if (!gve_is_gqi (priv)) { priv->ptype_lut_dqo =kvzalloc (sizeof (*priv->ptype_lut_dqo),GFP_KERNEL );if (!priv->ptype_lut_dqo) {
@@ -1085,9 +1084,10 @@ static int gve_alloc_qpls(struct gve_priv *priv)int max_queues = priv->tx_cfg.max_queues + priv->rx_cfg.max_queues;int start_id;int i, j; +int page_count;int err; -if (priv->queue_format !=GVE_GQI_QPL_FORMAT ) +if (!gve_is_qpl (priv))return 0 ; priv->qpls =kvcalloc (max_queues,sizeof (*priv->qpls),GFP_KERNEL );
@@ -1095,17 +1095,25 @@ static int gve_alloc_qpls(struct gve_priv *priv)return -ENOMEM ; start_id =gve_tx_start_qpl_id (priv); + page_count = priv->tx_pages_per_qpl;for (i = start_id; i < start_id +gve_num_tx_qpls (priv); i++) { err =gve_alloc_queue_page_list (priv, i, - priv->tx_pages_per_qpl); + page_count);if (err) goto free_qpls; } start_id =gve_rx_start_qpl_id (priv); + +/* For GQI_QPL number of pages allocated have 1:1 relationship with +* number of descriptors. For DQO, number of pages required are +* more than descriptors (because of out of order completions). +*/ + page_count = priv->queue_format ==GVE_GQI_QPL_FORMAT ? + priv->rx_data_slot_cnt : priv->rx_pages_per_qpl;for (i = start_id; i < start_id +gve_num_rx_qpls (priv); i++) { err =gve_alloc_queue_page_list (priv, i, - priv->rx_data_slot_cnt); + page_count);if (err) goto free_qpls; }
@@ -2051,7 +2059,7 @@ static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)/* Big TCP is only supported on DQ*/ if (!gve_is_gqi (priv)) -netif_set_tso_max_size (priv->dev,DQO_TX_MAX ); +netif_set_tso_max_size (priv->dev,GVE_DQO_TX_MAX ); priv->num_registered_pages =0 ; priv->rx_copybreak =GVE_DEFAULT_RX_COPYBREAK ;
Signature · 1 line
2.41.0.585.gd2178a4bd4-goog
PATCH 2/4 gve: Tx path for QPL
In reply to Rushil GuptaEach QPL page is divided into GVE_TX_BUFS_PER_PAGE_DQO buffers. When a packet needs to be transmitted, we break the packet into max GVE_TX_BUF_SIZE_DQO sized chunks and transmit each chunk using a TX descriptor.
We allocate the TX buffers from the free list in dqo_tx. We store these TX buffer indices in an array in the pending_packet structure.
The TX buffers are returned to the free list in dqo_compl after receiving packet completion or when removing packets from miss completions list.
--- drivers/net/ethernet/google/gve/gve.h | 83 +++- drivers/net/ethernet/google/gve/gve_tx_dqo.c | 404 +++++++++++++++---- 2 files changed, 404 insertions(+), 83 deletions(-)
drivers/net/ethernet/google/gve/gve.h +81 -2
@@ -57,6 +57,20 @@/* Maximum TSO size supported on DQO */ #define GVE_DQO_TX_MAX 0x3FFFF +#define GVE_TX_BUF_SHIFT_DQO 11 + +/* 2K buffers for DQO-QPL */ +#define GVE_TX_BUF_SIZE_DQO BIT(GVE_TX_BUF_SHIFT_DQO) +#define GVE_TX_BUFS_PER_PAGE_DQO (PAGE_SIZE >> GVE_TX_BUF_SHIFT_DQO) +#define GVE_MAX_TX_BUFS_PER_PKT (DIV_ROUND_UP(GVE_DQO_TX_MAX, GVE_TX_BUF_SIZE_DQO)) + +/* If number of free/recyclable buffers are less than this threshold; driver +* allocs and uses a non-qpl page on the receive path of DQO QPL to free +* up buffers. +* Value is set big enough to post at least 3 64K LRO packet via 2K buffer to NIC. +*/ +#define GVE_DQO_QPL_ONDEMAND_ALLOC_THRESHOLD 96 +/* Each slot in the desc ring has a 1:1 mapping to a slot in the data ring */ struct gve_rx_desc_queue {struct gve_rx_desc *desc_ring;/* the descriptor ring */
@@ -334,8 +348,14 @@ struct gve_tx_pending_packet_dqo {* All others correspond to `skb`'s frags and should be unmapped with * `dma_unmap_page`. */ -DEFINE_DMA_UNMAP_ADDR (dma[MAX_SKB_FRAGS +1 ]); -DEFINE_DMA_UNMAP_LEN (len[MAX_SKB_FRAGS +1 ]); +union { +struct { +DEFINE_DMA_UNMAP_ADDR (dma[MAX_SKB_FRAGS +1 ]); +DEFINE_DMA_UNMAP_LEN (len[MAX_SKB_FRAGS +1 ]); + }; +s16 tx_qpl_buf_ids[GVE_MAX_TX_BUFS_PER_PKT ]; + }; +u16 num_bufs;/* Linked list index to next element in the list, or -1 if none */
@@ -390,6 +410,32 @@ struct gve_tx_ring {* set. */ u32 last_re_idx; + +/* free running number of packet buf descriptors posted */ +u16 posted_packet_desc_cnt; +/* free running number of packet buf descriptors completed */ +u16 completed_packet_desc_cnt; + +/* QPL fields */ +struct { +/* Linked list of gve_tx_buf_dqo. Index into +* tx_qpl_buf_next, or -1 if empty. +* +* This is a consumer list owned by the TX path. When it +* runs out, the producer list is stolen from the +* completion handling path +* (dqo_compl.free_tx_qpl_buf_head). +*/ +s16 free_tx_qpl_buf_head; + +/* Free running count of the number of QPL tx buffers +* allocated +*/ +u32 alloc_tx_qpl_buf_cnt; + +/* Cached value of `dqo_compl.free_tx_qpl_buf_cnt` */ +u32 free_tx_qpl_buf_cnt; + }; } dqo_tx; };
@@ -433,6 +479,24 @@ struct gve_tx_ring {* reached a specified timeout. */ struct gve_index_list timed_out_completions; + +/* QPL fields */ +struct { +/* Linked list of gve_tx_buf_dqo. Index into +* tx_qpl_buf_next, or -1 if empty. +* +* This is the producer list, owned by the completion +* handling path. When the consumer list +* (dqo_tx.free_tx_qpl_buf_head) is runs out, this list +* will be stolen. +*/ +atomic_t free_tx_qpl_buf_head; + +/* Free running count of the number of tx buffers +* freed +*/ +atomic_t free_tx_qpl_buf_cnt; + }; } dqo_compl; }____cacheline_aligned ;u64 pkt_done;/* free-running - total packets completed */
@@ -459,6 +523,21 @@ struct gve_tx_ring {s16 num_pending_packets;u32 complq_mask;/* complq size is complq_mask + 1 */ + +/* QPL fields */ +struct { +/* qpl assigned to this queue */ +struct gve_queue_page_list *qpl; + +/* Each QPL page is divided into TX bounce buffers +* of size GVE_TX_BUF_SIZE_DQO. tx_qpl_buf_next is +* an array to manage linked lists of TX buffers. +* An entry j at index i implies that j'th buffer +* is next on the list after i +*/ +s16 *tx_qpl_buf_next; +u32 num_tx_qpl_bufs; + }; } dqo; }____cacheline_aligned ;struct netdev_queue *netdev_txq;
drivers/net/ethernet/google/gve/gve_tx_dqo.c +323 -81
527 lines not on this page · raw message
Signature · 1 line
2.41.0.585.gd2178a4bd4-goog
PATCH 3/4 gve: RX path for DQO-QPL
In reply to Rushil GuptaThe RX path allocates the QPL page pool at queue creation, and tries to reuse these pages through page recycling. This patch ensures that on refill no non-QPL pages are posted to the device.
To avoid the scenario where driver is running low on used buffers we added an ondemand allocation step where the driver allocates an internal page for SKB business to free up the QPL page in use.
gve_try_recycle_buf was moved to gve_rx_append_frags so that driver does not attempt to mark buffer as used if a non-qpl page was allocated ondemand.
--- drivers/net/ethernet/google/gve/gve.h | 9 ++ drivers/net/ethernet/google/gve/gve_rx_dqo.c | 126 ++++++++++++++++--- 2 files changed, 117 insertions(+), 18 deletions(-)
drivers/net/ethernet/google/gve/gve.h +9 -0
@@ -237,6 +237,15 @@ struct gve_rx_ring {* which cannot be reused yet. */ struct gve_index_list used_buf_states; + +/* index into queue page list */ +u32 next_qpl_page_idx; + +/* qpl assigned to this queue */ +struct gve_queue_page_list *qpl; + +/* track number of used buffers */ +u16 used_buf_states_cnt; } dqo; };
drivers/net/ethernet/google/gve/gve_rx_dqo.c +108 -18
228 lines not on this page · raw message
Signature · 1 line
2.41.0.585.gd2178a4bd4-goog
Jakub Kicinski
maintainerQuoted 3 lines · On Tue, 1 Aug 2023 21:54:02 +0000 Rushil Gupta wrote:
GVE supports QPL ("queue-page-list") mode where all data is communicated through a set of pre-registered pages. Adding this mode to DQO.
In case patch 4/4 does not make it to the list I'd like to remind you of the mandatory 24h wait period between postings. Sadly it does not include patches not CCed to the list...
Thanks for catching that. I'll update the patch after 24h.
Quoted 9 lines · Jakub Kicinski
Quoted 3 lines · On Tue, 1 Aug 2023 21:54:02 +0000 Rushil Gupta wrote:
GVE supports QPL ("queue-page-list") mode where all data is communicated through a set of pre-registered pages. Adding this mode to DQO.
In case patch 4/4 does not make it to the list I'd like to remind you of the mandatory 24h wait period between postings. Sadly it does not include patches not CCed to the list...