sfc: TXQ refactor
Subsystems (3)
v2 cover sfc: TXQ refactor
Refactor and unify partner-TXQ handling in the EF100 and legacy drivers.
The main thrust of this series is to remove from the legacy (Siena/EF10)
driver the assumption that a netdev TX queue has precisely two hardware TXQs (checksummed and unchecksummed) associated with it, so that in future we can have more (e.g. for handling inner-header checksums) or fewer (e.g. to free up hardware queues for XDP usage).
Changes from v1:
* better explain patch #1 in the commit message, and rename xmit_more_available to xmit_pending
* add new patch #2 applying the same approach to ef100, for consistency
Edward Cree (6):
sfc: add and use efx_tx_send_pending in tx.c
sfc: make ef100 xmit_more handling look more like ef10's
sfc: use tx_queue->old_read_count in EF100 TX path
sfc: use efx_channel_tx_[old_]fill_level() in Siena/EF10 TX datapath
sfc: rewrite efx_tx_may_pio
sfc: remove efx_tx_queue_partner
drivers/net/ethernet/sfc/ef10.c | 2 +- drivers/net/ethernet/sfc/ef100_tx.c | 34 +++++---- drivers/net/ethernet/sfc/ef100_tx.h | 1 - drivers/net/ethernet/sfc/farch.c | 2 +- drivers/net/ethernet/sfc/net_driver.h | 22 ++++-- drivers/net/ethernet/sfc/nic_common.h | 40 +---------- drivers/net/ethernet/sfc/tx.c | 99 +++++++++++++++++---------- drivers/net/ethernet/sfc/tx_common.c | 9 +-- 8 files changed, 104 insertions(+), 105 deletions(-)
PATCH v2 1/6 sfc: add and use efx_tx_send_pending in tx.c
Instead of using efx_tx_queue_partner(), which relies on the assumption
that tx_queues_per_channel is 2, efx_tx_send_pending() iterates over txqs with efx_for_each_channel_tx_queue().
We unconditionally set tx_queue->xmit_pending (renamed from
xmit_more_available), then condition on xmit_more for the call to efx_tx_send_pending(), which will clear xmit_pending. Thus, after an xmit_more TX, the doorbell is un-rung and xmit_pending is true.
--- drivers/net/ethernet/sfc/ef10.c | 2 +- drivers/net/ethernet/sfc/ef100_tx.c | 14 +++---- drivers/net/ethernet/sfc/farch.c | 2 +- drivers/net/ethernet/sfc/net_driver.h | 4 +- drivers/net/ethernet/sfc/tx.c | 59 ++++++++++++++------------- drivers/net/ethernet/sfc/tx_common.c | 4 +- 6 files changed, 44 insertions(+), 41 deletions(-)
drivers/net/ethernet/sfc/ef10.c +1 -1
@@ -2367,7 +2367,7 @@ static void efx_ef10_tx_write(struct efx_tx_queue *tx_queue)unsigned int write_ptr;efx_qword_t *txd; - tx_queue->xmit_more_available = false; + tx_queue->xmit_pending = false;if (unlikely (tx_queue->write_count == tx_queue->insert_count))return ;
drivers/net/ethernet/sfc/ef100_tx.c +7 -7
@@ -131,7 +131,7 @@ void ef100_notify_tx_desc(struct efx_tx_queue *tx_queue)efx_writed_page (tx_queue->efx, ®,ER_GZ_TX_RING_DOORBELL , tx_queue->queue); tx_queue->notify_count = tx_queue->write_count; - tx_queue->xmit_more_available = false; + tx_queue->xmit_pending = false; }static void ef100_tx_push_buffers (struct efx_tx_queue *tx_queue)
@@ -373,14 +373,14 @@ int ef100_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb) }if (__netdev_tx_sent_queue (tx_queue->core_txq, skb->len, xmit_more)) - tx_queue->xmit_more_available = false;/* push doorbell */ + tx_queue->xmit_pending = false;/* push doorbell */ else if (tx_queue->write_count - tx_queue->notify_count >255 )/* Ensure we never push more than 256 packets at once */ - tx_queue->xmit_more_available = false;/* push */ + tx_queue->xmit_pending = false;/* push */ else - tx_queue->xmit_more_available = true;/* don't push yet */ + tx_queue->xmit_pending = true;/* don't push yet */ -if (!tx_queue->xmit_more_available) +if (!tx_queue->xmit_pending)ef100_tx_push_buffers (tx_queue);if (segments) {
@@ -400,9 +400,9 @@ int ef100_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb)/* If we're not expecting another transmit and we had something to push * on this queue then we need to push here to get the previous packets * out. We only enter this branch from before the 'Update BQL' section -* above, so xmit_more_available still refers to the old state. +* above, so xmit_pending still refers to the old state. */ -if (tx_queue->xmit_more_available && !xmit_more) +if (tx_queue->xmit_pending && !xmit_more)ef100_tx_push_buffers (tx_queue);return rc; }
drivers/net/ethernet/sfc/farch.c +1 -1
@@ -320,7 +320,7 @@ void efx_farch_tx_write(struct efx_tx_queue *tx_queue)unsigned write_ptr;unsigned old_write_count = tx_queue->write_count; - tx_queue->xmit_more_available = false; + tx_queue->xmit_pending = false;if (unlikely (tx_queue->write_count == tx_queue->insert_count))return ;
drivers/net/ethernet/sfc/net_driver.h +2 -2
@@ -244,7 +244,7 @@ struct efx_tx_buffer {* @tso_fallbacks: Number of times TSO fallback used * @pushes: Number of times the TX push feature has been used * @pio_packets: Number of times the TX PIO feature has been used -* @xmit_more_available: Are any packets waiting to be pushed to the NIC +* @xmit_pending: Are any packets waiting to be pushed to the NIC * @cb_packets: Number of times the TX copybreak feature has been used * @notify_count: Count of notified descriptors to the NIC * @empty_read_count: If the completion path has seen the queue as empty
@@ -292,7 +292,7 @@ struct efx_tx_queue {unsigned int tso_fallbacks;unsigned int pushes;unsigned int pio_packets; -bool xmit_more_available; +bool xmit_pending;unsigned int cb_packets;unsigned int notify_count;/* Statistics to supplement MAC stats */
drivers/net/ethernet/sfc/tx.c +31 -28
90 lines not on this page · raw message
drivers/net/ethernet/sfc/tx_common.c +2 -2
@@ -78,7 +78,7 @@ void efx_init_tx_queue(struct efx_tx_queue *tx_queue) tx_queue->read_count =0 ; tx_queue->old_read_count =0 ; tx_queue->empty_read_count =0 |EFX_EMPTY_COUNT_VALID ; - tx_queue->xmit_more_available = false; + tx_queue->xmit_pending = false; tx_queue->timestamping = (efx_ptp_use_mac_tx_timestamps (efx) && tx_queue->channel ==efx_ptp_channel (efx)); tx_queue->completed_timestamp_major =0 ;
@@ -116,7 +116,7 @@ void efx_fini_tx_queue(struct efx_tx_queue *tx_queue) ++tx_queue->read_count; } - tx_queue->xmit_more_available = false; + tx_queue->xmit_pending = false;netdev_tx_reset_queue (tx_queue->core_txq); }
PATCH v2 2/6 sfc: make ef100 xmit_more handling look more like ef10's
In reply to Edward CreeThis should cause no functional change; merely make there only be one
design of xmit_more handling to understand. As with the EF10/Siena version, we set tx_queue->xmit_pending when we queue up a TX, and clear it when we ring the doorbell (in ef100_notify_tx_desc).
While we're at it, make ef100_notify_tx_desc static since nothing
outside of ef100_tx.c uses it.
--- drivers/net/ethernet/sfc/ef100_tx.c | 22 +++++++++++----------- drivers/net/ethernet/sfc/ef100_tx.h | 1 - 2 files changed, 11 insertions(+), 12 deletions(-)
drivers/net/ethernet/sfc/ef100_tx.c +11 -11
@@ -117,11 +117,13 @@ static efx_oword_t *ef100_tx_desc(struct efx_tx_queue *tx_queue, unsigned int inreturn NULL ; } -void ef100_notify_tx_desc (struct efx_tx_queue *tx_queue) +static void ef100_notify_tx_desc (struct efx_tx_queue *tx_queue) {unsigned int write_ptr;efx_dword_t reg; + tx_queue->xmit_pending = false; +if (unlikely (tx_queue->notify_count == tx_queue->write_count))return ;
@@ -131,7 +133,6 @@ void ef100_notify_tx_desc(struct efx_tx_queue *tx_queue)efx_writed_page (tx_queue->efx, ®,ER_GZ_TX_RING_DOORBELL , tx_queue->queue); tx_queue->notify_count = tx_queue->write_count; - tx_queue->xmit_pending = false; }static void ef100_tx_push_buffers (struct efx_tx_queue *tx_queue)
@@ -372,15 +373,14 @@ int ef100_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb)netif_tx_start_queue (tx_queue->core_txq); } -if (__netdev_tx_sent_queue (tx_queue->core_txq, skb->len, xmit_more)) - tx_queue->xmit_pending = false;/* push doorbell */ -else if (tx_queue->write_count - tx_queue->notify_count >255 ) -/* Ensure we never push more than 256 packets at once */ - tx_queue->xmit_pending = false;/* push */ -else - tx_queue->xmit_pending = true;/* don't push yet */ + tx_queue->xmit_pending = true; -if (!tx_queue->xmit_pending) +/* If xmit_more then we don't need to push the doorbell, unless there +* are 256 descriptors already queued in which case we have to push to +* ensure we never push more than 256 at once. +*/ +if (__netdev_tx_sent_queue (tx_queue->core_txq, skb->len, xmit_more) || + tx_queue->write_count - tx_queue->notify_count >255 )ef100_tx_push_buffers (tx_queue);if (segments) {
@@ -399,7 +399,7 @@ int ef100_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb)/* If we're not expecting another transmit and we had something to push * on this queue then we need to push here to get the previous packets -* out. We only enter this branch from before the 'Update BQL' section +* out. We only enter this branch from before the xmit_more handling * above, so xmit_pending still refers to the old state. */ if (tx_queue->xmit_pending && !xmit_more)
drivers/net/ethernet/sfc/ef100_tx.h +0 -1
@@ -17,7 +17,6 @@int ef100_tx_probe (struct efx_tx_queue *tx_queue);void ef100_tx_init (struct efx_tx_queue *tx_queue);void ef100_tx_write (struct efx_tx_queue *tx_queue); -void ef100_notify_tx_desc (struct efx_tx_queue *tx_queue);unsigned int ef100_tx_max_skb_descs (struct efx_nic *efx);void ef100_ev_tx (struct efx_channel *channel,const efx_qword_t *p_event);
PATCH v2 3/6 sfc: use tx_queue->old_read_count in EF100 TX path
In reply to Edward CreeAs in the Siena/EF10 case, it minimises cacheline ping-pong between
the TX and completion paths.
--- drivers/net/ethernet/sfc/ef100_tx.c | 8 ++++++-- drivers/net/ethernet/sfc/net_driver.h | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-)
drivers/net/ethernet/sfc/ef100_tx.c +6 -2
@@ -360,15 +360,19 @@ int ef100_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb) goto err;ef100_tx_make_descriptors (tx_queue, skb, segments); - fill_level =efx_channel_tx_fill_level (tx_queue->channel); + fill_level =efx_channel_tx_old_fill_level (tx_queue->channel);if (fill_level > efx->txq_stop_thresh) { +struct efx_tx_queue *txq2; +netif_tx_stop_queue (tx_queue->core_txq);/* Re-read after a memory barrier in case we've raced with * the completion path. Otherwise there's a danger we'll never * restart the queue if all completions have just happened. */ smp_mb (); - fill_level =efx_channel_tx_fill_level (tx_queue->channel); +efx_for_each_channel_tx_queue (txq2, tx_queue->channel) + txq2->old_read_count =READ_ONCE (txq2->read_count); + fill_level =efx_channel_tx_old_fill_level (tx_queue->channel);if (fill_level < efx->txq_stop_thresh)netif_tx_start_queue (tx_queue->core_txq); }
drivers/net/ethernet/sfc/net_driver.h +14 -0
@@ -1692,6 +1692,20 @@ efx_channel_tx_fill_level(struct efx_channel *channel)return fill_level; } +/* Conservative approximation of efx_channel_tx_fill_level using cached value */ +static inline unsigned int +efx_channel_tx_old_fill_level (struct efx_channel *channel) +{ +struct efx_tx_queue *tx_queue; +unsigned int fill_level =0 ; + +efx_for_each_channel_tx_queue (tx_queue, channel) + fill_level =max (fill_level, + tx_queue->insert_count - tx_queue->old_read_count); + +return fill_level; +} +/* Get all supported features. * If a feature is not fixed, it is present in hw_features. * If a feature is fixed, it does not present in hw_features, but
PATCH v2 4/6 sfc: use efx_channel_tx_[old_]fill_level() in Siena/EF10 TX datapath
In reply to Edward CreeInstead of open-coding the calculation with efx_tx_queue_partner(), use
the functions that iterate over numbers of queues other than 2 with efx_for_each_channel_tx_queue().
--- drivers/net/ethernet/sfc/net_driver.h | 4 ---- drivers/net/ethernet/sfc/tx.c | 14 ++++++-------- drivers/net/ethernet/sfc/tx_common.c | 5 +---- 3 files changed, 7 insertions(+), 16 deletions(-)
drivers/net/ethernet/sfc/net_driver.h +0 -4
@@ -1681,10 +1681,6 @@ efx_channel_tx_fill_level(struct efx_channel *channel)struct efx_tx_queue *tx_queue;unsigned int fill_level =0 ; -/* This function is currently only used by EF100, which maybe -* could do something simpler and just compute the fill level -* of the single TXQ that's really in use. -*/ efx_for_each_channel_tx_queue (tx_queue, channel) fill_level =max (fill_level, tx_queue->insert_count - tx_queue->read_count);
drivers/net/ethernet/sfc/tx.c +6 -8
@@ -59,13 +59,12 @@ u8 *efx_tx_get_copy_buffer_limited(struct efx_tx_queue *tx_queue,static void efx_tx_maybe_stop_queue (struct efx_tx_queue *txq1) { -/* We need to consider both queues that the net core sees as one */ -struct efx_tx_queue *txq2 =efx_tx_queue_partner (txq1); +/* We need to consider all queues that the net core sees as one */ struct efx_nic *efx = txq1->efx; +struct efx_tx_queue *txq2;unsigned int fill_level; - fill_level =max (txq1->insert_count - txq1->old_read_count, - txq2->insert_count - txq2->old_read_count); + fill_level =efx_channel_tx_old_fill_level (txq1->channel);if (likely (fill_level < efx->txq_stop_thresh))return ;
@@ -85,11 +84,10 @@ static void efx_tx_maybe_stop_queue(struct efx_tx_queue *txq1)*/ netif_tx_stop_queue (txq1->core_txq);smp_mb (); - txq1->old_read_count =READ_ONCE (txq1->read_count); - txq2->old_read_count =READ_ONCE (txq2->read_count); +efx_for_each_channel_tx_queue (txq2, txq1->channel) + txq2->old_read_count =READ_ONCE (txq2->read_count); - fill_level =max (txq1->insert_count - txq1->old_read_count, - txq2->insert_count - txq2->old_read_count); + fill_level =efx_channel_tx_old_fill_level (txq1->channel);EFX_WARN_ON_ONCE_PARANOID (fill_level >= efx->txq_entries);if (likely (fill_level < efx->txq_stop_thresh)) {smp_mb ();
drivers/net/ethernet/sfc/tx_common.c +1 -4
@@ -242,7 +242,6 @@ void efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index) {unsigned int fill_level, pkts_compl =0 , bytes_compl =0 ;struct efx_nic *efx = tx_queue->efx; -struct efx_tx_queue *txq2;EFX_WARN_ON_ONCE_PARANOID (index > tx_queue->ptr_mask);
@@ -261,9 +260,7 @@ void efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index)if (unlikely (netif_tx_queue_stopped (tx_queue->core_txq)) &&likely (efx->port_enabled) &&likely (netif_device_present (efx->net_dev))) { - txq2 =efx_tx_queue_partner (tx_queue); - fill_level =max (tx_queue->insert_count - tx_queue->read_count, - txq2->insert_count - txq2->read_count); + fill_level =efx_channel_tx_fill_level (tx_queue->channel);if (fill_level <= efx->txq_wake_thresh)netif_tx_wake_queue (tx_queue->core_txq); }
PATCH v2 5/6 sfc: rewrite efx_tx_may_pio
In reply to Edward CreeUse efx_for_each_channel_tx_queue() rather than efx_tx_queue_partner(). Make some related simplifications of efx_nic_tx_is_empty() to remove
entry points that aren't used.
--- drivers/net/ethernet/sfc/nic_common.h | 30 ++------------------------- drivers/net/ethernet/sfc/tx.c | 26 ++++++++++++++++++++++- 2 files changed, 27 insertions(+), 29 deletions(-)
drivers/net/ethernet/sfc/nic_common.h +2 -28
@@ -65,8 +65,7 @@ efx_tx_desc(struct efx_tx_queue *tx_queue, unsigned int index)/* Report whether this TX queue would be empty for the given write_count. * May return false negative. */ -static inline bool __efx_nic_tx_is_empty (struct efx_tx_queue *tx_queue, -unsigned int write_count) +static inline bool efx_nic_tx_is_empty (struct efx_tx_queue *tx_queue,unsigned int write_count) {unsigned int empty_read_count =READ_ONCE (tx_queue->empty_read_count);
@@ -76,17 +75,6 @@ static inline bool __efx_nic_tx_is_empty(struct efx_tx_queue *tx_queue,return ((empty_read_count ^ write_count) & ~EFX_EMPTY_COUNT_VALID ) ==0 ; } -/* Report whether the NIC considers this TX queue empty, using -* packet_write_count (the write count recorded for the last completable -* doorbell push). May return false negative. EF10 only, which is OK -* because only EF10 supports PIO. -*/ -static inline bool efx_nic_tx_is_empty (struct efx_tx_queue *tx_queue) -{ -EFX_WARN_ON_ONCE_PARANOID (!tx_queue->efx->type->option_descriptors); -return __efx_nic_tx_is_empty (tx_queue, tx_queue->packet_write_count); -} -/* Get partner of a TX queue, seen as part of the same net core queue */ /* XXX is this a thing on EF100? */ static inline struct efx_tx_queue *efx_tx_queue_partner (struct efx_tx_queue *tx_queue)
@@ -97,20 +85,6 @@ static inline struct efx_tx_queue *efx_tx_queue_partner(struct efx_tx_queue *tx_return tx_queue +EFX_TXQ_TYPE_OFFLOAD ; } -/* Decide whether we can use TX PIO, ie. write packet data directly into -* a buffer on the device. This can reduce latency at the expense of -* throughput, so we only do this if both hardware and software TX rings -* are empty. This also ensures that only one packet at a time can be -* using the PIO buffer. -*/ -static inline bool efx_nic_may_tx_pio (struct efx_tx_queue *tx_queue) -{ -struct efx_tx_queue *partner =efx_tx_queue_partner (tx_queue); - -return tx_queue->piobuf &&efx_nic_tx_is_empty (tx_queue) && -efx_nic_tx_is_empty (partner); -} -int efx_enqueue_skb_tso (struct efx_tx_queue *tx_queue,struct sk_buff *skb,bool *data_mapped);
@@ -125,7 +99,7 @@ int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue, struct sk_buff *skb,static inline bool efx_nic_may_push_tx_desc (struct efx_tx_queue *tx_queue,unsigned int write_count) { -bool was_empty =__efx_nic_tx_is_empty (tx_queue, write_count); +bool was_empty =efx_nic_tx_is_empty (tx_queue, write_count); tx_queue->empty_read_count =0 ;return was_empty && tx_queue->write_count - write_count ==1 ;
drivers/net/ethernet/sfc/tx.c +25 -1
@@ -264,6 +264,30 @@ static int efx_enqueue_skb_pio(struct efx_tx_queue *tx_queue, ++tx_queue->insert_count; return 0; } + +/* Decide whether we can use TX PIO, ie. write packet data directly into +* a buffer on the device. This can reduce latency at the expense of +* throughput, so we only do this if both hardware and software TX rings +* are empty, including all queues for the channel. This also ensures that +* only one packet at a time can be using the PIO buffer. If the xmit_more +* flag is set then we don't use this - there'll be another packet along +* shortly and we want to hold off the doorbell. +*/ +static bool efx_tx_may_pio (struct efx_tx_queue *tx_queue) +{ +struct efx_channel *channel = tx_queue->channel; + +if (!tx_queue->piobuf) +return false; + +EFX_WARN_ON_ONCE_PARANOID (!channel->efx->type->option_descriptors); + +efx_for_each_channel_tx_queue (tx_queue, channel) +if (!efx_nic_tx_is_empty (tx_queue, tx_queue->packet_write_count)) +return false; + +return true; +}#endif /* EFX_USE_PIO */ /* Send any pending traffic for a channel. xmit_more is shared across all
@@ -326,7 +350,7 @@ netdev_tx_t __efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb goto err; #ifdef EFX_USE_PIO }else if (skb_len <= efx_piobuf_size && !xmit_more && -efx_nic_may_tx_pio (tx_queue)) { +efx_tx_may_pio (tx_queue)) {/* Use PIO for short packets with an empty queue. */ if (efx_enqueue_skb_pio (tx_queue, skb)) goto err;
PATCH v2 6/6 sfc: remove efx_tx_queue_partner
In reply to Edward CreeAll users of this function are now gone.
--- drivers/net/ethernet/sfc/nic_common.h | 10 ---------- 1 file changed, 10 deletions(-)
drivers/net/ethernet/sfc/nic_common.h +0 -10
@@ -75,16 +75,6 @@ static inline bool efx_nic_tx_is_empty(struct efx_tx_queue *tx_queue, unsigned ireturn ((empty_read_count ^ write_count) & ~EFX_EMPTY_COUNT_VALID ) ==0 ; } -/* Get partner of a TX queue, seen as part of the same net core queue */ -/* XXX is this a thing on EF100? */ -static inline struct efx_tx_queue *efx_tx_queue_partner (struct efx_tx_queue *tx_queue) -{ -if (tx_queue->label &EFX_TXQ_TYPE_OFFLOAD ) -return tx_queue -EFX_TXQ_TYPE_OFFLOAD ; -else -return tx_queue +EFX_TXQ_TYPE_OFFLOAD ; -} -int efx_enqueue_skb_tso (struct efx_tx_queue *tx_queue,struct sk_buff *skb,bool *data_mapped);
Quoted 8 lines · On Thu, 3 Sep 2020 22:30:58 +0100 Edward Cree wrote:
Refactor and unify partner-TXQ handling in the EF100 and legacy drivers.
The main thrust of this series is to remove from the legacy (Siena/EF10)
driver the assumption that a netdev TX queue has precisely two hardware TXQs (checksummed and unchecksummed) associated with it, so that in future we can have more (e.g. for handling inner-header checksums) or fewer (e.g. to free up hardware queues for XDP usage).
Quoted 4 lines
Changes from v1:
* better explain patch #1 in the commit message, and renamexmit_more_availabletoxmit_pending
* add new patch #2 applying the same approach to ef100, for consistency
Applied, thanks!