net: Bluetooth: add TX timestamping for ISO/L2CAP/SCO

This is an automated email and please do not reply to this email.

Dear Submitter,

Thank you for submitting the patches to the linux bluetooth mailing list.
While preparing the CI tests, the patches you submitted couldn't be applied to the current HEAD of the repository.

----- Output -----

error: patch failed: include/linux/skbuff.h:478
error: include/linux/skbuff.h: patch does not apply
error: patch failed: net/core/skbuff.c:5523
error: net/core/skbuff.c: patch does not apply
hint: Use 'git am --show-current-patch' to see the failed patch

Please resolve the issue and submit the patches again.

Add SOF_TIMESTAMPING_TX_COMPLETION, for requesting a software timestamp when hardware reports a packet completed.

Completion tstamp is useful for Bluetooth, as hardware timestamps do not exist in the HCI specification except for ISO packets, and the hardware has a queue where packets may wait. In this case the software SND timestamp only reflects the kernel-side part of the total latency (usually small) and queue length (usually 0 unless HW buffers congested), whereas the completion report time is more informative of the true latency.

It may also be useful in other cases where HW TX timestamps cannot be obtained and user wants to estimate an upper bound to when the TX probably happened.

Signed-off-by Pauli Virtanen <[email protected]>

---

Notes:
v5:
- back to decoupled COMPLETION & SND, like in v3
- BPF reporting not implemented here

 Documentation/networking/timestamping.rst | 8 ++++++++
 include/linux/skbuff.h                    | 7 ++++---
 include/uapi/linux/errqueue.h             | 1 +
 include/uapi/linux/net_tstamp.h           | 6 ++++--
 net/core/skbuff.c                         | 2 ++
 net/ethtool/common.c                      | 1 +
 net/socket.c                              | 3 +++
 7 files changed, 23 insertions(+), 5 deletions(-)

Documentation/networking/timestamping.rst +8 -0

index 61ef9da10e28..b8fef8101176 100644
@@ -140,6 +140,14 @@ SOF_TIMESTAMPING_TX_ACK:
   cumulative acknowledgment. The mechanism ignores SACK and FACK.
   This flag can be enabled via both socket options and control messages.
 
+SOF_TIMESTAMPING_TX_COMPLETION:
+  Request tx timestamps on packet tx completion.  The completion
+  timestamp is generated by the kernel when it receives packet a
+  completion report from the hardware. Hardware may report multiple
+  packets at once, and completion timestamps reflect the timing of the
+  report and not actual tx time. This flag can be enabled via both
+  socket options and control messages.
+
 
 1.3.2 Timestamp Reporting
 ^^^^^^^^^^^^^^^^^^^^^^^^^

include/linux/skbuff.h +4 -3

index cd8294cdc249..b974a277975a 100644
@@ -478,8 +478,8 @@ enum {
        /* device driver is going to provide hardware time stamp */
        SKBTX_IN_PROGRESS = 1 << 2,
 
-       /* reserved */
-       SKBTX_RESERVED = 1 << 3,
+       /* generate software time stamp on packet tx completion */
+       SKBTX_COMPLETION_TSTAMP = 1 << 3,
 
        /* generate wifi status information (where possible) */
        SKBTX_WIFI_STATUS = 1 << 4,

@@ -498,7 +498,8 @@ enum {
 
 #define SKBTX_ANY_SW_TSTAMP    (SKBTX_SW_TSTAMP    | \
                                 SKBTX_SCHED_TSTAMP | \
-                                SKBTX_BPF)
+                                SKBTX_BPF          | \
+                                SKBTX_COMPLETION_TSTAMP)
 #define SKBTX_ANY_TSTAMP       (SKBTX_HW_TSTAMP | \
                                 SKBTX_ANY_SW_TSTAMP)
 

include/uapi/linux/errqueue.h +1 -0

index 3c70e8ac14b8..1ea47309d772 100644
@@ -73,6 +73,7 @@ enum {
        SCM_TSTAMP_SND,         /* driver passed skb to NIC, or HW */
        SCM_TSTAMP_SCHED,       /* data entered the packet scheduler */
        SCM_TSTAMP_ACK,         /* data acknowledged by peer */
+       SCM_TSTAMP_COMPLETION,  /* packet tx completion */
 };
 
 #endif /* _UAPI_LINUX_ERRQUEUE_H */

include/uapi/linux/net_tstamp.h +4 -2

index 55b0ab51096c..383213de612a 100644
@@ -44,8 +44,9 @@ enum {
        SOF_TIMESTAMPING_BIND_PHC = (1 << 15),
        SOF_TIMESTAMPING_OPT_ID_TCP = (1 << 16),
        SOF_TIMESTAMPING_OPT_RX_FILTER = (1 << 17),
+       SOF_TIMESTAMPING_TX_COMPLETION = (1 << 18),
 
-       SOF_TIMESTAMPING_LAST = SOF_TIMESTAMPING_OPT_RX_FILTER,
+       SOF_TIMESTAMPING_LAST = SOF_TIMESTAMPING_TX_COMPLETION,
        SOF_TIMESTAMPING_MASK = (SOF_TIMESTAMPING_LAST - 1) |
                                 SOF_TIMESTAMPING_LAST
 };

@@ -58,7 +59,8 @@ enum {
 #define SOF_TIMESTAMPING_TX_RECORD_MASK        (SOF_TIMESTAMPING_TX_HARDWARE | \
                                         SOF_TIMESTAMPING_TX_SOFTWARE | \
                                         SOF_TIMESTAMPING_TX_SCHED | \
-                                        SOF_TIMESTAMPING_TX_ACK)
+                                        SOF_TIMESTAMPING_TX_ACK | \
+                                        SOF_TIMESTAMPING_TX_COMPLETION)
 
 /**
  * struct so_timestamping - SO_TIMESTAMPING parameter

net/core/skbuff.c +2 -0

index ab8acb737b93..6cbf77bc61fc 100644
@@ -5523,6 +5523,8 @@ static bool skb_tstamp_tx_report_so_timestamping(struct sk_buff *skb,
                                                    SKBTX_SW_TSTAMP);
        case SCM_TSTAMP_ACK:
                return TCP_SKB_CB(skb)->txstamp_ack & TSTAMP_ACK_SK;
+       case SCM_TSTAMP_COMPLETION:
+               return skb_shinfo(skb)->tx_flags & SKBTX_COMPLETION_TSTAMP;
        }
 
        return false;

net/ethtool/common.c +1 -0

index 7e3c16856c1a..0cb6da1f692a 100644
@@ -476,6 +476,7 @@ const char sof_timestamping_names[][ETH_GSTRING_LEN] = {
        [const_ilog2(SOF_TIMESTAMPING_BIND_PHC)]     = "bind-phc",
        [const_ilog2(SOF_TIMESTAMPING_OPT_ID_TCP)]   = "option-id-tcp",
        [const_ilog2(SOF_TIMESTAMPING_OPT_RX_FILTER)] = "option-rx-filter",
+       [const_ilog2(SOF_TIMESTAMPING_TX_COMPLETION)] = "tx-completion",
 };
 static_assert(ARRAY_SIZE(sof_timestamping_names) == __SOF_TIMESTAMPING_CNT);
 

net/socket.c +3 -1

index b64ecf2722e7..e3d879b53278 100644
@@ -689,6 +689,9 @@ void __sock_tx_timestamp(__u32 tsflags, __u8 *tx_flags)
        if (tsflags & SOF_TIMESTAMPING_TX_SCHED)
                flags |= SKBTX_SCHED_TSTAMP;
 
+       if (tsflags & SOF_TIMESTAMPING_TX_COMPLETION)
+               flags |= SKBTX_COMPLETION_TSTAMP;
+
        *tx_flags = flags;
 }
 EXPORT_SYMBOL(__sock_tx_timestamp);
--

2.48.1

Add SOF_TIMESTAMPING_TX_COMPLETION, for requesting a software timestamp when hardware reports a packet completed.

Completion tstamp is useful for Bluetooth, as hardware timestamps do not exist in the HCI specification except for ISO packets, and the hardware has a queue where packets may wait. In this case the software SND timestamp only reflects the kernel-side part of the total latency (usually small) and queue length (usually 0 unless HW buffers congested), whereas the completion report time is more informative of the true latency.

It may also be useful in other cases where HW TX timestamps cannot be obtained and user wants to estimate an upper bound to when the TX probably happened.

Signed-off-by Pauli Virtanen <[email protected]>

---

Notes:
v5:
- back to decoupled COMPLETION & SND, like in v3
- BPF reporting not implemented here

 Documentation/networking/timestamping.rst | 8 ++++++++
 include/linux/skbuff.h                    | 7 ++++---
 include/uapi/linux/errqueue.h             | 1 +
 include/uapi/linux/net_tstamp.h           | 6 ++++--
 net/core/skbuff.c                         | 2 ++
 net/ethtool/common.c                      | 1 +
 net/socket.c                              | 3 +++
 7 files changed, 23 insertions(+), 5 deletions(-)

Documentation/networking/timestamping.rst +0 -0

index 61ef9da10e28..b8fef8101176 100644

Add SOF_TIMESTAMPING_TX_COMPLETION, for requesting a software timestamp when hardware reports a packet completed.

Completion tstamp is useful for Bluetooth, as hardware timestamps do not exist in the HCI specification except for ISO packets, and the hardware has a queue where packets may wait. In this case the software SND timestamp only reflects the kernel-side part of the total latency (usually small) and queue length (usually 0 unless HW buffers congested), whereas the completion report time is more informative of the true latency.

It may also be useful in other cases where HW TX timestamps cannot be obtained and user wants to estimate an upper bound to when the TX probably happened.

Signed-off-by Pauli Virtanen <[email protected]>

---

Notes:
v5:
- back to decoupled COMPLETION & SND, like in v3
- BPF reporting not implemented here

  Documentation/networking/timestamping.rst | 8 ++++++++
  include/linux/skbuff.h                    | 7 ++++---
  include/uapi/linux/errqueue.h             | 1 +
  include/uapi/linux/net_tstamp.h           | 6 ++++--
  net/core/skbuff.c                         | 2 ++
  net/ethtool/common.c                      | 1 +
  net/socket.c                              | 3 +++
  7 files changed, 23 insertions(+), 5 deletions(-)

Documentation/networking/timestamping.rst +0 -0

index 61ef9da10e28..b8fef8101176 100644

Dear Pauli,

Thank you for your patch. Two minor comments, should you resend.

You could make the summary/title a statement:

Add COMPLETION timestamp on packet tx completion

Am 18.03.25 um 20:06 schrieb Pauli Virtanen:

Quoted 37 lines

Add SOF_TIMESTAMPING_TX_COMPLETION, for requesting a software timestamp when hardware reports a packet completed.

Completion tstamp is useful for Bluetooth, as hardware timestamps do not exist in the HCI specification except for ISO packets, and the hardware has a queue where packets may wait. In this case the software SND timestamp only reflects the kernel-side part of the total latency (usually small) and queue length (usually 0 unless HW buffers congested), whereas the completion report time is more informative of the true latency.

It may also be useful in other cases where HW TX timestamps cannot be obtained and user wants to estimate an upper bound to when the TX probably happened.

Signed-off-by Pauli Virtanen <[email protected]>

---

Notes:
v5:
- back to decoupled COMPLETION & SND, like in v3
- BPF reporting not implemented here

  Documentation/networking/timestamping.rst | 8 ++++++++
  include/linux/skbuff.h                    | 7 ++++---
  include/uapi/linux/errqueue.h             | 1 +
  include/uapi/linux/net_tstamp.h           | 6 ++++--
  net/core/skbuff.c                         | 2 ++
  net/ethtool/common.c                      | 1 +
  net/socket.c                              | 3 +++
  7 files changed, 23 insertions(+), 5 deletions(-)

Documentation/networking/timestamping.rst +0 -0

index 61ef9da10e28..b8fef8101176 100644

Documentation/networking/timestamping.rst SOF_TIMESTAMPING_TX_ACK: · line 146

Quoted 7 lines

Documentation/networking/timestamping.rst +4 -0

@@ -140,6 +140,14 @@ SOF_TIMESTAMPING_TX_ACK:
    cumulative acknowledgment. The mechanism ignores SACK and FACK.
    This flag can be enabled via both socket options and control messages.
 
+SOF_TIMESTAMPING_TX_COMPLETION:
+  Request tx timestamps on packet tx completion.  The completion
+  timestamp is generated by the kernel when it receives packet a
+  completion report from the hardware. Hardware may report multiple

… receives packate a completion … sounds strange to me, but I am a non-native speaker.

[…]

Kind regards,

Paul

Hi Pauli, Willem, Jason,

Quoted 62 lines · Paul Menzel

Dear Pauli,

Thank you for your patch. Two minor comments, should you resend.

You could make the summary/title a statement:

Add COMPLETION timestamp on packet tx completion

Am 18.03.25 um 20:06 schrieb Pauli Virtanen:

Quoted 37 lines

Add SOF_TIMESTAMPING_TX_COMPLETION, for requesting a software timestamp when hardware reports a packet completed.

Completion tstamp is useful for Bluetooth, as hardware timestamps do not exist in the HCI specification except for ISO packets, and the hardware has a queue where packets may wait. In this case the software SND timestamp only reflects the kernel-side part of the total latency (usually small) and queue length (usually 0 unless HW buffers congested), whereas the completion report time is more informative of the true latency.

It may also be useful in other cases where HW TX timestamps cannot be obtained and user wants to estimate an upper bound to when the TX probably happened.

Signed-off-by Pauli Virtanen <[email protected]>

---

Notes:
v5:
- back to decoupled COMPLETION & SND, like in v3
- BPF reporting not implemented here

  Documentation/networking/timestamping.rst | 8 ++++++++
  include/linux/skbuff.h                    | 7 ++++---
  include/uapi/linux/errqueue.h             | 1 +
  include/uapi/linux/net_tstamp.h           | 6 ++++--
  net/core/skbuff.c                         | 2 ++
  net/ethtool/common.c                      | 1 +
  net/socket.c                              | 3 +++
  7 files changed, 23 insertions(+), 5 deletions(-)

Documentation/networking/timestamping.rst +0 -0

index 61ef9da10e28..b8fef8101176 100644

Documentation/networking/timestamping.rst SOF_TIMESTAMPING_TX_ACK: · line 146

Quoted 7 lines

Documentation/networking/timestamping.rst +4 -0

@@ -140,6 +140,14 @@ SOF_TIMESTAMPING_TX_ACK:
    cumulative acknowledgment. The mechanism ignores SACK and FACK.
    This flag can be enabled via both socket options and control messages.
 
+SOF_TIMESTAMPING_TX_COMPLETION:
+  Request tx timestamps on packet tx completion.  The completion
+  timestamp is generated by the kernel when it receives packet a
+  completion report from the hardware. Hardware may report multiple

… receives packate a completion … sounds strange to me, but I am a non-native speaker.

[…]

Quoted 3 lines

Kind regards,

Paul

Is v5 considered good enough to be merged into bluetooth-next and can this be send to in this merge window or you think it is best to leave for the next? In my opinion it could go in so we use the RC period to stabilize it.

Hi Pauli, Willem, Jason,

Quoted 62 lines · Paul Menzel

Dear Pauli,

Thank you for your patch. Two minor comments, should you resend.

You could make the summary/title a statement:

Add COMPLETION timestamp on packet tx completion

Am 18.03.25 um 20:06 schrieb Pauli Virtanen:

Quoted 37 lines

Add SOF_TIMESTAMPING_TX_COMPLETION, for requesting a software timestamp when hardware reports a packet completed.

Completion tstamp is useful for Bluetooth, as hardware timestamps do not exist in the HCI specification except for ISO packets, and the hardware has a queue where packets may wait. In this case the software SND timestamp only reflects the kernel-side part of the total latency (usually small) and queue length (usually 0 unless HW buffers congested), whereas the completion report time is more informative of the true latency.

It may also be useful in other cases where HW TX timestamps cannot be obtained and user wants to estimate an upper bound to when the TX probably happened.

Signed-off-by Pauli Virtanen <[email protected]>

---

Notes:
v5:
- back to decoupled COMPLETION & SND, like in v3
- BPF reporting not implemented here

  Documentation/networking/timestamping.rst | 8 ++++++++
  include/linux/skbuff.h                    | 7 ++++---
  include/uapi/linux/errqueue.h             | 1 +
  include/uapi/linux/net_tstamp.h           | 6 ++++--
  net/core/skbuff.c                         | 2 ++
  net/ethtool/common.c                      | 1 +
  net/socket.c                              | 3 +++
  7 files changed, 23 insertions(+), 5 deletions(-)

Documentation/networking/timestamping.rst +0 -0

index 61ef9da10e28..b8fef8101176 100644

Documentation/networking/timestamping.rst SOF_TIMESTAMPING_TX_ACK: · line 146

Quoted 7 lines

Documentation/networking/timestamping.rst +4 -0

@@ -140,6 +140,14 @@ SOF_TIMESTAMPING_TX_ACK:
    cumulative acknowledgment. The mechanism ignores SACK and FACK.
    This flag can be enabled via both socket options and control messages.
 
+SOF_TIMESTAMPING_TX_COMPLETION:
+  Request tx timestamps on packet tx completion.  The completion
+  timestamp is generated by the kernel when it receives packet a
+  completion report from the hardware. Hardware may report multiple

… receives packate a completion … sounds strange to me, but I am a non-native speaker.

[…]

Quoted 3 lines

Kind regards,

Paul

Is v5 considered good enough to be merged into bluetooth-next and can this be send to in this merge window or you think it is best to leave for the next? In my opinion it could go in so we use the RC period to stabilize it.

Hi,

to, 2025-03-20 kello 10:43 -0400, Luiz Augusto von Dentz kirjoitti:

Quoted 67 lines

Hi Pauli, Willem, Jason,

Quoted 64 lines · Paul Menzel

Dear Pauli,

Thank you for your patch. Two minor comments, should you resend.

You could make the summary/title a statement:

Add COMPLETION timestamp on packet tx completion

Am 18.03.25 um 20:06 schrieb Pauli Virtanen:

Quoted 37 lines

Add SOF_TIMESTAMPING_TX_COMPLETION, for requesting a software timestamp when hardware reports a packet completed.

Completion tstamp is useful for Bluetooth, as hardware timestamps do not exist in the HCI specification except for ISO packets, and the hardware has a queue where packets may wait. In this case the software SND timestamp only reflects the kernel-side part of the total latency (usually small) and queue length (usually 0 unless HW buffers congested), whereas the completion report time is more informative of the true latency.

It may also be useful in other cases where HW TX timestamps cannot be obtained and user wants to estimate an upper bound to when the TX probably happened.

Signed-off-by Pauli Virtanen <[email protected]>

---

Notes:
v5:
- back to decoupled COMPLETION & SND, like in v3
- BPF reporting not implemented here

  Documentation/networking/timestamping.rst | 8 ++++++++
  include/linux/skbuff.h                    | 7 ++++---
  include/uapi/linux/errqueue.h             | 1 +
  include/uapi/linux/net_tstamp.h           | 6 ++++--
  net/core/skbuff.c                         | 2 ++
  net/ethtool/common.c                      | 1 +
  net/socket.c                              | 3 +++
  7 files changed, 23 insertions(+), 5 deletions(-)

Documentation/networking/timestamping.rst +0 -0

index 61ef9da10e28..b8fef8101176 100644

Documentation/networking/timestamping.rst SOF_TIMESTAMPING_TX_ACK: · line 146

Quoted 7 lines

Documentation/networking/timestamping.rst +4 -0

@@ -140,6 +140,14 @@ SOF_TIMESTAMPING_TX_ACK:
    cumulative acknowledgment. The mechanism ignores SACK and FACK.
    This flag can be enabled via both socket options and control messages.
 
+SOF_TIMESTAMPING_TX_COMPLETION:
+  Request tx timestamps on packet tx completion.  The completion
+  timestamp is generated by the kernel when it receives packet a
+  completion report from the hardware. Hardware may report multiple

… receives packate a completion … sounds strange to me, but I am a non-native speaker.

[…]

Kind regards,

Quoted 6 lines

Quoted 1 line

Paul

Is v5 considered good enough to be merged into bluetooth-next and can this be send to in this merge window or you think it is best to leave for the next? In my opinion it could go in so we use the RC period to stabilize it.

From my side v5 should be good enough, if we want it now.

The remaining things were:

- Typo in documentation

- Better tx_queue implementation: probably not highly important for these use cases, as queues are likely just a few packets so there will be only that amount of non-timestamped skbs cloned.

In future, we might consider emitting SCM_TSTAMP_ACK timestamps eg. for L2CAP LE credit-based flow control, and there this would matter more as you'd need to hang on to the packets for a longer time.

- I'd leave handling of unsupported sockcm fields as it is in v5, as BT sockets have also previously just silently ignored them so no change in behavior here.

- BPF: separate patch series, also need the tests for that

--
Pauli Virtanen

Support enabling TX timestamping for some skbs, and track them until packet completion. Generate software SCM_TSTAMP_COMPLETION when getting completion report from hardware.

Generate software SCM_TSTAMP_SND before sending to driver. Sending from driver requires changes in the driver API, and drivers mostly are going to send the skb immediately.

Make the default situation with no COMPLETION TX timestamping more efficient by only counting packets in the queue when there is nothing to track. When there is something to track, we need to make clones, since the driver may modify sent skbs.

The tx_q queue length is bounded by the hdev flow control, which will not send new packets before it has got completion reports for old ones.

Signed-off-by Pauli Virtanen <[email protected]>

---

Notes:
v5:
- Add hci_sockm_init()
- Back to decoupled COMPLETION & SND, like in v3
- Handle SCO flow controlled case

 include/net/bluetooth/hci_core.h |  20 +++++
 net/bluetooth/hci_conn.c         | 122 +++++++++++++++++++++++++++++++
 net/bluetooth/hci_core.c         |  15 +++-
 net/bluetooth/hci_event.c        |   4 +
 4 files changed, 157 insertions(+), 4 deletions(-)

include/net/bluetooth/hci_core.h +20 -0

index f78e4298e39a..5115da34f881 100644
@@ -261,6 +261,12 @@ struct adv_info {
        struct delayed_work     rpa_expired_cb;
 };
 
+struct tx_queue {
+       struct sk_buff_head queue;
+       unsigned int extra;
+       unsigned int tracked;
+};
+
 #define HCI_MAX_ADV_INSTANCES          5
 #define HCI_DEFAULT_ADV_DURATION       2
 

@@ -733,6 +739,8 @@ struct hci_conn {
        struct sk_buff_head data_q;
        struct list_head chan_list;
 
+       struct tx_queue tx_q;
+
        struct delayed_work disc_work;
        struct delayed_work auto_accept_work;
        struct delayed_work idle_work;

@@ -1572,6 +1580,18 @@ void hci_conn_enter_active_mode(struct hci_conn *conn, __u8 force_active);
 void hci_conn_failed(struct hci_conn *conn, u8 status);
 u8 hci_conn_set_handle(struct hci_conn *conn, u16 handle);
 
+void hci_conn_tx_queue(struct hci_conn *conn, struct sk_buff *skb);
+void hci_conn_tx_dequeue(struct hci_conn *conn);
+void hci_setup_tx_timestamp(struct sk_buff *skb, size_t key_offset,
+                           const struct sockcm_cookie *sockc);
+
+static inline void hci_sockcm_init(struct sockcm_cookie *sockc, struct sock *sk)
+{
+       *sockc = (struct sockcm_cookie) {
+               .tsflags = READ_ONCE(sk->sk_tsflags),
+       };
+}
+
 /*
  * hci_conn_get() and hci_conn_put() are used to control the life-time of an
  * "hci_conn" object. They do not guarantee that the hci_conn object is running,

net/bluetooth/hci_conn.c +33 -0

index d097e308a755..95972fd4c784 100644
@@ -27,6 +27,7 @@
 
 #include <linux/export.h>
 #include <linux/debugfs.h>
+#include <linux/errqueue.h>
 
 #include <net/bluetooth/bluetooth.h>
 #include <net/bluetooth/hci_core.h>

@@ -1002,6 +1003,7 @@ static struct hci_conn *__hci_conn_add(struct hci_dev *hdev, int type, bdaddr_t
        }
 
        skb_queue_head_init(&conn->data_q);
+       skb_queue_head_init(&conn->tx_q.queue);
 
        INIT_LIST_HEAD(&conn->chan_list);
        INIT_LIST_HEAD(&conn->link_list);

@@ -1155,6 +1157,7 @@ void hci_conn_del(struct hci_conn *conn)
        }
 
        skb_queue_purge(&conn->data_q);
+       skb_queue_purge(&conn->tx_q.queue);
 
        /* Remove the connection from the list and cleanup its remaining
         * state. This is a separate function since for some cases like

@@ -3064,3 +3067,122 @@ int hci_abort_conn(struct hci_conn *conn, u8 reason)
         */
        return hci_cmd_sync_run_once(hdev, abort_conn_sync, conn, NULL);
 }
+
+void hci_setup_tx_timestamp(struct sk_buff *skb, size_t key_offset,
+                           const struct sockcm_cookie *sockc)
+{
+       struct sock *sk = skb ? skb->sk : NULL;
+
+       /* This shall be called on a single skb of those generated by user
+        * sendmsg(), and only when the sendmsg() does not return error to
+        * user. This is required for keeping the tskey that increments here in
+        * sync with possible sendmsg() counting by user.
+        *
+        * Stream sockets shall set key_offset to sendmsg() length in bytes
+        * and call with the last fragment, others to 1 and first fragment.
+        */
+
+       if (!skb || !sockc || !sk || !key_offset)
+               return;
+
+       sock_tx_timestamp(sk, sockc, &skb_shinfo(skb)->tx_flags);
+
+       if (sockc->tsflags & SOF_TIMESTAMPING_OPT_ID &&
+           sockc->tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK) {
+               if (sockc->tsflags & SOCKCM_FLAG_TS_OPT_ID) {
+                       skb_shinfo(skb)->tskey = sockc->ts_opt_id;
+               } else {
+                       int key = atomic_add_return(key_offset, &sk->sk_tskey);
+
+                       skb_shinfo(skb)->tskey = key - 1;
+               }
+       }

net/bluetooth/hci_conn.c +63 -0

+
+       /* COMPLETION tstamp is emitted for tracked skb later in Number of
+        * Completed Packets event. Available only for flow controlled cases.
+        *
+        * TODO: SCO support without flowctl (needs to be done in drivers)
+        */
+       switch (conn->type) {
+       case ISO_LINK:
+       case ACL_LINK:
+       case LE_LINK:
+               break;
+       case SCO_LINK:
+       case ESCO_LINK:
+               if (!hci_dev_test_flag(conn->hdev, HCI_SCO_FLOWCTL))
+                       return;
+               break;
+       default:
+               return;
+       }
+
+       if (skb->sk && (skb_shinfo(skb)->tx_flags & SKBTX_COMPLETION_TSTAMP))
+               track = true;
+
+       /* If nothing is tracked, just count extra skbs at the queue head */
+       if (!track && !comp->tracked) {
+               comp->extra++;
+               return;
+       }
+
+       if (track) {
+               skb = skb_clone_sk(skb);
+               if (!skb)
+                       goto count_only;
+
+               comp->tracked++;
+       } else {
+               skb = skb_clone(skb, GFP_KERNEL);
+               if (!skb)
+                       goto count_only;
+       }
+
+       skb_queue_tail(&comp->queue, skb);
+       return;
+
+count_only:
+       /* Stop tracking skbs, and only count. This will not emit timestamps for
+        * the packets, but if we get here something is more seriously wrong.
+        */
+       comp->tracked = 0;
+       comp->extra += skb_queue_len(&comp->queue) + 1;
+       skb_queue_purge(&comp->queue);
+}
+
+void hci_conn_tx_dequeue(struct hci_conn *conn)
+{
+       struct tx_queue *comp = &conn->tx_q;
+       struct sk_buff *skb;
+
+       /* If there are tracked skbs, the counted extra go before dequeuing real
+        * skbs, to keep ordering. When nothing is tracked, the ordering doesn't
+        * matter so dequeue real skbs first to get rid of them ASAP.
+        */
+       if (comp->extra && (comp->tracked || skb_queue_empty(&comp->queue))) {

net/bluetooth/hci_conn.c +4 -0

+       }
+
+       kfree_skb(skb);
+}

net/bluetooth/hci_core.c +11 -4

index 94d9147612da..5eb0600bbd03 100644
@@ -3029,6 +3029,13 @@ static int hci_send_frame(struct hci_dev *hdev, struct sk_buff *skb)
        return 0;
 }
 
+static int hci_send_conn_frame(struct hci_dev *hdev, struct hci_conn *conn,
+                              struct sk_buff *skb)
+{
+       hci_conn_tx_queue(conn, skb);
+       return hci_send_frame(hdev, skb);
+}
+
 /* Send HCI command */
 int hci_send_cmd(struct hci_dev *hdev, __u16 opcode, __u32 plen,
                 const void *param)

@@ -3575,7 +3582,7 @@ static void hci_sched_sco(struct hci_dev *hdev, __u8 type)
        while (*cnt && (conn = hci_low_sent(hdev, type, &quote))) {
                while (quote-- && (skb = skb_dequeue(&conn->data_q))) {
                        BT_DBG("skb %p len %d", skb, skb->len);
-                       hci_send_frame(hdev, skb);
+                       hci_send_conn_frame(hdev, conn, skb);
 
                        conn->sent++;
                        if (conn->sent == ~0)

@@ -3618,7 +3625,7 @@ static void hci_sched_acl_pkt(struct hci_dev *hdev)
                        hci_conn_enter_active_mode(chan->conn,
                                                   bt_cb(skb)->force_active);
 
-                       hci_send_frame(hdev, skb);
+                       hci_send_conn_frame(hdev, chan->conn, skb);
                        hdev->acl_last_tx = jiffies;
 
                        hdev->acl_cnt--;

@@ -3674,7 +3681,7 @@ static void hci_sched_le(struct hci_dev *hdev)
 
                        skb = skb_dequeue(&chan->data_q);
 
-                       hci_send_frame(hdev, skb);
+                       hci_send_conn_frame(hdev, chan->conn, skb);
                        hdev->le_last_tx = jiffies;
 
                        (*cnt)--;

@@ -3708,7 +3715,7 @@ static void hci_sched_iso(struct hci_dev *hdev)
        while (*cnt && (conn = hci_low_sent(hdev, ISO_LINK, &quote))) {
                while (quote-- && (skb = skb_dequeue(&conn->data_q))) {
                        BT_DBG("skb %p len %d", skb, skb->len);
-                       hci_send_frame(hdev, skb);
+                       hci_send_conn_frame(hdev, conn, skb);
 
                        conn->sent++;
                        if (conn->sent == ~0)

net/bluetooth/hci_event.c +4 -1

index 0df4a0e082c8..83990c975c1f 100644
@@ -4415,6 +4415,7 @@ static void hci_num_comp_pkts_evt(struct hci_dev *hdev, void *data,
                struct hci_comp_pkts_info *info = &ev->handles[i];
                struct hci_conn *conn;
                __u16  handle, count;
+               unsigned int i;
 
                handle = __le16_to_cpu(info->handle);
                count  = __le16_to_cpu(info->count);

@@ -4425,6 +4426,9 @@ static void hci_num_comp_pkts_evt(struct hci_dev *hdev, void *data,
 
                conn->sent -= count;
 
+               for (i = 0; i < count; ++i)
+                       hci_conn_tx_dequeue(conn);
+
                switch (conn->type) {
                case ACL_LINK:
                        hdev->acl_cnt += count;
--

2.48.1

Quoted 5 lines · Pauli Virtanen

Support enabling TX timestamping for some skbs, and track them until packet completion. Generate software SCM_TSTAMP_COMPLETION when getting completion report from hardware.

Quoted 8 lines

Generate software SCM_TSTAMP_SND before sending to driver. Sending from driver requires changes in the driver API, and drivers mostly are going to send the skb immediately.

Make the default situation with no COMPLETION TX timestamping more efficient by only counting packets in the queue when there is nothing to track. When there is something to track, we need to make clones, since the driver may modify sent skbs.

Quoted 4 lines

net/bluetooth/hci_conn.c +4 -0

+void hci_conn_tx_queue(struct hci_conn *conn, struct sk_buff *skb)
+{
+       struct tx_queue *comp = &conn->tx_q;
+       bool track = false;

net/bluetooth/hci_conn.c int hci_abort_conn(struct hci_conn *conn, u8 reason) · line 3109

Quoted 4 lines

net/bluetooth/hci_conn.c +4 -0

+
+       /* Emit SND now, ie. just before sending to driver */
+       if (skb_shinfo(skb)->tx_flags & SKBTX_SW_TSTAMP)
+               __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SND);

It's a bit strange that SCM_TSTAMP_SND is under the control of SKBTX_SW_TSTAMP. Can we use the same flag for both lines here directly? I suppose I would use SKBTX_SW_TSTAMP then.

Quoted 33 lines

net/bluetooth/hci_conn.c +32 -0

+
+       /* COMPLETION tstamp is emitted for tracked skb later in Number of
+        * Completed Packets event. Available only for flow controlled cases.
+        *
+        * TODO: SCO support without flowctl (needs to be done in drivers)
+        */
+       switch (conn->type) {
+       case ISO_LINK:
+       case ACL_LINK:
+       case LE_LINK:
+               break;
+       case SCO_LINK:
+       case ESCO_LINK:
+               if (!hci_dev_test_flag(conn->hdev, HCI_SCO_FLOWCTL))
+                       return;
+               break;
+       default:
+               return;
+       }
+
+       if (skb->sk && (skb_shinfo(skb)->tx_flags & SKBTX_COMPLETION_TSTAMP))
+               track = true;
+
+       /* If nothing is tracked, just count extra skbs at the queue head */
+       if (!track && !comp->tracked) {
+               comp->extra++;
+               return;
+       }
+
+       if (track) {
+               skb = skb_clone_sk(skb);
+               if (!skb)

net/bluetooth/hci_conn.c int hci_abort_conn(struct hci_conn *conn, u8 reason) · line 3149

Quoted 8 lines

net/bluetooth/hci_conn.c +8 -0

+                       goto count_only;
+
+               comp->tracked++;
+       } else {
+               skb = skb_clone(skb, GFP_KERNEL);
+               if (!skb)
+                       goto count_only;
+       }

Quoted 10 lines · Jason Xing

Quoted 9 lines · Pauli Virtanen

Support enabling TX timestamping for some skbs, and track them until packet completion. Generate software SCM_TSTAMP_COMPLETION when getting completion report from hardware.

Generate software SCM_TSTAMP_SND before sending to driver. Sending from driver requires changes in the driver API, and drivers mostly are going to send the skb immediately.

Quoted 4 lines

Quoted 4 lines

Make the default situation with no COMPLETION TX timestamping more efficient by only counting packets in the queue when there is nothing to track. When there is something to track, we need to make clones, since the driver may modify sent skbs.

Why count packets at all? And if useful separate from completions, should that be a separate patch?

Quoted 4 lines

Quoted 4 lines

net/bluetooth/hci_conn.c +4 -0

+void hci_conn_tx_queue(struct hci_conn *conn, struct sk_buff *skb)
+{
+       struct tx_queue *comp = &conn->tx_q;
+       bool track = false;

Quoted 8 lines

net/bluetooth/hci_conn.c int hci_abort_conn(struct hci_conn *conn, u8 reason) · line 3109

Quoted 4 lines

net/bluetooth/hci_conn.c +4 -0

+
+       /* Emit SND now, ie. just before sending to driver */
+       if (skb_shinfo(skb)->tx_flags & SKBTX_SW_TSTAMP)
+               __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SND);

It's a bit strange that SCM_TSTAMP_SND is under the control of SKBTX_SW_TSTAMP. Can we use the same flag for both lines here directly? I suppose I would use SKBTX_SW_TSTAMP then.

This is the established behavior.

Quoted 35 lines

Quoted 35 lines

net/bluetooth/hci_conn.c +34 -0

+
+       /* COMPLETION tstamp is emitted for tracked skb later in Number of
+        * Completed Packets event. Available only for flow controlled cases.
+        *
+        * TODO: SCO support without flowctl (needs to be done in drivers)
+        */
+       switch (conn->type) {
+       case ISO_LINK:
+       case ACL_LINK:
+       case LE_LINK:
+               break;
+       case SCO_LINK:
+       case ESCO_LINK:
+               if (!hci_dev_test_flag(conn->hdev, HCI_SCO_FLOWCTL))
+                       return;
+               break;
+       default:
+               return;
+       }
+
+       if (skb->sk && (skb_shinfo(skb)->tx_flags & SKBTX_COMPLETION_TSTAMP))
+               track = true;
+
+       /* If nothing is tracked, just count extra skbs at the queue head */
+       if (!track && !comp->tracked) {
+               comp->extra++;
+               return;
+       }
+
+       if (track) {
+               skb = skb_clone_sk(skb);
+               if (!skb)
+                       goto count_only;
+

Quoted 6 lines

net/bluetooth/hci_conn.c int hci_abort_conn(struct hci_conn *conn, u8 reason) · line 3149

Quoted 6 lines

net/bluetooth/hci_conn.c +6 -0

+               comp->tracked++;
+       } else {
+               skb = skb_clone(skb, GFP_KERNEL);
+               if (!skb)
+                       goto count_only;
+       }

What is the difference between track and comp->tracked

Hi,

ke, 2025-03-19 kello 10:44 -0400, Willem de Bruijn kirjoitti:

Quoted 11 lines

Quoted 10 lines · Jason Xing

Quoted 9 lines · Pauli Virtanen

Support enabling TX timestamping for some skbs, and track them until packet completion. Generate software SCM_TSTAMP_COMPLETION when getting completion report from hardware.

Generate software SCM_TSTAMP_SND before sending to driver. Sending from driver requires changes in the driver API, and drivers mostly are going to send the skb immediately.

Quoted 7 lines

Quoted 4 lines

Quoted 4 lines

Make the default situation with no COMPLETION TX timestamping more efficient by only counting packets in the queue when there is nothing to track. When there is something to track, we need to make clones, since the driver may modify sent skbs.

Why count packets at all? And if useful separate from completions, should that be a separate patch?

This paragraph was commenting on the implementation of struct tx_queue, and maybe how it works should be explicitly explained somewhere (code comment?). Here's some explanation of it:

1) We have to hang on (clones of) skbs until completion reports for them arrive, in order to emit COMPLETION timestamps. There's no existing queue that does this in net/bluetooth (drivers may just copy data & discard skbs, and they don't know about completion reports), so something new needs to be added.

2) It is only needed for emitting COMPLETION timestamps. So it's better to not do any extra work (clones etc.) when there are no such timestamps to be emitted.

3) The new queue should work correctly when timestamping is turned on or off, or only some packets are timestamped. It should also eventually return to a state where no extra work is done, when new skbs don't request COMPLETION timestamps.

struct tx_queue implements such queue that only "tracks" some skbs. Logical structure:

HEAD

<no stored skb>  }
<no stored skb>  }  tx_queue::extra is the number of non-tracked
...              }  logical items at queue head
<no stored skb>  }

<tracked skb> } tx_queue::queue contains mixture of
<non-tracked skb> } tracked items (skb->sk != NULL) and
<non-tracked skb> } non-tracked items (skb->sk == NULL).
<tracked skb> } These are ordered after the "extra" items.
TAIL

tx_queue::tracked is the number of tracked skbs in tx_queue::queue.

hci_conn_tx_queue() determines whether skb is tracked (= COMPLETION timestamp shall be emitted for it) and pushes a logical item to TAIL.

hci_conn_tx_dequeue() pops a logical item from HEAD, and emits timestamp if it corresponds to a tracked skb.

When tracked == 0, queue() can just increment tx_queue::extra, and dequeue() can remove any skb from tx_queue::queue, or if empty then decrement tx_queue::extra. This allows it to return to a state with empty tx_queue::queue when new skbs no longer request timestamps.

When tracked != 0, the ordering of items in the queue needs to be respected strictly, so queue() always pushes real skb (tracked or not) to TAIL, and dequeue() has to decrement extra to zero, before it can pop skb from queue head.

Quoted 30 lines · Pauli Virtanen

ke, 2025-03-19 kello 10:44 -0400, Willem de Bruijn kirjoitti:

Quoted 11 lines

Quoted 10 lines · Jason Xing

Quoted 9 lines · Pauli Virtanen

Support enabling TX timestamping for some skbs, and track them until packet completion. Generate software SCM_TSTAMP_COMPLETION when getting completion report from hardware.

Generate software SCM_TSTAMP_SND before sending to driver. Sending from driver requires changes in the driver API, and drivers mostly are going to send the skb immediately.

Quoted 7 lines

Quoted 4 lines

Quoted 4 lines

Make the default situation with no COMPLETION TX timestamping more efficient by only counting packets in the queue when there is nothing to track. When there is something to track, we need to make clones, since the driver may modify sent skbs.

Why count packets at all? And if useful separate from completions, should that be a separate patch?

This paragraph was commenting on the implementation of struct tx_queue, and maybe how it works should be explicitly explained somewhere (code comment?). Here's some explanation of it:

1) We have to hang on (clones of) skbs until completion reports for them arrive, in order to emit COMPLETION timestamps. There's no existing queue that does this in net/bluetooth (drivers may just copy data & discard skbs, and they don't know about completion reports), so something new needs to be added.

Quoted 8 lines

2) It is only needed for emitting COMPLETION timestamps. So it's better to not do any extra work (clones etc.) when there are no such timestamps to be emitted.

3) The new queue should work correctly when timestamping is turned on or off, or only some packets are timestamped. It should also eventually return to a state where no extra work is done, when new skbs don't request COMPLETION timestamps.

So far, fully understood.

Quoted 27 lines

struct tx_queue implements such queue that only "tracks" some skbs. Logical structure:

HEAD

<no stored skb>  }
<no stored skb>  }  tx_queue::extra is the number of non-tracked
...              }  logical items at queue head
<no stored skb>  }

<tracked skb> } tx_queue::queue contains mixture of
<non-tracked skb> } tracked items (skb->sk != NULL) and
<non-tracked skb> } non-tracked items (skb->sk == NULL).
<tracked skb> } These are ordered after the "extra" items.
TAIL

tx_queue::tracked is the number of tracked skbs in tx_queue::queue.

hci_conn_tx_queue() determines whether skb is tracked (= COMPLETION timestamp shall be emitted for it) and pushes a logical item to TAIL.

hci_conn_tx_dequeue() pops a logical item from HEAD, and emits timestamp if it corresponds to a tracked skb.

When tracked == 0, queue() can just increment tx_queue::extra, and dequeue() can remove any skb from tx_queue::queue, or if empty then decrement tx_queue::extra. This allows it to return to a state with empty tx_queue::queue when new skbs no longer request timestamps.

Quoted 4 lines

When tracked != 0, the ordering of items in the queue needs to be respected strictly, so queue() always pushes real skb (tracked or not) to TAIL, and dequeue() has to decrement extra to zero, before it can pop skb from queue head.

Thanks. I did not understand why you need to queue or track any sbs aside from those that have SKBTX_COMPLETION_TSTAMP.

If I follow correctly this is to be able to associate the tx completion with the right skb on the queue.

The usual model in Ethernet drivers is that every tx descriptor (and completion descriptor) in the ring is associated with a pure software ring of metadata structures, which can point to an skb (or NULL).

In a pinch, instead the skb on the queue itself could record the descriptor id that it is associated with. But hci_conn_tx_queue is too far removed from the HW, so has no direct access to that. And similarly hci_conn_tx_dequeue has no such low level details.

So long story short you indeed have to track this out of band with a separate counter. I also don't immediately see a simpler way.

Though you can perhaps replace the skb_clone (not the skb_clone_sk!) with some sentinel value that just helps count?

Hi,

ke, 2025-03-19 kello 16:00 -0400, Willem de Bruijn kirjoitti:

Quoted 74 lines

Quoted 30 lines · Pauli Virtanen

ke, 2025-03-19 kello 10:44 -0400, Willem de Bruijn kirjoitti:

Quoted 11 lines

Quoted 10 lines · Jason Xing

Quoted 9 lines · Pauli Virtanen

Support enabling TX timestamping for some skbs, and track them until packet completion. Generate software SCM_TSTAMP_COMPLETION when getting completion report from hardware.

Generate software SCM_TSTAMP_SND before sending to driver. Sending from driver requires changes in the driver API, and drivers mostly are going to send the skb immediately.

Quoted 7 lines

Quoted 4 lines

Quoted 4 lines

Make the default situation with no COMPLETION TX timestamping more efficient by only counting packets in the queue when there is nothing to track. When there is something to track, we need to make clones, since the driver may modify sent skbs.

Why count packets at all? And if useful separate from completions, should that be a separate patch?

This paragraph was commenting on the implementation of struct tx_queue, and maybe how it works should be explicitly explained somewhere (code comment?). Here's some explanation of it:

1) We have to hang on (clones of) skbs until completion reports for them arrive, in order to emit COMPLETION timestamps. There's no existing queue that does this in net/bluetooth (drivers may just copy data & discard skbs, and they don't know about completion reports), so something new needs to be added.

Quoted 8 lines

2) It is only needed for emitting COMPLETION timestamps. So it's better to not do any extra work (clones etc.) when there are no such timestamps to be emitted.

3) The new queue should work correctly when timestamping is turned on or off, or only some packets are timestamped. It should also eventually return to a state where no extra work is done, when new skbs don't request COMPLETION timestamps.

So far, fully understood.

Quoted 27 lines

struct tx_queue implements such queue that only "tracks" some skbs. Logical structure:

HEAD

<no stored skb>  }
<no stored skb>  }  tx_queue::extra is the number of non-tracked
...              }  logical items at queue head
<no stored skb>  }

<tracked skb> } tx_queue::queue contains mixture of
<non-tracked skb> } tracked items (skb->sk != NULL) and
<non-tracked skb> } non-tracked items (skb->sk == NULL).
<tracked skb> } These are ordered after the "extra" items.
TAIL

tx_queue::tracked is the number of tracked skbs in tx_queue::queue.

hci_conn_tx_queue() determines whether skb is tracked (= COMPLETION timestamp shall be emitted for it) and pushes a logical item to TAIL.

hci_conn_tx_dequeue() pops a logical item from HEAD, and emits timestamp if it corresponds to a tracked skb.

When tracked == 0, queue() can just increment tx_queue::extra, and dequeue() can remove any skb from tx_queue::queue, or if empty then decrement tx_queue::extra. This allows it to return to a state with empty tx_queue::queue when new skbs no longer request timestamps.

Quoted 4 lines

When tracked != 0, the ordering of items in the queue needs to be respected strictly, so queue() always pushes real skb (tracked or not) to TAIL, and dequeue() has to decrement extra to zero, before it can pop skb from queue head.

Quoted 5 lines

Thanks. I did not understand why you need to queue or track any sbs aside from those that have SKBTX_COMPLETION_TSTAMP.

If I follow correctly this is to be able to associate the tx completion with the right skb on the queue.

Yes, it was done to maintain the queue/dequeue ordering.

Quoted 9 lines

The usual model in Ethernet drivers is that every tx descriptor (and completion descriptor) in the ring is associated with a pure software ring of metadata structures, which can point to an skb (or NULL).

In a pinch, instead the skb on the queue itself could record the descriptor id that it is associated with. But hci_conn_tx_queue is too far removed from the HW, so has no direct access to that. And similarly hci_conn_tx_dequeue has no such low level details.

Quoted 5 lines

So long story short you indeed have to track this out of band with a separate counter. I also don't immediately see a simpler way.

Though you can perhaps replace the skb_clone (not the skb_clone_sk!) with some sentinel value that just helps count?

It probably could be done a bit smarter, it could eg. use something else than skb_queue. Or, I think we can clobber cb here as the clones are only used for timestamping, so:

struct tx_queue {
	unsigned int pre_items;
	struct sk_buff_head queue;
};

struct tx_queue_cb {
	unsigned int post_items;
};

static void hci_tx_queue_push(struct tx_queue *q, struct sk_buff *skb)

{
	struct tx_queue_cb *cb;

	/* HEAD
	 * <non-tracked item>  }
	 * ...                 } tx_queue::pre_items of these
	 * <non-tracked item>  }
	 * <tracked skb1>     <- tx_queue::queue first item
	 * <non-tracked item>  }
	 * ...                 } ((struct tx_queue_cb *)skb1->cb)->post_items
	 * <non-tracked item>  }
	 * ...
	 * <tracked skbn>     <- tx_queue::queue n-th item
	 * <non-tracked item>  }
	 * ...                 } ((struct tx_queue_cb *)skbn->cb)->post_items
	 * <non-tracked item>  }
	 * TAIL
	 */
	if (skb) {
		cb = (struct tx_queue_cb *)skb->cb;
		cb->post_items = 0;
		skb_queue_tail(&q->queue, skb);
	} else {
		skb = skb_peek_tail(&q->queue);
		if (skb) {
			cb = (struct tx_queue_cb *)skb->cb;
			cb->post_items++;
		} else {
			q->pre_items++;
		}
	}
}

static struct sk_buff *hci_tx_queue_pop(struct tx_queue *q)

{
	struct sk_buff *skb;
	struct tx_queue_cb *cb;

	if (q->pre_items) {
		q->pre_items--;
		return NULL;
	}

	skb = skb_dequeue(&q->queue);
	if (skb) {
		cb = (struct tx_queue_cb *)skb->cb;
		q->pre_items += cb->post_items;
	}

	return skb;
}

void hci_conn_tx_queue(struct hci_conn *conn, struct sk_buff *skb)

{
	/* Emit SND now, ie. just before sending to driver */
	if (skb_shinfo(skb)->tx_flags & SKBTX_SW_TSTAMP)
		__skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SND);

	/* COMPLETION tstamp is emitted for tracked skb later in Number of
	 * Completed Packets event. Available only for flow controlled cases.
	 *
	 * TODO: SCO support without flowctl (needs to be done in drivers)
	 */
	switch (conn->type) {
	case ISO_LINK:
	case ACL_LINK:
	case LE_LINK:
		break;
	case SCO_LINK:
	case ESCO_LINK:
		if (!hci_dev_test_flag(conn->hdev, HCI_SCO_FLOWCTL))
			return;
		break;
	default:
		return;
	}

	if (skb->sk && (skb_shinfo(skb)->tx_flags & SKBTX_COMPLETION_TSTAMP))
		skb = skb_clone_sk(skb);
	else
		skb = NULL;

	hci_tx_queue_push(&conn->tx_q, skb);
	return;
}

void hci_conn_tx_dequeue(struct hci_conn *conn)

{
	struct sk_buff *skb = hci_tx_queue_pop(&conn->tx_q);

	if (skb) {
		__skb_tstamp_tx(skb, NULL, NULL, skb->sk,
				SCM_TSTAMP_COMPLETION);
		kfree_skb(skb);
	}
}

Hi,

ke, 2025-03-19 kello 10:44 -0400, Willem de Bruijn kirjoitti:

Quoted 11 lines

Quoted 10 lines · Jason Xing

Quoted 9 lines · Pauli Virtanen

Support enabling TX timestamping for some skbs, and track them until packet completion. Generate software SCM_TSTAMP_COMPLETION when getting completion report from hardware.

Generate software SCM_TSTAMP_SND before sending to driver. Sending from driver requires changes in the driver API, and drivers mostly are going to send the skb immediately.

Quoted 7 lines

Quoted 4 lines

Quoted 4 lines

Make the default situation with no COMPLETION TX timestamping more efficient by only counting packets in the queue when there is nothing to track. When there is something to track, we need to make clones, since the driver may modify sent skbs.

Why count packets at all? And if useful separate from completions, should that be a separate patch?

This paragraph was commenting on the implementation of struct tx_queue, and maybe how it works should be explicitly explained somewhere (code comment?). Here's some explanation of it:

1) We have to hang on (clones of) skbs until completion reports for them arrive, in order to emit COMPLETION timestamps. There's no existing queue that does this in net/bluetooth (drivers may just copy data & discard skbs, and they don't know about completion reports), so something new needs to be added.

2) It is only needed for emitting COMPLETION timestamps. So it's better to not do any extra work (clones etc.) when there are no such timestamps to be emitted.

3) The new queue should work correctly when timestamping is turned on or off, or only some packets are timestamped. It should also eventually return to a state where no extra work is done, when new skbs don't request COMPLETION timestamps.

struct tx_queue implements such queue that only "tracks" some skbs. Logical structure:

HEAD

<no stored skb>  }
<no stored skb>  }  tx_queue::extra is the number of non-tracked
...              }  logical items at queue head
<no stored skb>  }
<tracked skb>           } tx_queue::queue contains mixture of
<non-tracked skb>       } tracked items  (skb->sk != NULL) and
<non-tracked skb>       } non-tracked items  (skb->sk == NULL).
<tracked skb>           } These are ordered after the "extra" items.

TAIL

tx_queue::tracked is the number of tracked skbs in tx_queue::queue.

hci_conn_tx_queue() determines whether skb is tracked (= COMPLETION timestamp shall be emitted for it) and pushes a logical item to TAIL.

hci_conn_tx_dequeue() pops a logical item from HEAD, and emits timestamp if it corresponds to a tracked skb.

When tracked == 0, queue() can just increment tx_queue::extra, and dequeue() can remove any skb from tx_queue::queue, or if empty then decrement tx_queue::extra. This allows it to return to a state with empty tx_queue::queue when new skbs no longer request timestamps.

When tracked != 0, the ordering of items in the queue needs to be respected strictly, so queue() always pushes real skb (tracked or not) to TAIL, and dequeue() has to decrement extra to zero, before it can pop skb from queue head.

Quoted 49 lines

Quoted 4 lines

Quoted 4 lines

net/bluetooth/hci_conn.c +4 -0

+void hci_conn_tx_queue(struct hci_conn *conn, struct sk_buff *skb)
+{
+       struct tx_queue *comp = &conn->tx_q;
+       bool track = false;

Quoted 8 lines

net/bluetooth/hci_conn.c int hci_abort_conn(struct hci_conn *conn, u8 reason) · line 3109

Quoted 4 lines

net/bluetooth/hci_conn.c +4 -0

+
+       /* Emit SND now, ie. just before sending to driver */
+       if (skb_shinfo(skb)->tx_flags & SKBTX_SW_TSTAMP)
+               __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SND);

It's a bit strange that SCM_TSTAMP_SND is under the control of SKBTX_SW_TSTAMP. Can we use the same flag for both lines here directly? I suppose I would use SKBTX_SW_TSTAMP then.

This is the established behavior.

Quoted 35 lines

Quoted 35 lines

net/bluetooth/hci_conn.c +34 -0

+
+       /* COMPLETION tstamp is emitted for tracked skb later in Number of
+        * Completed Packets event. Available only for flow controlled cases.
+        *
+        * TODO: SCO support without flowctl (needs to be done in drivers)
+        */
+       switch (conn->type) {
+       case ISO_LINK:
+       case ACL_LINK:
+       case LE_LINK:
+               break;
+       case SCO_LINK:
+       case ESCO_LINK:
+               if (!hci_dev_test_flag(conn->hdev, HCI_SCO_FLOWCTL))
+                       return;
+               break;
+       default:
+               return;
+       }
+
+       if (skb->sk && (skb_shinfo(skb)->tx_flags & SKBTX_COMPLETION_TSTAMP))
+               track = true;
+
+       /* If nothing is tracked, just count extra skbs at the queue head */
+       if (!track && !comp->tracked) {
+               comp->extra++;
+               return;
+       }
+
+       if (track) {
+               skb = skb_clone_sk(skb);
+               if (!skb)
+                       goto count_only;
+

Quoted 8 lines

Quoted 6 lines

net/bluetooth/hci_conn.c int hci_abort_conn(struct hci_conn *conn, u8 reason) · line 3149

Quoted 6 lines

net/bluetooth/hci_conn.c +6 -0

+               comp->tracked++;
+       } else {
+               skb = skb_clone(skb, GFP_KERNEL);
+               if (!skb)
+                       goto count_only;
+       }

What is the difference between track and comp->tracked

I hope the answer above is clear.

--
Pauli Virtanen

Quoted 145 lines

Support enabling TX timestamping for some skbs, and track them until packet completion. Generate software SCM_TSTAMP_COMPLETION when getting completion report from hardware.

Generate software SCM_TSTAMP_SND before sending to driver. Sending from driver requires changes in the driver API, and drivers mostly are going to send the skb immediately.

Make the default situation with no COMPLETION TX timestamping more efficient by only counting packets in the queue when there is nothing to track. When there is something to track, we need to make clones, since the driver may modify sent skbs.

The tx_q queue length is bounded by the hdev flow control, which will not send new packets before it has got completion reports for old ones.

Signed-off-by Pauli Virtanen <[email protected]>

---

Notes:
v5:
- Add hci_sockm_init()
- Back to decoupled COMPLETION & SND, like in v3
- Handle SCO flow controlled case

 include/net/bluetooth/hci_core.h |  20 +++++
 net/bluetooth/hci_conn.c         | 122 +++++++++++++++++++++++++++++++
 net/bluetooth/hci_core.c         |  15 +++-
 net/bluetooth/hci_event.c        |   4 +
 4 files changed, 157 insertions(+), 4 deletions(-)

include/net/bluetooth/hci_core.h +20 -0

index f78e4298e39a..5115da34f881 100644
@@ -261,6 +261,12 @@ struct adv_info {
        struct delayed_work     rpa_expired_cb;
 };
 
+struct tx_queue {
+       struct sk_buff_head queue;
+       unsigned int extra;
+       unsigned int tracked;
+};
+
 #define HCI_MAX_ADV_INSTANCES          5
 #define HCI_DEFAULT_ADV_DURATION       2
 

@@ -733,6 +739,8 @@ struct hci_conn {
        struct sk_buff_head data_q;
        struct list_head chan_list;
 
+       struct tx_queue tx_q;
+
        struct delayed_work disc_work;
        struct delayed_work auto_accept_work;
        struct delayed_work idle_work;

@@ -1572,6 +1580,18 @@ void hci_conn_enter_active_mode(struct hci_conn *conn, __u8 force_active);
 void hci_conn_failed(struct hci_conn *conn, u8 status);
 u8 hci_conn_set_handle(struct hci_conn *conn, u16 handle);
 
+void hci_conn_tx_queue(struct hci_conn *conn, struct sk_buff *skb);
+void hci_conn_tx_dequeue(struct hci_conn *conn);
+void hci_setup_tx_timestamp(struct sk_buff *skb, size_t key_offset,
+                           const struct sockcm_cookie *sockc);
+
+static inline void hci_sockcm_init(struct sockcm_cookie *sockc, struct sock *sk)
+{
+       *sockc = (struct sockcm_cookie) {
+               .tsflags = READ_ONCE(sk->sk_tsflags),
+       };
+}
+
 /*
  * hci_conn_get() and hci_conn_put() are used to control the life-time of an
  * "hci_conn" object. They do not guarantee that the hci_conn object is running,

net/bluetooth/hci_conn.c +39 -0

index d097e308a755..95972fd4c784 100644
@@ -27,6 +27,7 @@
 
 #include <linux/export.h>
 #include <linux/debugfs.h>
+#include <linux/errqueue.h>
 
 #include <net/bluetooth/bluetooth.h>
 #include <net/bluetooth/hci_core.h>

@@ -1002,6 +1003,7 @@ static struct hci_conn *__hci_conn_add(struct hci_dev *hdev, int type, bdaddr_t
        }
 
        skb_queue_head_init(&conn->data_q);
+       skb_queue_head_init(&conn->tx_q.queue);
 
        INIT_LIST_HEAD(&conn->chan_list);
        INIT_LIST_HEAD(&conn->link_list);

@@ -1155,6 +1157,7 @@ void hci_conn_del(struct hci_conn *conn)
        }
 
        skb_queue_purge(&conn->data_q);
+       skb_queue_purge(&conn->tx_q.queue);
 
        /* Remove the connection from the list and cleanup its remaining
         * state. This is a separate function since for some cases like

@@ -3064,3 +3067,122 @@ int hci_abort_conn(struct hci_conn *conn, u8 reason)
         */
        return hci_cmd_sync_run_once(hdev, abort_conn_sync, conn, NULL);
 }
+
+void hci_setup_tx_timestamp(struct sk_buff *skb, size_t key_offset,
+                           const struct sockcm_cookie *sockc)
+{
+       struct sock *sk = skb ? skb->sk : NULL;
+
+       /* This shall be called on a single skb of those generated by user
+        * sendmsg(), and only when the sendmsg() does not return error to
+        * user. This is required for keeping the tskey that increments here in
+        * sync with possible sendmsg() counting by user.
+        *
+        * Stream sockets shall set key_offset to sendmsg() length in bytes
+        * and call with the last fragment, others to 1 and first fragment.
+        */
+
+       if (!skb || !sockc || !sk || !key_offset)
+               return;
+
+       sock_tx_timestamp(sk, sockc, &skb_shinfo(skb)->tx_flags);
+
+       if (sockc->tsflags & SOF_TIMESTAMPING_OPT_ID &&
+           sockc->tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK) {
+               if (sockc->tsflags & SOCKCM_FLAG_TS_OPT_ID) {
+                       skb_shinfo(skb)->tskey = sockc->ts_opt_id;
+               } else {
+                       int key = atomic_add_return(key_offset, &sk->sk_tskey);
+
+                       skb_shinfo(skb)->tskey = key - 1;
+               }
+       }
+}
+
+void hci_conn_tx_queue(struct hci_conn *conn, struct sk_buff *skb)
+{
+       struct tx_queue *comp = &conn->tx_q;
+       bool track = false;

net/bluetooth/hci_conn.c int hci_abort_conn(struct hci_conn *conn, u8 reason) · line 3109

Quoted 4 lines

net/bluetooth/hci_conn.c +4 -0

+
+       /* Emit SND now, ie. just before sending to driver */
+       if (skb_shinfo(skb)->tx_flags & SKBTX_SW_TSTAMP)
+               __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SND);

It's a bit strange that SCM_TSTAMP_SND is under the control of SKBTX_SW_TSTAMP. Can we use the same flag for both lines here directly? I suppose I would use SKBTX_SW_TSTAMP then.

Quoted 68 lines

net/bluetooth/hci_conn.c +67 -0

+
+       /* COMPLETION tstamp is emitted for tracked skb later in Number of
+        * Completed Packets event. Available only for flow controlled cases.
+        *
+        * TODO: SCO support without flowctl (needs to be done in drivers)
+        */
+       switch (conn->type) {
+       case ISO_LINK:
+       case ACL_LINK:
+       case LE_LINK:
+               break;
+       case SCO_LINK:
+       case ESCO_LINK:
+               if (!hci_dev_test_flag(conn->hdev, HCI_SCO_FLOWCTL))
+                       return;
+               break;
+       default:
+               return;
+       }
+
+       if (skb->sk && (skb_shinfo(skb)->tx_flags & SKBTX_COMPLETION_TSTAMP))
+               track = true;
+
+       /* If nothing is tracked, just count extra skbs at the queue head */
+       if (!track && !comp->tracked) {
+               comp->extra++;
+               return;
+       }
+
+       if (track) {
+               skb = skb_clone_sk(skb);
+               if (!skb)
+                       goto count_only;
+
+               comp->tracked++;
+       } else {
+               skb = skb_clone(skb, GFP_KERNEL);
+               if (!skb)
+                       goto count_only;
+       }
+
+       skb_queue_tail(&comp->queue, skb);
+       return;
+
+count_only:
+       /* Stop tracking skbs, and only count. This will not emit timestamps for
+        * the packets, but if we get here something is more seriously wrong.
+        */
+       comp->tracked = 0;
+       comp->extra += skb_queue_len(&comp->queue) + 1;
+       skb_queue_purge(&comp->queue);
+}
+
+void hci_conn_tx_dequeue(struct hci_conn *conn)
+{
+       struct tx_queue *comp = &conn->tx_q;
+       struct sk_buff *skb;
+
+       /* If there are tracked skbs, the counted extra go before dequeuing real
+        * skbs, to keep ordering. When nothing is tracked, the ordering doesn't
+        * matter so dequeue real skbs first to get rid of them ASAP.
+        */
+       if (comp->extra && (comp->tracked || skb_queue_empty(&comp->queue))) {
+               comp->extra--;
+               return;
+       }
+

net/bluetooth/hci_conn.c int hci_abort_conn(struct hci_conn *conn, u8 reason) · line 3182

Quoted 6 lines

net/bluetooth/hci_conn.c +6 -0

+       skb = skb_dequeue(&comp->queue);
+       if (!skb)
+               return;
+
+       if (skb->sk) {
+               comp->tracked--;

Need an explicit if statement here?

net/bluetooth/hci_conn.c int hci_abort_conn(struct hci_conn *conn, u8 reason) · line 3184

Quoted 2 lines

net/bluetooth/hci_conn.c +2 -0

+               __skb_tstamp_tx(skb, NULL, NULL, skb->sk,
+                               SCM_TSTAMP_COMPLETION);

This is the socket timestamping, and that's right. My minor question
is: for the use of bpf timestamping (that should be easy as you've done in patch 1, I presume), I'm not sure if you're familiar with it. If not, I plan to implement it myself in a follow-up patch and then let you do some tests? Of course, I will provide the bpf test script

:)

Thanks,
Jason

Quoted 84 lines

net/bluetooth/hci_conn.c +4 -0

+       }
+
+       kfree_skb(skb);
+}

net/bluetooth/hci_core.c +11 -4

index 94d9147612da..5eb0600bbd03 100644
@@ -3029,6 +3029,13 @@ static int hci_send_frame(struct hci_dev *hdev, struct sk_buff *skb)
        return 0;
 }
 
+static int hci_send_conn_frame(struct hci_dev *hdev, struct hci_conn *conn,
+                              struct sk_buff *skb)
+{
+       hci_conn_tx_queue(conn, skb);
+       return hci_send_frame(hdev, skb);
+}
+
 /* Send HCI command */
 int hci_send_cmd(struct hci_dev *hdev, __u16 opcode, __u32 plen,
                 const void *param)

@@ -3575,7 +3582,7 @@ static void hci_sched_sco(struct hci_dev *hdev, __u8 type)
        while (*cnt && (conn = hci_low_sent(hdev, type, &quote))) {
                while (quote-- && (skb = skb_dequeue(&conn->data_q))) {
                        BT_DBG("skb %p len %d", skb, skb->len);
-                       hci_send_frame(hdev, skb);
+                       hci_send_conn_frame(hdev, conn, skb);
 
                        conn->sent++;
                        if (conn->sent == ~0)

@@ -3618,7 +3625,7 @@ static void hci_sched_acl_pkt(struct hci_dev *hdev)
                        hci_conn_enter_active_mode(chan->conn,
                                                   bt_cb(skb)->force_active);
 
-                       hci_send_frame(hdev, skb);
+                       hci_send_conn_frame(hdev, chan->conn, skb);
                        hdev->acl_last_tx = jiffies;
 
                        hdev->acl_cnt--;

@@ -3674,7 +3681,7 @@ static void hci_sched_le(struct hci_dev *hdev)
 
                        skb = skb_dequeue(&chan->data_q);
 
-                       hci_send_frame(hdev, skb);
+                       hci_send_conn_frame(hdev, chan->conn, skb);
                        hdev->le_last_tx = jiffies;
 
                        (*cnt)--;

@@ -3708,7 +3715,7 @@ static void hci_sched_iso(struct hci_dev *hdev)
        while (*cnt && (conn = hci_low_sent(hdev, ISO_LINK, &quote))) {
                while (quote-- && (skb = skb_dequeue(&conn->data_q))) {
                        BT_DBG("skb %p len %d", skb, skb->len);
-                       hci_send_frame(hdev, skb);
+                       hci_send_conn_frame(hdev, conn, skb);
 
                        conn->sent++;
                        if (conn->sent == ~0)

net/bluetooth/hci_event.c +4 -1

index 0df4a0e082c8..83990c975c1f 100644
@@ -4415,6 +4415,7 @@ static void hci_num_comp_pkts_evt(struct hci_dev *hdev, void *data,
                struct hci_comp_pkts_info *info = &ev->handles[i];
                struct hci_conn *conn;
                __u16  handle, count;
+               unsigned int i;
 
                handle = __le16_to_cpu(info->handle);
                count  = __le16_to_cpu(info->count);

@@ -4425,6 +4426,9 @@ static void hci_num_comp_pkts_evt(struct hci_dev *hdev, void *data,
 
                conn->sent -= count;
 
+               for (i = 0; i < count; ++i)
+                       hci_conn_tx_dequeue(conn);
+
                switch (conn->type) {
                case ACL_LINK:
                        hdev->acl_cnt += count;
--

2.48.1

Hi,

ke, 2025-03-19 kello 08:39 +0800, Jason Xing kirjoitti:

Quoted 146 lines · Pauli Virtanen

Quoted 145 lines

Support enabling TX timestamping for some skbs, and track them until packet completion. Generate software SCM_TSTAMP_COMPLETION when getting completion report from hardware.

Generate software SCM_TSTAMP_SND before sending to driver. Sending from driver requires changes in the driver API, and drivers mostly are going to send the skb immediately.

Make the default situation with no COMPLETION TX timestamping more efficient by only counting packets in the queue when there is nothing to track. When there is something to track, we need to make clones, since the driver may modify sent skbs.

The tx_q queue length is bounded by the hdev flow control, which will not send new packets before it has got completion reports for old ones.

Signed-off-by Pauli Virtanen <[email protected]>

---

Notes:
v5:
- Add hci_sockm_init()
- Back to decoupled COMPLETION & SND, like in v3
- Handle SCO flow controlled case

 include/net/bluetooth/hci_core.h |  20 +++++
 net/bluetooth/hci_conn.c         | 122 +++++++++++++++++++++++++++++++
 net/bluetooth/hci_core.c         |  15 +++-
 net/bluetooth/hci_event.c        |   4 +
 4 files changed, 157 insertions(+), 4 deletions(-)

include/net/bluetooth/hci_core.h +20 -0

index f78e4298e39a..5115da34f881 100644
@@ -261,6 +261,12 @@ struct adv_info {
        struct delayed_work     rpa_expired_cb;
 };
 
+struct tx_queue {
+       struct sk_buff_head queue;
+       unsigned int extra;
+       unsigned int tracked;
+};
+
 #define HCI_MAX_ADV_INSTANCES          5
 #define HCI_DEFAULT_ADV_DURATION       2
 

@@ -733,6 +739,8 @@ struct hci_conn {
        struct sk_buff_head data_q;
        struct list_head chan_list;
 
+       struct tx_queue tx_q;
+
        struct delayed_work disc_work;
        struct delayed_work auto_accept_work;
        struct delayed_work idle_work;

@@ -1572,6 +1580,18 @@ void hci_conn_enter_active_mode(struct hci_conn *conn, __u8 force_active);
 void hci_conn_failed(struct hci_conn *conn, u8 status);
 u8 hci_conn_set_handle(struct hci_conn *conn, u16 handle);
 
+void hci_conn_tx_queue(struct hci_conn *conn, struct sk_buff *skb);
+void hci_conn_tx_dequeue(struct hci_conn *conn);
+void hci_setup_tx_timestamp(struct sk_buff *skb, size_t key_offset,
+                           const struct sockcm_cookie *sockc);
+
+static inline void hci_sockcm_init(struct sockcm_cookie *sockc, struct sock *sk)
+{
+       *sockc = (struct sockcm_cookie) {
+               .tsflags = READ_ONCE(sk->sk_tsflags),
+       };
+}
+
 /*
  * hci_conn_get() and hci_conn_put() are used to control the life-time of an
  * "hci_conn" object. They do not guarantee that the hci_conn object is running,

net/bluetooth/hci_conn.c +39 -0

index d097e308a755..95972fd4c784 100644
@@ -27,6 +27,7 @@
 
 #include <linux/export.h>
 #include <linux/debugfs.h>
+#include <linux/errqueue.h>
 
 #include <net/bluetooth/bluetooth.h>
 #include <net/bluetooth/hci_core.h>

@@ -1002,6 +1003,7 @@ static struct hci_conn *__hci_conn_add(struct hci_dev *hdev, int type, bdaddr_t
        }
 
        skb_queue_head_init(&conn->data_q);
+       skb_queue_head_init(&conn->tx_q.queue);
 
        INIT_LIST_HEAD(&conn->chan_list);
        INIT_LIST_HEAD(&conn->link_list);

@@ -1155,6 +1157,7 @@ void hci_conn_del(struct hci_conn *conn)
        }
 
        skb_queue_purge(&conn->data_q);
+       skb_queue_purge(&conn->tx_q.queue);
 
        /* Remove the connection from the list and cleanup its remaining
         * state. This is a separate function since for some cases like

@@ -3064,3 +3067,122 @@ int hci_abort_conn(struct hci_conn *conn, u8 reason)
         */
        return hci_cmd_sync_run_once(hdev, abort_conn_sync, conn, NULL);
 }
+
+void hci_setup_tx_timestamp(struct sk_buff *skb, size_t key_offset,
+                           const struct sockcm_cookie *sockc)
+{
+       struct sock *sk = skb ? skb->sk : NULL;
+
+       /* This shall be called on a single skb of those generated by user
+        * sendmsg(), and only when the sendmsg() does not return error to
+        * user. This is required for keeping the tskey that increments here in
+        * sync with possible sendmsg() counting by user.
+        *
+        * Stream sockets shall set key_offset to sendmsg() length in bytes
+        * and call with the last fragment, others to 1 and first fragment.
+        */
+
+       if (!skb || !sockc || !sk || !key_offset)
+               return;
+
+       sock_tx_timestamp(sk, sockc, &skb_shinfo(skb)->tx_flags);
+
+       if (sockc->tsflags & SOF_TIMESTAMPING_OPT_ID &&
+           sockc->tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK) {
+               if (sockc->tsflags & SOCKCM_FLAG_TS_OPT_ID) {
+                       skb_shinfo(skb)->tskey = sockc->ts_opt_id;
+               } else {
+                       int key = atomic_add_return(key_offset, &sk->sk_tskey);
+
+                       skb_shinfo(skb)->tskey = key - 1;
+               }
+       }
+}
+
+void hci_conn_tx_queue(struct hci_conn *conn, struct sk_buff *skb)
+{
+       struct tx_queue *comp = &conn->tx_q;
+       bool track = false;

Quoted 8 lines

net/bluetooth/hci_conn.c int hci_abort_conn(struct hci_conn *conn, u8 reason) · line 3109

Quoted 4 lines

net/bluetooth/hci_conn.c +4 -0

+
+       /* Emit SND now, ie. just before sending to driver */
+       if (skb_shinfo(skb)->tx_flags & SKBTX_SW_TSTAMP)
+               __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SND);

It's a bit strange that SCM_TSTAMP_SND is under the control of SKBTX_SW_TSTAMP. Can we use the same flag for both lines here directly? I suppose I would use SKBTX_SW_TSTAMP then.

This is more or less open-coded skb_tx_timestamp(), which drivers do before sending to HW, for the Bluetooth case. AFAIK it should be done like this.

Quoted 69 lines

Quoted 69 lines

net/bluetooth/hci_conn.c +67 -0

+
+       /* COMPLETION tstamp is emitted for tracked skb later in Number of
+        * Completed Packets event. Available only for flow controlled cases.
+        *
+        * TODO: SCO support without flowctl (needs to be done in drivers)
+        */
+       switch (conn->type) {
+       case ISO_LINK:
+       case ACL_LINK:
+       case LE_LINK:
+               break;
+       case SCO_LINK:
+       case ESCO_LINK:
+               if (!hci_dev_test_flag(conn->hdev, HCI_SCO_FLOWCTL))
+                       return;
+               break;
+       default:
+               return;
+       }
+
+       if (skb->sk && (skb_shinfo(skb)->tx_flags & SKBTX_COMPLETION_TSTAMP))
+               track = true;
+
+       /* If nothing is tracked, just count extra skbs at the queue head */
+       if (!track && !comp->tracked) {
+               comp->extra++;
+               return;
+       }
+
+       if (track) {
+               skb = skb_clone_sk(skb);
+               if (!skb)
+                       goto count_only;
+
+               comp->tracked++;
+       } else {
+               skb = skb_clone(skb, GFP_KERNEL);
+               if (!skb)
+                       goto count_only;
+       }
+
+       skb_queue_tail(&comp->queue, skb);
+       return;
+
+count_only:
+       /* Stop tracking skbs, and only count. This will not emit timestamps for
+        * the packets, but if we get here something is more seriously wrong.
+        */
+       comp->tracked = 0;
+       comp->extra += skb_queue_len(&comp->queue) + 1;
+       skb_queue_purge(&comp->queue);
+}
+
+void hci_conn_tx_dequeue(struct hci_conn *conn)
+{
+       struct tx_queue *comp = &conn->tx_q;
+       struct sk_buff *skb;
+
+       /* If there are tracked skbs, the counted extra go before dequeuing real
+        * skbs, to keep ordering. When nothing is tracked, the ordering doesn't
+        * matter so dequeue real skbs first to get rid of them ASAP.
+        */
+       if (comp->extra && (comp->tracked || skb_queue_empty(&comp->queue))) {
+               comp->extra--;
+               return;
+       }
+

Quoted 8 lines

net/bluetooth/hci_conn.c int hci_abort_conn(struct hci_conn *conn, u8 reason) · line 3182

Quoted 6 lines

net/bluetooth/hci_conn.c +6 -0

+       skb = skb_dequeue(&comp->queue);
+       if (!skb)
+               return;
+
+       if (skb->sk) {
+               comp->tracked--;

Need an explicit if statement here?

I think no, see explanation of how it works in the reply to Willem:
https://lore.kernel.org/linux-bluetooth/[email protected]/

Quoted 8 lines

net/bluetooth/hci_conn.c int hci_abort_conn(struct hci_conn *conn, u8 reason) · line 3184

Quoted 2 lines

net/bluetooth/hci_conn.c +2 -0

+               __skb_tstamp_tx(skb, NULL, NULL, skb->sk,
+                               SCM_TSTAMP_COMPLETION);

This is the socket timestamping, and that's right. My minor question
is: for the use of bpf timestamping (that should be easy as you've done in patch 1, I presume), I'm not sure if you're familiar with it. If not, I plan to implement it myself in a follow-up patch and then let you do some tests? Of course, I will provide the bpf test script

I saw the BPF timestamping things, but didn't look in full detail yet. I don't know much about BPF, but IIUC, is it just as simple as adding BPF_SOCK_OPS_TSTAMP_COMPLETION_CB and then modifying skb_tstamp_tx_report_bpf_timestamping() accordingly?

I think we'd want to add also the BPF tests in the Bluetooth socket timestamping tests

https://web.git.kernel.org/pub/scm/bluetooth/bluez.git/tree/doc/test-runner.rst
https://web.git.kernel.org/pub/scm/bluetooth/bluez.git/tree/tools/tester.h#n91
https://web.git.kernel.org/pub/scm/bluetooth/bluez.git/tree/tools/iso-tester.c#n2275
https://web.git.kernel.org/pub/scm/bluetooth/bluez.git/tree/tools/sco-tester.c#n755
https://web.git.kernel.org/pub/scm/bluetooth/bluez.git/tree/tools/l2cap-tester.c#n1369

If you could show an example how to setup the BPF tstamps and pass the resulting tstamp data in some way back to the application, that could be very helpful (and I could postpone learning BPF for a little while longer :)

Quoted 80 lines

:)

Thanks,
Jason

Quoted 74 lines

net/bluetooth/hci_conn.c +4 -0

+       }
+
+       kfree_skb(skb);
+}

net/bluetooth/hci_core.c +11 -4

index 94d9147612da..5eb0600bbd03 100644
@@ -3029,6 +3029,13 @@ static int hci_send_frame(struct hci_dev *hdev, struct sk_buff *skb)
        return 0;
 }
 
+static int hci_send_conn_frame(struct hci_dev *hdev, struct hci_conn *conn,
+                              struct sk_buff *skb)
+{
+       hci_conn_tx_queue(conn, skb);
+       return hci_send_frame(hdev, skb);
+}
+
 /* Send HCI command */
 int hci_send_cmd(struct hci_dev *hdev, __u16 opcode, __u32 plen,
                 const void *param)

@@ -3575,7 +3582,7 @@ static void hci_sched_sco(struct hci_dev *hdev, __u8 type)
        while (*cnt && (conn = hci_low_sent(hdev, type, &quote))) {
                while (quote-- && (skb = skb_dequeue(&conn->data_q))) {
                        BT_DBG("skb %p len %d", skb, skb->len);
-                       hci_send_frame(hdev, skb);
+                       hci_send_conn_frame(hdev, conn, skb);
 
                        conn->sent++;
                        if (conn->sent == ~0)

@@ -3618,7 +3625,7 @@ static void hci_sched_acl_pkt(struct hci_dev *hdev)
                        hci_conn_enter_active_mode(chan->conn,
                                                   bt_cb(skb)->force_active);
 
-                       hci_send_frame(hdev, skb);
+                       hci_send_conn_frame(hdev, chan->conn, skb);
                        hdev->acl_last_tx = jiffies;
 
                        hdev->acl_cnt--;

@@ -3674,7 +3681,7 @@ static void hci_sched_le(struct hci_dev *hdev)
 
                        skb = skb_dequeue(&chan->data_q);
 
-                       hci_send_frame(hdev, skb);
+                       hci_send_conn_frame(hdev, chan->conn, skb);
                        hdev->le_last_tx = jiffies;
 
                        (*cnt)--;

@@ -3708,7 +3715,7 @@ static void hci_sched_iso(struct hci_dev *hdev)
        while (*cnt && (conn = hci_low_sent(hdev, ISO_LINK, &quote))) {
                while (quote-- && (skb = skb_dequeue(&conn->data_q))) {
                        BT_DBG("skb %p len %d", skb, skb->len);
-                       hci_send_frame(hdev, skb);
+                       hci_send_conn_frame(hdev, conn, skb);
 
                        conn->sent++;
                        if (conn->sent == ~0)

net/bluetooth/hci_event.c +1 -0

index 0df4a0e082c8..83990c975c1f 100644
@@ -4415,6 +4415,7 @@ static void hci_num_comp_pkts_evt(struct hci_dev *hdev, void *data,
                struct hci_comp_pkts_info *info = &ev->handles[i];
                struct hci_conn *conn;
                __u16  handle, count;
+               unsigned int i;
 
                handle = __le16_to_cpu(info->handle);
                count  = __le16_to_cpu(info->count);

@@ -4425,6 +4426,9 @@ static void hci_num_comp_pkts_evt(struct hci_dev *hdev, void *data,
 
                conn->sent -= count;
 

Quoted 10 lines

Quoted 10 lines

net/bluetooth/hci_event.c +3 -1

+               for (i = 0; i < count; ++i)
+                       hci_conn_tx_dequeue(conn);
+
                switch (conn->type) {
                case ACL_LINK:
                        hdev->acl_cnt += count;
--

2.48.1

--
Pauli Virtanen

Add BT_SCM_ERROR socket CMSG type.

Support TX timestamping in ISO sockets.

Support MSG_ERRQUEUE in ISO recvmsg.

If a packet from sendmsg() is fragmented, only the first ACL fragment is timestamped.

Signed-off-by Pauli Virtanen <[email protected]>

---

Notes:
v5:
- use sockcm_init -> hci_sockcm_init

 include/net/bluetooth/bluetooth.h |  1 +
 net/bluetooth/iso.c               | 24 ++++++++++++++++++++----
 2 files changed, 21 insertions(+), 4 deletions(-)

include/net/bluetooth/bluetooth.h +1 -0

index 435250c72d56..bbefde319f95 100644
@@ -156,6 +156,7 @@ struct bt_voice {
 #define BT_PKT_STATUS           16
 
 #define BT_SCM_PKT_STATUS	0x03
+#define BT_SCM_ERROR		0x04
 
 #define BT_ISO_QOS		17
 

net/bluetooth/iso.c +7 -3

index 0cb52a3308ba..3501a991f1c6 100644
@@ -518,7 +518,8 @@ static struct bt_iso_qos *iso_sock_get_qos(struct sock *sk)
 	return &iso_pi(sk)->qos;
 }
 
-static int iso_send_frame(struct sock *sk, struct sk_buff *skb)
+static int iso_send_frame(struct sock *sk, struct sk_buff *skb,
+			  const struct sockcm_cookie *sockc)
 {
 	struct iso_conn *conn = iso_pi(sk)->conn;
 	struct bt_iso_qos *qos = iso_sock_get_qos(sk);

@@ -538,10 +539,12 @@ static int iso_send_frame(struct sock *sk, struct sk_buff *skb)
 	hdr->slen = cpu_to_le16(hci_iso_data_len_pack(len,
 						      HCI_ISO_STATUS_VALID));
 
-	if (sk->sk_state == BT_CONNECTED)
+	if (sk->sk_state == BT_CONNECTED) {
+		hci_setup_tx_timestamp(skb, 1, sockc);
 		hci_send_iso(conn->hcon, skb);
-	else
+	} else {
 		len = -ENOTCONN;
+	}
 
 	return len;
 }

@@ -1348,6 +1351,7 @@ static int iso_sock_sendmsg(struct socket *sock, struct msghdr *msg,
 {
 	struct sock *sk = sock->sk;
 	struct sk_buff *skb, **frag;
+	struct sockcm_cookie sockc;
 	size_t mtu;
 	int err;
 

@@ -1360,6 +1364,14 @@ static int iso_sock_sendmsg(struct socket *sock, struct msghdr *msg,
 	if (msg->msg_flags & MSG_OOB)