ovpn: preserve GSO batching

index f14eb4e6c46a..3b0c9b30e879 100644
@@ -356,6 +356,7 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev)
 	struct ovpn_priv *ovpn = netdev_priv(dev);
 	struct sk_buff *segments, *curr, *next;
 	struct sk_buff_head skb_list;
+	netdev_features_t features;
 	unsigned int tx_bytes = 0;
 	struct ovpn_peer *peer;
 	__be16 proto;

@@ -390,8 +391,13 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev)
 	skb_dst_drop(skb);
 
 	if (skb_is_gso(skb)) {
-		segments = skb_gso_segment(skb, 0);
-		if (IS_ERR(segments)) {
+		/* force software segmentation, but keep ovpn's non-GSO feature
+		 * bits so the generated segments can preserve non-linear skb
+		 * data where possible
+		 */
+		features = netif_skb_features(skb);
+		segments = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK);
+		if (IS_ERR_OR_NULL(segments)) {
 			ret = PTR_ERR(segments);
 			net_err_ratelimited("%s: cannot segment payload packet: %d\n",
 					    netdev_name(dev), ret);

@@ -416,6 +422,16 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev)
 			continue;
 		}
 
+		/* NETIF_F_HW_CSUM requires completing partial checksums */
+		if (unlikely(curr->ip_summed == CHECKSUM_PARTIAL &&
+			     skb_checksum_help(curr) < 0)) {
+			net_err_ratelimited("%s: skb_checksum_help failed for payload packet\n",
+					    netdev_name(dev));
+			ovpn_dev_dstats_tx_dropped(ovpn->dev);
+			kfree_skb(curr);
+			continue;
+		}
+
 		/* only count what we actually send */
 		tx_bytes += curr->len;
 		__skb_queue_tail(&skb_list, curr);

@@ -430,7 +446,8 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev)
 	}
 	skb_list.prev->next = NULL;
 
-	ovpn_peer_stats_increment_tx(&peer->vpn_stats, tx_bytes);
+	ovpn_peer_stats_add_tx(&peer->vpn_stats, tx_bytes,
+			       skb_queue_len(&skb_list));
 	ovpn_send(ovpn, skb_list.next, peer);
 
 	return NETDEV_TX_OK;

index 0708249e9607..28e1eb06e127 100644
@@ -157,7 +157,7 @@ static const struct ethtool_ops ovpn_ethtool_ops = {
 
 static void ovpn_setup(struct net_device *dev)
 {
-	netdev_features_t feat = NETIF_F_SG | NETIF_F_GSO |
+	netdev_features_t feat = NETIF_F_HW_CSUM | NETIF_F_SG | NETIF_F_GSO |
 				 NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA;
 
 	dev->needs_free_netdev = true;

index 3a45b97c0056..9331f5e9179b 100644
@@ -40,10 +40,18 @@ static inline void ovpn_peer_stats_increment_rx(struct ovpn_peer_stats *stats,
 	ovpn_peer_stats_increment(&stats->rx, n);
 }
 
+static inline void ovpn_peer_stats_add_tx(struct ovpn_peer_stats *stats,
+					  const unsigned int bytes,
+					  unsigned int packets)
+{
+	atomic64_add(bytes, &stats->tx.bytes);
+	atomic64_add(packets, &stats->tx.packets);
+}
+
 static inline void ovpn_peer_stats_increment_tx(struct ovpn_peer_stats *stats,
 						const unsigned int n)
 {
-	ovpn_peer_stats_increment(&stats->tx, n);
+	ovpn_peer_stats_add_tx(stats, n, 1);
 }
 
 static inline void ovpn_dev_dstats_tx_dropped(struct net_device *dev)

index 74eaf6fac2f5..2af493fd5735 100644
@@ -168,8 +168,12 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 	if (unlikely(nfrags < 0))
 		return nfrags;
 
-	if (unlikely(nfrags + 2 > (MAX_SKB_FRAGS + 2)))
-		return -ENOSPC;
+	if (unlikely(nfrags > MAX_SKB_FRAGS)) {
+		ret = skb_linearize(skb);
+		if (unlikely(ret))
+			return ret;
+		nfrags = 1;
+	}
 
 	/* allocate temporary memory for iv, sg and req */
 	tmp = kmalloc(ovpn_aead_crypto_tmp_size(ks->encrypt, nfrags),

index 28e1eb06e127..ac4e0d85e215 100644
@@ -158,7 +158,8 @@ static const struct ethtool_ops ovpn_ethtool_ops = {
 static void ovpn_setup(struct net_device *dev)
 {
 	netdev_features_t feat = NETIF_F_HW_CSUM | NETIF_F_SG | NETIF_F_GSO |
-				 NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA;
+				 NETIF_F_GSO_SOFTWARE | NETIF_F_FRAGLIST |
+				 NETIF_F_HIGHDMA;
 
 	dev->needs_free_netdev = true;
 

index 2af493fd5735..30299581422d 100644
@@ -24,9 +24,6 @@
 #include "proto.h"
 #include "skb.h"
 
-#define OVPN_AUTH_TAG_SIZE	16
-#define OVPN_AAD_SIZE		(OVPN_OPCODE_SIZE + OVPN_NONCE_WIRE_SIZE)
-
 #define ALG_NAME_AES		"gcm(aes)"
 #define ALG_NAME_CHACHAPOLY	"rfc7539(chacha20,poly1305)"
 

@@ -42,7 +39,7 @@ static int ovpn_aead_encap_overhead(const struct ovpn_crypto_key_slot *ks)
  *			       an AEAD request structure with extra space for SG
  *			       and IV.
  * @tfm: the AEAD cipher handle
- * @nfrags: the number of fragments in the skb
+ * @nents: the number of scatterlist entries
  *
  * This function calculates the size of a contiguous memory block that includes
  * the initialization vector (IV), the AEAD request, and an array of scatterlist

@@ -54,7 +51,7 @@ static int ovpn_aead_encap_overhead(const struct ovpn_crypto_key_slot *ks)
  * Return: the size of the temporary memory that needs to be allocated
  */
 static unsigned int ovpn_aead_crypto_tmp_size(struct crypto_aead *tfm,
-					      const unsigned int nfrags)
+					      const unsigned int nents)
 {
 	unsigned int len = OVPN_NONCE_SIZE;
 

@@ -70,8 +67,8 @@ static unsigned int ovpn_aead_crypto_tmp_size(struct crypto_aead *tfm,
 	/* round up to the next multiple of the scatterlist alignment */
 	len = ALIGN(len, __alignof__(struct scatterlist));
 
-	/* add enough space for nfrags + 2 scatterlist entries */
-	len += array_size(sizeof(struct scatterlist), nfrags + 2);
+	/* add enough space for the scatterlist entries */
+	len += array_size(sizeof(struct scatterlist), nents);
 	return len;
 }
 

@@ -135,6 +132,53 @@ static struct scatterlist *ovpn_aead_crypto_req_sg(struct crypto_aead *aead,
 			     __alignof__(struct scatterlist));
 }
 
+static struct aead_request *ovpn_aead_request_alloc(struct crypto_aead *aead,
+						    struct sk_buff *skb,
+						    unsigned int nents, u8 **iv)
+{
+	struct aead_request *req;
+	void *tmp;
+
+	/* allocate IV, request and scatterlist entries in one block */
+	tmp = kmalloc(ovpn_aead_crypto_tmp_size(aead, nents), GFP_ATOMIC);
+	if (unlikely(!tmp))
+		return ERR_PTR(-ENOMEM);
+
+	ovpn_skb_cb(skb)->crypto_tmp = tmp;
+	*iv = ovpn_aead_crypto_tmp_iv(aead, tmp);
+	req = ovpn_aead_crypto_tmp_req(aead, *iv);
+
+	return req;
+}
+
+static int ovpn_aead_encrypt_header(struct ovpn_peer *peer,
+				    struct ovpn_crypto_key_slot *ks,
+				    u8 *iv, u8 *data)
+{
+	u32 pktid, op;
+	int ret;
+
+	/* obtain packet ID, which is used both as a first
+	 * 4 bytes of nonce and last 4 bytes of associated data.
+	 */
+	ret = ovpn_pktid_xmit_next(&ks->pid_xmit, &pktid);
+	if (unlikely(ret < 0))
+		return ret;
+
+	/* concat 4 bytes packet id and 8 bytes nonce tail into 12 bytes
+	 * nonce
+	 */
+	ovpn_pktid_aead_write(pktid, ks->nonce_tail_xmit, iv);
+
+	/* add the packet opcode and wire nonce as associated data */
+	op = ovpn_opcode_compose(OVPN_DATA_V2, ks->key_id, peer->tx_id);
+	BUILD_BUG_ON(sizeof(op) != OVPN_OPCODE_SIZE);
+	*(__force __be32 *)data = htonl(op);
+	memcpy(data + OVPN_OPCODE_SIZE, iv, OVPN_NONCE_WIRE_SIZE);
+
+	return 0;
+}
+
 int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 		      struct sk_buff *skb)
 {

@@ -143,8 +187,6 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 	struct sk_buff *trailer;
 	struct scatterlist *sg;
 	int nfrags, ret;
-	u32 pktid, op;
-	void *tmp;
 	u8 *iv;
 
 	ovpn_skb_cb(skb)->peer = peer;

@@ -175,16 +217,9 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 		nfrags = 1;
 	}
 
-	/* allocate temporary memory for iv, sg and req */
-	tmp = kmalloc(ovpn_aead_crypto_tmp_size(ks->encrypt, nfrags),
-		      GFP_ATOMIC);
-	if (unlikely(!tmp))
-		return -ENOMEM;
-
-	ovpn_skb_cb(skb)->crypto_tmp = tmp;
-
-	iv = ovpn_aead_crypto_tmp_iv(ks->encrypt, tmp);
-	req = ovpn_aead_crypto_tmp_req(ks->encrypt, iv);
+	req = ovpn_aead_request_alloc(ks->encrypt, skb, nfrags + 2, &iv);
+	if (IS_ERR(req))
+		return PTR_ERR(req);
 	sg = ovpn_aead_crypto_req_sg(ks->encrypt, req);
 
 	/* sg table:

@@ -206,28 +241,12 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 	__skb_push(skb, tag_size);
 	sg_set_buf(sg + ret + 1, skb->data, tag_size);
 
-	/* obtain packet ID, which is used both as a first
-	 * 4 bytes of nonce and last 4 bytes of associated data.
-	 */
-	ret = ovpn_pktid_xmit_next(&ks->pid_xmit, &pktid);
+	/* make space for the additional data and push it to the front */
+	__skb_push(skb, OVPN_AAD_SIZE);
+	ret = ovpn_aead_encrypt_header(peer, ks, iv, skb->data);
 	if (unlikely(ret < 0))
 		return ret;
 
-	/* concat 4 bytes packet id and 8 bytes nonce tail into 12 bytes
-	 * nonce
-	 */
-	ovpn_pktid_aead_write(pktid, ks->nonce_tail_xmit, iv);
-
-	/* make space for packet id and push it to the front */
-	__skb_push(skb, OVPN_NONCE_WIRE_SIZE);
-	memcpy(skb->data, iv, OVPN_NONCE_WIRE_SIZE);
-
-	/* add packet op as head of additional data */
-	op = ovpn_opcode_compose(OVPN_DATA_V2, ks->key_id, peer->tx_id);
-	__skb_push(skb, OVPN_OPCODE_SIZE);
-	BUILD_BUG_ON(sizeof(op) != OVPN_OPCODE_SIZE);
-	*((__force __be32 *)skb->data) = htonl(op);
-
 	/* AEAD Additional data */
 	sg_set_buf(sg, skb->data, OVPN_AAD_SIZE);
 

@@ -281,7 +300,7 @@ int ovpn_aead_decrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 		return -ENOSPC;
 
 	/* allocate temporary memory for iv, sg and req */
-	tmp = kmalloc(ovpn_aead_crypto_tmp_size(ks->decrypt, nfrags),
+	tmp = kmalloc(ovpn_aead_crypto_tmp_size(ks->decrypt, nfrags + 2),
 		      GFP_ATOMIC);
 	if (unlikely(!tmp))
 		return -ENOMEM;

index db9e10f9077c..1a94f0fda1d1 100644
@@ -11,8 +11,7 @@
 #define _NET_OVPN_OVPN_H_
 
 /* DATA_V2 header size with AEAD encryption */
-#define OVPN_HEAD_ROOM (OVPN_OPCODE_SIZE + OVPN_NONCE_WIRE_SIZE +	   \
-			16 /* AEAD TAG length */ +			   \
+#define OVPN_HEAD_ROOM (OVPN_DATA_V2_OVERHEAD +				   \
 			max(sizeof(struct udphdr), sizeof(struct tcphdr)) +\
 			max(sizeof(struct ipv6hdr), sizeof(struct iphdr)))
 

index b7d285b4d9c1..f3b305cbefe5 100644
@@ -39,6 +39,10 @@
 #define OVPN_NONCE_WIRE_SIZE (OVPN_NONCE_SIZE - OVPN_NONCE_TAIL_SIZE)
 
 #define OVPN_OPCODE_SIZE		4 /* DATA_V2 opcode size */
+#define OVPN_AUTH_TAG_SIZE		16
+#define OVPN_AAD_SIZE			(OVPN_OPCODE_SIZE + \
+					 OVPN_NONCE_WIRE_SIZE)
+#define OVPN_DATA_V2_OVERHEAD		(OVPN_AAD_SIZE + OVPN_AUTH_TAG_SIZE)
 #define OVPN_OPCODE_KEYID_MASK		0x07000000
 #define OVPN_OPCODE_PKTTYPE_MASK	0xF8000000
 #define OVPN_OPCODE_PEERID_MASK		0x00FFFFFF

index e3feb16d5498..8ce324e9b58f 100644
@@ -39,6 +39,8 @@ struct ovpn_peer_key_reset {
 
 struct ovpn_crypto_key_slot {
 	u8 key_id;
+	/* whether the encrypt transform is asynchronous */
+	bool async;
 
 	struct crypto_aead *encrypt;
 	struct crypto_aead *decrypt;

index 30299581422d..5e5320338165 100644
@@ -462,6 +462,7 @@ ovpn_aead_crypto_key_slot_new(const struct ovpn_key_config *kc)
 		ks->encrypt = NULL;
 		goto destroy_ks;
 	}
+	ks->async = crypto_tfm_is_async(crypto_aead_tfm(ks->encrypt));
 
 	ks->decrypt = ovpn_aead_init("decrypt", alg_name,
 				     kc->decrypt.cipher_key,

index 5e5320338165..26305984197e 100644
@@ -134,13 +134,17 @@ static struct scatterlist *ovpn_aead_crypto_req_sg(struct crypto_aead *aead,
 
 static struct aead_request *ovpn_aead_request_alloc(struct crypto_aead *aead,
 						    struct sk_buff *skb,
-						    unsigned int nents, u8 **iv)
+						    unsigned int nents,
+						    unsigned int extra, u8 **iv)
 {
 	struct aead_request *req;
 	void *tmp;
 
-	/* allocate IV, request and scatterlist entries in one block */
-	tmp = kmalloc(ovpn_aead_crypto_tmp_size(aead, nents), GFP_ATOMIC);
+	/* allocate IV, request, scatterlist entries and caller scratch space
+	 * in one block
+	 */
+	tmp = kmalloc(ovpn_aead_crypto_tmp_size(aead, nents) + extra,
+		      GFP_ATOMIC);
 	if (unlikely(!tmp))
 		return ERR_PTR(-ENOMEM);
 

@@ -151,19 +155,11 @@ static struct aead_request *ovpn_aead_request_alloc(struct crypto_aead *aead,
 	return req;
 }
 
-static int ovpn_aead_encrypt_header(struct ovpn_peer *peer,
-				    struct ovpn_crypto_key_slot *ks,
-				    u8 *iv, u8 *data)
+static void ovpn_aead_encrypt_header(struct ovpn_peer *peer,
+				     struct ovpn_crypto_key_slot *ks,
+				     u32 pktid, u8 *iv, u8 *data)
 {
-	u32 pktid, op;
-	int ret;
-
-	/* obtain packet ID, which is used both as a first
-	 * 4 bytes of nonce and last 4 bytes of associated data.
-	 */
-	ret = ovpn_pktid_xmit_next(&ks->pid_xmit, &pktid);
-	if (unlikely(ret < 0))
-		return ret;
+	u32 op;
 
 	/* concat 4 bytes packet id and 8 bytes nonce tail into 12 bytes
 	 * nonce

@@ -175,8 +171,74 @@ static int ovpn_aead_encrypt_header(struct ovpn_peer *peer,
 	BUILD_BUG_ON(sizeof(op) != OVPN_OPCODE_SIZE);
 	*(__force __be32 *)data = htonl(op);
 	memcpy(data + OVPN_OPCODE_SIZE, iv, OVPN_NONCE_WIRE_SIZE);
+}
+
+int ovpn_aead_encrypt_gso(struct ovpn_peer *peer,
+			  struct ovpn_crypto_key_slot *ks, struct sk_buff *skb,
+			  struct sk_buff *gso_skb, unsigned int offset)
+{
+	const unsigned int dst_nents = skb_shinfo(gso_skb)->nr_frags + 4;
+	const unsigned int src_nents = skb_shinfo(skb)->nr_frags + 2;
+	struct scatterlist *src, *dst;
+	struct aead_request *req;
+	unsigned int payload_off;
+	int dst_idx, mapped, ret;
+	u8 *aad, *iv;
+	u32 pktid;
+
+	req = ovpn_aead_request_alloc(ks->encrypt, skb, src_nents + dst_nents,
+				      OVPN_AAD_SIZE, &iv);
+	if (IS_ERR(req))
+		return PTR_ERR(req);
+	src = ovpn_aead_crypto_req_sg(ks->encrypt, req);
+	dst = src + src_nents;
+	aad = (u8 *)(dst + dst_nents);
 
-	return 0;
+	ret = ovpn_pktid_xmit_next(&ks->pid_xmit, &pktid);
+	if (unlikely(ret < 0))
+		return ret;
+	ovpn_aead_encrypt_header(peer, ks, pktid, iv, aad);
+
+	ret = skb_store_bits(gso_skb, offset, aad, OVPN_AAD_SIZE);
+	if (unlikely(ret < 0))
+		return ret;
+
+	/* encrypt out of place from the original segmented skb directly into
+	 * its final range in the UDP GSO skb
+	 */
+	sg_init_table(src, src_nents);
+	sg_set_buf(src, aad, OVPN_AAD_SIZE);
+	mapped = skb_to_sgvec_nomark(skb, src + 1, 0, skb->len);
+	if (unlikely(mapped < 0))
+		return mapped;
+	sg_mark_end(&src[mapped]);
+
+	sg_init_table(dst, dst_nents);
+	dst_idx = skb_to_sgvec_nomark(gso_skb, dst, offset, OVPN_AAD_SIZE);
+	if (unlikely(dst_idx < 0))
+		return dst_idx;
+
+	payload_off = offset + OVPN_AAD_SIZE + OVPN_AUTH_TAG_SIZE;
+	mapped = skb_to_sgvec_nomark(gso_skb, dst + dst_idx, payload_off,
+				     skb->len);
+	if (unlikely(mapped < 0))
+		return mapped;
+	dst_idx += mapped;
+
+	mapped = skb_to_sgvec_nomark(gso_skb, dst + dst_idx,
+				     offset + OVPN_AAD_SIZE,
+				     OVPN_AUTH_TAG_SIZE);
+	if (unlikely(mapped < 0))
+		return mapped;
+	dst_idx += mapped;
+	sg_mark_end(&dst[dst_idx - 1]);
+
+	aead_request_set_tfm(req, ks->encrypt);
+	aead_request_set_callback(req, 0, NULL, NULL);
+	aead_request_set_crypt(req, src, dst, skb->len, iv);
+	aead_request_set_ad(req, OVPN_AAD_SIZE);
+
+	return crypto_aead_encrypt(req);
 }
 
 int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,

@@ -187,6 +249,7 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 	struct sk_buff *trailer;
 	struct scatterlist *sg;
 	int nfrags, ret;
+	u32 pktid;
 	u8 *iv;
 
 	ovpn_skb_cb(skb)->peer = peer;

@@ -217,7 +280,7 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 		nfrags = 1;
 	}
 
-	req = ovpn_aead_request_alloc(ks->encrypt, skb, nfrags + 2, &iv);
+	req = ovpn_aead_request_alloc(ks->encrypt, skb, nfrags + 2, 0, &iv);
 	if (IS_ERR(req))
 		return PTR_ERR(req);
 	sg = ovpn_aead_crypto_req_sg(ks->encrypt, req);

@@ -243,9 +306,11 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 
 	/* make space for the additional data and push it to the front */
 	__skb_push(skb, OVPN_AAD_SIZE);
-	ret = ovpn_aead_encrypt_header(peer, ks, iv, skb->data);
+	/* obtain the packet ID used by both the nonce and associated data */
+	ret = ovpn_pktid_xmit_next(&ks->pid_xmit, &pktid);
 	if (unlikely(ret < 0))
 		return ret;
+	ovpn_aead_encrypt_header(peer, ks, pktid, iv, skb->data);
 
 	/* AEAD Additional data */
 	sg_set_buf(sg, skb->data, OVPN_AAD_SIZE);

index fae3b585a43b..8b444744944e 100644
@@ -17,6 +17,10 @@
 
 int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 		      struct sk_buff *skb);
+int ovpn_aead_encrypt_gso(struct ovpn_peer *peer,
+			  struct ovpn_crypto_key_slot *ks,
+			  struct sk_buff *skb, struct sk_buff *gso_skb,
+			  unsigned int offset);
 int ovpn_aead_decrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 		      struct sk_buff *skb);
 

449 lines not on this page · raw message

index 9331f5e9179b..b3fe006c01f6 100644
@@ -54,10 +54,12 @@ static inline void ovpn_peer_stats_increment_tx(struct ovpn_peer_stats *stats,
 	ovpn_peer_stats_add_tx(stats, n, 1);
 }
 
-static inline void ovpn_dev_dstats_tx_dropped(struct net_device *dev)
+static inline void ovpn_dev_dstats_tx_dropped(struct net_device *dev,
+					      unsigned int packets)
 {
 	local_bh_disable();
-	dev_dstats_tx_dropped(dev);
+	while (packets--)
+		dev_dstats_tx_dropped(dev);
 	local_bh_enable();
 }
 

index 8fe8a8e750a4..5cba35e4a8ee 100644
@@ -332,7 +332,7 @@ static void ovpn_tcp_send_sock_skb(struct ovpn_peer *peer, struct sock *sk,
 		ovpn_tcp_send_sock(peer, sk);
 
 	if (peer->tcp.out_msg.skb) {
-		ovpn_dev_dstats_tx_dropped(peer->ovpn->dev);
+		ovpn_dev_dstats_tx_dropped(peer->ovpn->dev, 1);
 		kfree_skb(skb);
 		return;
 	}

@@ -354,7 +354,7 @@ void ovpn_tcp_send_skb(struct ovpn_peer *peer, struct sock *sk,
 	if (sock_owned_by_user(sk)) {
 		if (skb_queue_len(&peer->tcp.out_queue) >=
 		    READ_ONCE(net_hotdata.max_backlog)) {
-			ovpn_dev_dstats_tx_dropped(peer->ovpn->dev);
+			ovpn_dev_dstats_tx_dropped(peer->ovpn->dev, 1);
 			kfree_skb(skb);
 			goto unlock;
 		}

index 055cdb1bee13..0eb6d32f4ceb 100644
@@ -288,9 +288,13 @@ static int ovpn_udp4_output(struct ovpn_peer *peer, struct ovpn_bind *bind,
 	spin_unlock_bh(&peer->lock);
 
 transmit:
+	/* an already-built UDP GSO needs a checksum seed even if the socket's
+	 * no-check option changed while encryption was in flight
+	 */
 	udp_tunnel_xmit_skb(rt, sk, skb, fl.saddr, fl.daddr, 0,
 			    ip4_dst_hoplimit(&rt->dst), 0, fl.fl4_sport,
-			    fl.fl4_dport, false, sk->sk_no_check_tx, 0);
+			    fl.fl4_dport, false,
+			    !skb_is_gso(skb) && sk->sk_no_check_tx, 0);
 	ret = 0;
 err:
 	local_bh_enable();

@@ -388,9 +392,13 @@ static int ovpn_udp6_output(struct ovpn_peer *peer, struct ovpn_bind *bind,
 	 * udp_tunnel_xmit_skb()
 	 */
 	skb->ignore_df = 1;
+	/* keep checksum offload enabled for an in-flight UDP GSO batch even if
+	 * the socket's no-check option has changed since batch creation
+	 */
 	udp_tunnel6_xmit_skb(dst, sk, skb, skb->dev, &fl.saddr, &fl.daddr, 0,
 			     ip6_dst_hoplimit(dst), 0, fl.fl6_sport,
-			     fl.fl6_dport, udp_get_no_check6_tx(sk), 0);
+			     fl.fl6_dport,
+			     !skb_is_gso(skb) && udp_get_no_check6_tx(sk), 0);
 	ret = 0;
 err:
 	local_bh_enable();

@@ -467,8 +475,18 @@ void ovpn_udp_send_skb(struct ovpn_peer *peer, struct sock *sk,
 
 	skb->dev = peer->ovpn->dev;
 	skb->mark = key.mark;
-	/* no checksum performed at this layer */
-	skb->ip_summed = CHECKSUM_NONE;
+	if (skb_is_gso(skb)) {
+		/* udp_tunnel_xmit_skb installs the outer UDP header after this
+		 * function returns: point CHECKSUM_PARTIAL at that future
+		 * header so both hw and sw UDP GSO can complete the checksum.
+		 */
+		skb->ip_summed = CHECKSUM_PARTIAL;
+		skb->csum_start = skb_headroom(skb) - sizeof(struct udphdr);
+		skb->csum_offset = offsetof(struct udphdr, check);
+	} else {
+		/* no checksum performed at this layer */
+		skb->ip_summed = CHECKSUM_NONE;
+	}
 
 	ovpn_dst_cache_check_key(peer, &peer->dst_cache, &key);
 

index 26305984197e..a75a0bf65c89 100644
@@ -175,7 +175,8 @@ static void ovpn_aead_encrypt_header(struct ovpn_peer *peer,
 
 int ovpn_aead_encrypt_gso(struct ovpn_peer *peer,
 			  struct ovpn_crypto_key_slot *ks, struct sk_buff *skb,
-			  struct sk_buff *gso_skb, unsigned int offset)
+			  struct sk_buff *gso_skb, unsigned int offset,
+			  u32 pktid)
 {
 	const unsigned int dst_nents = skb_shinfo(gso_skb)->nr_frags + 4;
 	const unsigned int src_nents = skb_shinfo(skb)->nr_frags + 2;

@@ -184,7 +185,6 @@ int ovpn_aead_encrypt_gso(struct ovpn_peer *peer,
 	unsigned int payload_off;
 	int dst_idx, mapped, ret;
 	u8 *aad, *iv;
-	u32 pktid;
 
 	req = ovpn_aead_request_alloc(ks->encrypt, skb, src_nents + dst_nents,
 				      OVPN_AAD_SIZE, &iv);

@@ -194,9 +194,6 @@ int ovpn_aead_encrypt_gso(struct ovpn_peer *peer,
 	dst = src + src_nents;
 	aad = (u8 *)(dst + dst_nents);
 
-	ret = ovpn_pktid_xmit_next(&ks->pid_xmit, &pktid);
-	if (unlikely(ret < 0))
-		return ret;
 	ovpn_aead_encrypt_header(peer, ks, pktid, iv, aad);
 
 	ret = skb_store_bits(gso_skb, offset, aad, OVPN_AAD_SIZE);

index 8b444744944e..92e8ed76fba6 100644
@@ -20,7 +20,7 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 int ovpn_aead_encrypt_gso(struct ovpn_peer *peer,
 			  struct ovpn_crypto_key_slot *ks,
 			  struct sk_buff *skb, struct sk_buff *gso_skb,
-			  unsigned int offset);
+			  unsigned int offset, u32 pktid);
 int ovpn_aead_decrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 		      struct sk_buff *skb);
 

index d544ccb24283..4caf1837b59e 100644
@@ -376,20 +376,31 @@ static bool ovpn_encrypt_gso_queue(struct sk_buff_head *skbs,
 {
 	unsigned int offset = 0, i, len;
 	struct sk_buff *skb;
+	u32 first_pktid;
 	int ret;
 
+	/* reserve the complete range before dequeuing any input skb so that a
+	 * failure leaves the ordinary transmit path available as a fallback
+	 */
+	if (unlikely(ovpn_pktid_xmit_range(&ks->pid_xmit, segments,
+					   &first_pktid))) {
+		ovpn_encrypt_key_exhausted(peer, ks);
+		ovpn_crypto_key_slot_put(ks);
+		ovpn_peer_put(peer);
+		return false;
+	}
+
 	for (i = 0; i < segments; i++) {
 		skb = __skb_dequeue(skbs);
 		len = skb->len + OVPN_DATA_V2_OVERHEAD;
 
 		memset(ovpn_skb_cb(skb), 0, sizeof(struct ovpn_cb));
-		ret = ovpn_aead_encrypt_gso(peer, ks, skb, batch, offset);
+		ret = ovpn_aead_encrypt_gso(peer, ks, skb, batch, offset,
+					    first_pktid + i);
 		kfree(ovpn_skb_cb(skb)->crypto_tmp);
 		if (unlikely(ret < 0)) {
 			DEBUG_NET_WARN_ON_ONCE(ret == -EINPROGRESS ||
 					       ret == -EBUSY);
-			if (ret == -ERANGE)
-				ovpn_encrypt_key_exhausted(peer, ks);
 			ovpn_dev_dstats_tx_dropped(peer->ovpn->dev, i + 1);
 			kfree_skb(skb);
 			goto err;

54 lines not on this page · raw message

index a75a0bf65c89..763fd4f2cc25 100644
@@ -8,7 +8,9 @@
  */
 
 #include <crypto/aead.h>
+#include <linux/highmem.h>
 #include <linux/skbuff.h>
+#include <linux/unaligned.h>
 #include <linux/workqueue.h>
 #include <net/ip.h>
 #include <net/ipv6.h>

@@ -169,10 +171,33 @@ static void ovpn_aead_encrypt_header(struct ovpn_peer *peer,
 	/* add the packet opcode and wire nonce as associated data */
 	op = ovpn_opcode_compose(OVPN_DATA_V2, ks->key_id, peer->tx_id);
 	BUILD_BUG_ON(sizeof(op) != OVPN_OPCODE_SIZE);
-	*(__force __be32 *)data = htonl(op);
+	put_unaligned_be32(op, data);
 	memcpy(data + OVPN_OPCODE_SIZE, iv, OVPN_NONCE_WIRE_SIZE);
 }
 
+static bool ovpn_aead_encrypt_header_direct(struct ovpn_peer *peer,
+					    struct ovpn_crypto_key_slot *ks,
+					    u32 pktid, u8 *iv,
+					    struct scatterlist *sg)
+{
+	unsigned int offset, page_offset;
+	struct page *page;
+	void *addr;
+
+	offset = sg->offset;
+	page_offset = offset_in_page(offset);
+	if (page_offset + OVPN_AAD_SIZE > PAGE_SIZE)
+		return false;
+
+	/* an SG offset can address a later base page in a compound page */
+	page = sg_page(sg) + (offset >> PAGE_SHIFT);
+	addr = kmap_local_page(page);
+	ovpn_aead_encrypt_header(peer, ks, pktid, iv, addr + page_offset);
+	kunmap_local(addr);
+
+	return true;
+}
+
 int ovpn_aead_encrypt_gso(struct ovpn_peer *peer,
 			  struct ovpn_crypto_key_slot *ks, struct sk_buff *skb,
 			  struct sk_buff *gso_skb, unsigned int offset,

@@ -184,6 +209,7 @@ int ovpn_aead_encrypt_gso(struct ovpn_peer *peer,
 	struct aead_request *req;
 	unsigned int payload_off;
 	int dst_idx, mapped, ret;
+	bool direct_aad;
 	u8 *aad, *iv;
 
 	req = ovpn_aead_request_alloc(ks->encrypt, skb, src_nents + dst_nents,

@@ -194,26 +220,42 @@ int ovpn_aead_encrypt_gso(struct ovpn_peer *peer,
 	dst = src + src_nents;
 	aad = (u8 *)(dst + dst_nents);
 
-	ovpn_aead_encrypt_header(peer, ks, pktid, iv, aad);
-
-	ret = skb_store_bits(gso_skb, offset, aad, OVPN_AAD_SIZE);
-	if (unlikely(ret < 0))
-		return ret;
-
 	/* encrypt out of place from the original segmented skb directly into
 	 * its final range in the UDP GSO skb
 	 */
 	sg_init_table(src, src_nents);
-	sg_set_buf(src, aad, OVPN_AAD_SIZE);
+	/* use the final output bytes as source AAD too when they can be safely
+	 * written through one base-page mapping
+	 */
+	mapped = skb_to_sgvec_nomark(gso_skb, src, offset, OVPN_AAD_SIZE);
+	if (unlikely(mapped < 0))
+		return mapped;
+	direct_aad = mapped == 1 &&
+		     ovpn_aead_encrypt_header_direct(peer, ks, pktid, iv, src);
+	if (unlikely(!direct_aad)) {
+		ovpn_aead_encrypt_header(peer, ks, pktid, iv, aad);
+		ret = skb_store_bits(gso_skb, offset, aad, OVPN_AAD_SIZE);
+		if (unlikely(ret < 0))
+			return ret;
+		sg_set_buf(src, aad, OVPN_AAD_SIZE);
+	}
+
 	mapped = skb_to_sgvec_nomark(skb, src + 1, 0, skb->len);
 	if (unlikely(mapped < 0))
 		return mapped;
 	sg_mark_end(&src[mapped]);
 
 	sg_init_table(dst, dst_nents);
-	dst_idx = skb_to_sgvec_nomark(gso_skb, dst, offset, OVPN_AAD_SIZE);
-	if (unlikely(dst_idx < 0))
-		return dst_idx;
+	if (likely(direct_aad)) {
+		/* destination AAD is not written and may alias source AAD */
+		sg_set_page(dst, sg_page(src), src->length, src->offset);
+		dst_idx = 1;
+	} else {
+		dst_idx = skb_to_sgvec_nomark(gso_skb, dst, offset,
+					      OVPN_AAD_SIZE);
+		if (unlikely(dst_idx < 0))
+			return dst_idx;
+	}
 
 	payload_off = offset + OVPN_AAD_SIZE + OVPN_AUTH_TAG_SIZE;
 	mapped = skb_to_sgvec_nomark(gso_skb, dst + dst_idx, payload_off,

index 763fd4f2cc25..e27b8c47c0fb 100644
@@ -136,17 +136,13 @@ static struct scatterlist *ovpn_aead_crypto_req_sg(struct crypto_aead *aead,
 
 static struct aead_request *ovpn_aead_request_alloc(struct crypto_aead *aead,
 						    struct sk_buff *skb,
-						    unsigned int nents,
-						    unsigned int extra, u8 **iv)
+						    unsigned int nents, u8 **iv)
 {
 	struct aead_request *req;
 	void *tmp;
 
-	/* allocate IV, request, scatterlist entries and caller scratch space
-	 * in one block
-	 */
-	tmp = kmalloc(ovpn_aead_crypto_tmp_size(aead, nents) + extra,
-		      GFP_ATOMIC);
+	/* allocate IV, request and scatterlist entries in one block */
+	tmp = kmalloc(ovpn_aead_crypto_tmp_size(aead, nents), GFP_ATOMIC);
 	if (unlikely(!tmp))
 		return ERR_PTR(-ENOMEM);
 

@@ -198,86 +194,108 @@ static bool ovpn_aead_encrypt_header_direct(struct ovpn_peer *peer,
 	return true;
 }
 
-int ovpn_aead_encrypt_gso(struct ovpn_peer *peer,
+int ovpn_aead_gso_ctx_init(struct ovpn_aead_gso_ctx *ctx,
+			   struct ovpn_crypto_key_slot *ks,
+			   struct sk_buff *gso_skb)
+{
+	const unsigned int dst_nents = skb_shinfo(gso_skb)->nr_frags + 4;
+	const unsigned int src_nents = MAX_SKB_FRAGS + 2;
+	unsigned int tmp_size;
+
+	tmp_size = ovpn_aead_crypto_tmp_size(ks->encrypt,
+					     src_nents + dst_nents);
+	tmp_size += OVPN_AAD_SIZE;
+	ctx->tmp = kmalloc(tmp_size, GFP_ATOMIC);
+	if (unlikely(!ctx->tmp))
+		return -ENOMEM;
+
+	ctx->dst_nents = dst_nents;
+	ctx->iv = ovpn_aead_crypto_tmp_iv(ks->encrypt, ctx->tmp);
+	ctx->req = ovpn_aead_crypto_tmp_req(ks->encrypt, ctx->iv);
+	ctx->src = ovpn_aead_crypto_req_sg(ks->encrypt, ctx->req);
+	ctx->dst = ctx->src + src_nents;
+	ctx->aad = (u8 *)(ctx->dst + dst_nents);
+	aead_request_set_tfm(ctx->req, ks->encrypt);
+	aead_request_set_callback(ctx->req, 0, NULL, NULL);
+	aead_request_set_ad(ctx->req, OVPN_AAD_SIZE);
+
+	return 0;
+}
+
+void ovpn_aead_gso_ctx_free(struct ovpn_aead_gso_ctx *ctx)
+{
+	kfree(ctx->tmp);
+}
+
+int ovpn_aead_encrypt_gso(struct ovpn_aead_gso_ctx *ctx, struct ovpn_peer *peer,
 			  struct ovpn_crypto_key_slot *ks, struct sk_buff *skb,
 			  struct sk_buff *gso_skb, unsigned int offset,
 			  u32 pktid)
 {
-	const unsigned int dst_nents = skb_shinfo(gso_skb)->nr_frags + 4;
-	const unsigned int src_nents = skb_shinfo(skb)->nr_frags + 2;
-	struct scatterlist *src, *dst;
-	struct aead_request *req;
 	unsigned int payload_off;
 	int dst_idx, mapped, ret;
 	bool direct_aad;
-	u8 *aad, *iv;
-
-	req = ovpn_aead_request_alloc(ks->encrypt, skb, src_nents + dst_nents,
-				      OVPN_AAD_SIZE, &iv);
-	if (IS_ERR(req))
-		return PTR_ERR(req);
-	src = ovpn_aead_crypto_req_sg(ks->encrypt, req);
-	dst = src + src_nents;
-	aad = (u8 *)(dst + dst_nents);
 
 	/* encrypt out of place from the original segmented skb directly into
 	 * its final range in the UDP GSO skb
 	 */
-	sg_init_table(src, src_nents);
+	sg_init_table(ctx->src, skb_shinfo(skb)->nr_frags + 2);
 	/* use the final output bytes as source AAD too when they can be safely
 	 * written through one base-page mapping
 	 */
-	mapped = skb_to_sgvec_nomark(gso_skb, src, offset, OVPN_AAD_SIZE);
+	mapped = skb_to_sgvec_nomark(gso_skb, ctx->src, offset,
+				     OVPN_AAD_SIZE);
 	if (unlikely(mapped < 0))
 		return mapped;
 	direct_aad = mapped == 1 &&
-		     ovpn_aead_encrypt_header_direct(peer, ks, pktid, iv, src);
+		     ovpn_aead_encrypt_header_direct(peer, ks, pktid, ctx->iv,
+						     ctx->src);
 	if (unlikely(!direct_aad)) {
-		ovpn_aead_encrypt_header(peer, ks, pktid, iv, aad);
-		ret = skb_store_bits(gso_skb, offset, aad, OVPN_AAD_SIZE);
+		ovpn_aead_encrypt_header(peer, ks, pktid, ctx->iv, ctx->aad);
+		ret = skb_store_bits(gso_skb, offset, ctx->aad,
+				     OVPN_AAD_SIZE);
 		if (unlikely(ret < 0))
 			return ret;
-		sg_set_buf(src, aad, OVPN_AAD_SIZE);
+		sg_set_buf(ctx->src, ctx->aad, OVPN_AAD_SIZE);
 	}
 
-	mapped = skb_to_sgvec_nomark(skb, src + 1, 0, skb->len);
+	mapped = skb_to_sgvec_nomark(skb, ctx->src + 1, 0, skb->len);
 	if (unlikely(mapped < 0))
 		return mapped;
-	sg_mark_end(&src[mapped]);
+	sg_mark_end(&ctx->src[mapped]);
 
-	sg_init_table(dst, dst_nents);
+	sg_init_table(ctx->dst, ctx->dst_nents);
 	if (likely(direct_aad)) {
 		/* destination AAD is not written and may alias source AAD */
-		sg_set_page(dst, sg_page(src), src->length, src->offset);
+		sg_set_page(ctx->dst, sg_page(ctx->src), ctx->src->length,
+			    ctx->src->offset);
 		dst_idx = 1;
 	} else {
-		dst_idx = skb_to_sgvec_nomark(gso_skb, dst, offset,
+		dst_idx = skb_to_sgvec_nomark(gso_skb, ctx->dst, offset,
 					      OVPN_AAD_SIZE);
 		if (unlikely(dst_idx < 0))
 			return dst_idx;
 	}
 
 	payload_off = offset + OVPN_AAD_SIZE + OVPN_AUTH_TAG_SIZE;
-	mapped = skb_to_sgvec_nomark(gso_skb, dst + dst_idx, payload_off,
-				     skb->len);
+	mapped = skb_to_sgvec_nomark(gso_skb, ctx->dst + dst_idx,
+				     payload_off, skb->len);
 	if (unlikely(mapped < 0))
 		return mapped;
 	dst_idx += mapped;
 
-	mapped = skb_to_sgvec_nomark(gso_skb, dst + dst_idx,
+	mapped = skb_to_sgvec_nomark(gso_skb, ctx->dst + dst_idx,
 				     offset + OVPN_AAD_SIZE,
 				     OVPN_AUTH_TAG_SIZE);
 	if (unlikely(mapped < 0))
 		return mapped;
 	dst_idx += mapped;
-	sg_mark_end(&dst[dst_idx - 1]);
+	sg_mark_end(&ctx->dst[dst_idx - 1]);
 
-	aead_request_set_tfm(req, ks->encrypt);
-	aead_request_set_callback(req, 0, NULL, NULL);
-	aead_request_set_crypt(req, src, dst, skb->len, iv);
-	aead_request_set_ad(req, OVPN_AAD_SIZE);
+	aead_request_set_crypt(ctx->req, ctx->src, ctx->dst, skb->len,
+			       ctx->iv);
 
-	return crypto_aead_encrypt(req);
+	return crypto_aead_encrypt(ctx->req);
 }
 
 int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,

@@ -319,7 +337,7 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 		nfrags = 1;
 	}
 
-	req = ovpn_aead_request_alloc(ks->encrypt, skb, nfrags + 2, 0, &iv);
+	req = ovpn_aead_request_alloc(ks->encrypt, skb, nfrags + 2, &iv);
 	if (IS_ERR(req))
 		return PTR_ERR(req);
 	sg = ovpn_aead_crypto_req_sg(ks->encrypt, req);

index 92e8ed76fba6..c0c777410661 100644
@@ -15,12 +15,29 @@
 #include <asm/types.h>
 #include <linux/skbuff.h>
 
+struct aead_request;
+struct scatterlist;
+
+struct ovpn_aead_gso_ctx {
+	struct aead_request *req;
+	struct scatterlist *src;
+	struct scatterlist *dst;
+	void *tmp;
+	u8 *iv;
+	u8 *aad;
+	unsigned int dst_nents;
+};
+
 int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 		      struct sk_buff *skb);
-int ovpn_aead_encrypt_gso(struct ovpn_peer *peer,
-			  struct ovpn_crypto_key_slot *ks,
-			  struct sk_buff *skb, struct sk_buff *gso_skb,
-			  unsigned int offset, u32 pktid);
+int ovpn_aead_gso_ctx_init(struct ovpn_aead_gso_ctx *ctx,
+			   struct ovpn_crypto_key_slot *ks,
+			   struct sk_buff *gso_skb);
+void ovpn_aead_gso_ctx_free(struct ovpn_aead_gso_ctx *ctx);
+int ovpn_aead_encrypt_gso(struct ovpn_aead_gso_ctx *ctx, struct ovpn_peer *peer,
+			  struct ovpn_crypto_key_slot *ks, struct sk_buff *skb,
+			  struct sk_buff *gso_skb, unsigned int offset,
+			  u32 pktid);
 int ovpn_aead_decrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 		      struct sk_buff *skb);
 

index 4caf1837b59e..cd50f7c206a6 100644
@@ -374,30 +374,30 @@ static bool ovpn_encrypt_gso_queue(struct sk_buff_head *skbs,
 				   struct sk_buff *batch,
 				   unsigned int segments)
 {
+	struct ovpn_aead_gso_ctx ctx = { };
 	unsigned int offset = 0, i, len;
 	struct sk_buff *skb;
 	u32 first_pktid;
 	int ret;
 
+	if (unlikely(ovpn_aead_gso_ctx_init(&ctx, ks, batch)))
+		goto err;
+
 	/* reserve the complete range before dequeuing any input skb so that a
 	 * failure leaves the ordinary transmit path available as a fallback
 	 */
 	if (unlikely(ovpn_pktid_xmit_range(&ks->pid_xmit, segments,
 					   &first_pktid))) {
 		ovpn_encrypt_key_exhausted(peer, ks);
-		ovpn_crypto_key_slot_put(ks);
-		ovpn_peer_put(peer);
-		return false;
+		goto err;
 	}
 
 	for (i = 0; i < segments; i++) {
 		skb = __skb_dequeue(skbs);
 		len = skb->len + OVPN_DATA_V2_OVERHEAD;
 
-		memset(ovpn_skb_cb(skb), 0, sizeof(struct ovpn_cb));
-		ret = ovpn_aead_encrypt_gso(peer, ks, skb, batch, offset,
+		ret = ovpn_aead_encrypt_gso(&ctx, peer, ks, skb, batch, offset,
 					    first_pktid + i);
-		kfree(ovpn_skb_cb(skb)->crypto_tmp);
 		if (unlikely(ret < 0)) {
 			DEBUG_NET_WARN_ON_ONCE(ret == -EINPROGRESS ||
 					       ret == -EBUSY);

@@ -411,9 +411,11 @@ static bool ovpn_encrypt_gso_queue(struct sk_buff_head *skbs,
 		offset += len;
 	}
 
+	ovpn_aead_gso_ctx_free(&ctx);
 	ovpn_encrypt_send(peer, ks, batch, segments);
 	return true;
 err:
+	ovpn_aead_gso_ctx_free(&ctx);
 	ovpn_crypto_key_slot_put(ks);
 	ovpn_peer_put(peer);
 	return false;