Skip to contentSkip to conversation

net/packet: better behavior under DDOS

PATCH In review Eric Dumazet 4 participants 12 messages Jun 12, 2019 – Jun 15, 2019

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

cover net/packet: better behavior under DDOS

Using tcpdump (or other af_packet user) on a busy host can lead to catastrophic consequences, because suddenly, potentially all cpus are spinning on a contended spinlock.

Both packet_rcv() and tpacket_rcv() grab the spinlock to eventually find there is no room for an additional packet.

This patch series align packet_rcv() and tpacket_rcv() to both check if the queue is full before grabbing the spinlock.

If the queue is full, they both increment a new atomic counter placed on a separate cache line to let readers drain the queue faster.

There is still false sharing on this new atomic counter, we might in the future make it per cpu if there is interest.

Eric Dumazet (8):
net/packet: constify __packet_get_status() argument net/packet: constify packet_lookup_frame() and __tpacket_has_room() net/packet: constify prb_lookup_block() and __tpacket_v3_has_room() net/packet: constify __packet_rcv_has_room()
net/packet: make tp_drops atomic
net/packet: implement shortcut in tpacket_rcv() net/packet: remove locking from packet_rcv_has_room() net/packet: introduce packet_rcv_try_clear_pressure() helper

 net/packet/af_packet.c | 96 ++++++++++++++++++++++++------------------
 net/packet/internal.h  |  1 +
 2 files changed, 56 insertions(+), 41 deletions(-)
Signature · 1 line
2.22.0.rc2.383.gf4fbbf30c2-goog
#1 lore raw

PATCH 1/8 net/packet: constify __packet_get_status() argument

struct packet_sock is only read.

---
 net/packet/af_packet.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
net/packet/af_packet.c +1 -1
index 7fa847dcea30e481b2f291cc6980a7b887629cd7..66fcfd5b51f82a861795e002e91d3cbc69ab545a 100644
@@ -384,7 +384,7 @@ static void __packet_set_status(struct packet_sock *po, void *frame, int status)
 	smp_wmb();
 }
 
-static int __packet_get_status(struct packet_sock *po, void *frame)
+static int __packet_get_status(const struct packet_sock *po, void *frame)
 {
 	union tpacket_uhdr h;
 
Signature · 1 line
2.22.0.rc2.383.gf4fbbf30c2-goog
#2 lore raw

PATCH 2/8 net/packet: constify packet_lookup_frame() and __tpacket_has_room()

In reply to Eric Dumazet

Goal is to be able to use __tpacket_has_room() without holding a lock.

---
 net/packet/af_packet.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)
net/packet/af_packet.c +7 -7
index 66fcfd5b51f82a861795e002e91d3cbc69ab545a..273bffd2130d36cceb9947540a8511a51895874a 100644
@@ -460,10 +460,10 @@ static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
 	return ts_status;
 }
 
-static void *packet_lookup_frame(struct packet_sock *po,
-		struct packet_ring_buffer *rb,
-		unsigned int position,
-		int status)
+static void *packet_lookup_frame(const struct packet_sock *po,
+				 const struct packet_ring_buffer *rb,
+				 unsigned int position,
+				 int status)
 {
 	unsigned int pg_vec_pos, frame_offset;
 	union tpacket_uhdr h;
@@ -1198,12 +1198,12 @@ static void packet_free_pending(struct packet_sock *po)
 #define ROOM_LOW	0x1
 #define ROOM_NORMAL	0x2
 
-static bool __tpacket_has_room(struct packet_sock *po, int pow_off)
+static bool __tpacket_has_room(const struct packet_sock *po, int pow_off)
 {
 	int idx, len;
 
-	len = po->rx_ring.frame_max + 1;
-	idx = po->rx_ring.head;
+	len = READ_ONCE(po->rx_ring.frame_max) + 1;
+	idx = READ_ONCE(po->rx_ring.head);
 	if (pow_off)
 		idx += len >> pow_off;
 	if (idx >= len)
Signature · 1 line
2.22.0.rc2.383.gf4fbbf30c2-goog
#3 lore raw

PATCH 3/8 net/packet: constify prb_lookup_block() and __tpacket_v3_has_room()

In reply to Eric Dumazet

Goal is to be able to use __tpacket_v3_has_room() without holding a lock.

---
 net/packet/af_packet.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)
net/packet/af_packet.c +7 -7
index 273bffd2130d36cceb9947540a8511a51895874a..5ef63d0c3ad0a184a03429fdd52cad26349647d1 100644
@@ -1082,10 +1082,10 @@ static void *packet_current_rx_frame(struct packet_sock *po,
 	}
 }
 
-static void *prb_lookup_block(struct packet_sock *po,
-				     struct packet_ring_buffer *rb,
-				     unsigned int idx,
-				     int status)
+static void *prb_lookup_block(const struct packet_sock *po,
+			      const struct packet_ring_buffer *rb,
+			      unsigned int idx,
+			      int status)
 {
 	struct tpacket_kbdq_core *pkc  = GET_PBDQC_FROM_RB(rb);
 	struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
@@ -1211,12 +1211,12 @@ static bool __tpacket_has_room(const struct packet_sock *po, int pow_off)
 	return packet_lookup_frame(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
 }
 
-static bool __tpacket_v3_has_room(struct packet_sock *po, int pow_off)
+static bool __tpacket_v3_has_room(const struct packet_sock *po, int pow_off)
 {
 	int idx, len;
 
-	len = po->rx_ring.prb_bdqc.knum_blocks;
-	idx = po->rx_ring.prb_bdqc.kactive_blk_num;
+	len = READ_ONCE(po->rx_ring.prb_bdqc.knum_blocks);
+	idx = READ_ONCE(po->rx_ring.prb_bdqc.kactive_blk_num);
 	if (pow_off)
 		idx += len >> pow_off;
 	if (idx >= len)
Signature · 1 line
2.22.0.rc2.383.gf4fbbf30c2-goog
#4 lore raw

PATCH 4/8 net/packet: constify __packet_rcv_has_room()

In reply to Eric Dumazet

Goal is use the helper without lock being held.

---
 net/packet/af_packet.c | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)
net/packet/af_packet.c +8 -5
index 5ef63d0c3ad0a184a03429fdd52cad26349647d1..a0564855ed9dca4be37f70ed81c6dee1b38aca39 100644
@@ -1224,15 +1224,18 @@ static bool __tpacket_v3_has_room(const struct packet_sock *po, int pow_off)
 	return prb_lookup_block(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
 }
 
-static int __packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
+static int __packet_rcv_has_room(const struct packet_sock *po,
+				 const struct sk_buff *skb)
 {
-	struct sock *sk = &po->sk;
+	const struct sock *sk = &po->sk;
 	int ret = ROOM_NONE;
 
 	if (po->prot_hook.func != tpacket_rcv) {
-		int avail = sk->sk_rcvbuf - atomic_read(&sk->sk_rmem_alloc)
-					  - (skb ? skb->truesize : 0);
-		if (avail > (sk->sk_rcvbuf >> ROOM_POW_OFF))
+		int rcvbuf = READ_ONCE(sk->sk_rcvbuf);
+		int avail = rcvbuf - atomic_read(&sk->sk_rmem_alloc)
+				   - (skb ? skb->truesize : 0);
+
+		if (avail > (rcvbuf >> ROOM_POW_OFF))
 			return ROOM_NORMAL;
 		else if (avail > 0)
 			return ROOM_LOW;
Signature · 1 line
2.22.0.rc2.383.gf4fbbf30c2-goog
#5 lore raw

PATCH 5/8 net/packet: make tp_drops atomic

In reply to Eric Dumazet

Under DDOS, we want to be able to increment tp_drops without touching the spinlock. This will help readers to drain the receive queue slightly faster :/

---
 net/packet/af_packet.c | 20 +++++++++++---------
 net/packet/internal.h  |  1 +
 2 files changed, 12 insertions(+), 9 deletions(-)
net/packet/af_packet.c +11 -9
index a0564855ed9dca4be37f70ed81c6dee1b38aca39..2d499679811af53886ce0c8a1cdd74cd73107eac 100644
@@ -758,7 +758,7 @@ static void prb_close_block(struct tpacket_kbdq_core *pkc1,
 	struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
 	struct sock *sk = &po->sk;
 
-	if (po->stats.stats3.tp_drops)
+	if (atomic_read(&po->tp_drops))
 		status |= TP_STATUS_LOSING;
 
 	last_pkt = (struct tpacket3_hdr *)pkc1->prev;
@@ -2128,10 +2128,8 @@ static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
 
 drop_n_acct:
 	is_drop_n_account = true;
-	spin_lock(&sk->sk_receive_queue.lock);
-	po->stats.stats1.tp_drops++;
+	atomic_inc(&po->tp_drops);
 	atomic_inc(&sk->sk_drops);
-	spin_unlock(&sk->sk_receive_queue.lock);
 
 drop_n_restore:
 	if (skb_head != skb->data && skb_shared(skb)) {
@@ -2265,7 +2263,7 @@ static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
 	 * Anyways, moving it for V1/V2 only as V3 doesn't need this
 	 * at packet level.
 	 */
-		if (po->stats.stats1.tp_drops)
+		if (atomic_read(&po->tp_drops))
 			status |= TP_STATUS_LOSING;
 	}
 
@@ -2381,9 +2379,9 @@ static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
 	return 0;
 
 drop_n_account:
-	is_drop_n_account = true;
-	po->stats.stats1.tp_drops++;
 	spin_unlock(&sk->sk_receive_queue.lock);
+	atomic_inc(&po->tp_drops);
+	is_drop_n_account = true;
 
 	sk->sk_data_ready(sk);
 	kfree_skb(copy_skb);
@@ -3879,6 +3877,7 @@ static int packet_getsockopt(struct socket *sock, int level, int optname,
 	void *data = &val;
 	union tpacket_stats_u st;
 	struct tpacket_rollover_stats rstats;
+	int drops;
 
 	if (level != SOL_PACKET)
 		return -ENOPROTOOPT;
@@ -3895,14 +3894,17 @@ static int packet_getsockopt(struct socket *sock, int level, int optname,
 		memcpy(&st, &po->stats, sizeof(st));
 		memset(&po->stats, 0, sizeof(po->stats));
 		spin_unlock_bh(&sk->sk_receive_queue.lock);
+		drops = atomic_xchg(&po->tp_drops, 0);
 
 		if (po->tp_version == TPACKET_V3) {
 			lv = sizeof(struct tpacket_stats_v3);
-			st.stats3.tp_packets += st.stats3.tp_drops;
+			st.stats3.tp_drops = drops;
+			st.stats3.tp_packets += drops;
 			data = &st.stats3;
 		} else {
 			lv = sizeof(struct tpacket_stats);
-			st.stats1.tp_packets += st.stats1.tp_drops;
+			st.stats1.tp_drops = drops;
+			st.stats1.tp_packets += drops;
 			data = &st.stats1;
 		}
 
net/packet/internal.h +1 -0
index 3bb7c5fb3bff2fd5d91c3d973d006d0cdde29a0b..b5bcff2b7a43b6c9cece329c8fe8b9b546b06cc5 100644
@@ -131,6 +131,7 @@ struct packet_sock {
 	struct net_device __rcu	*cached_dev;
 	int			(*xmit)(struct sk_buff *skb);
 	struct packet_type	prot_hook ____cacheline_aligned_in_smp;
+	atomic_t		tp_drops ____cacheline_aligned_in_smp;
 };
 
 static struct packet_sock *pkt_sk(struct sock *sk)
Signature · 1 line
2.22.0.rc2.383.gf4fbbf30c2-goog
#6 lore raw

PATCH 6/8 net/packet: implement shortcut in tpacket_rcv()

In reply to Eric Dumazet

tpacket_rcv() can be hit under DDOS quite hard, since it will always grab a socket spinlock, to eventually find there is no room for an additional packet.

Using tcpdump [1] on a busy host can lead to catastrophic consequences, because of all cpus spinning on a contended spinlock.

This replicates a similar strategy used in packet_rcv()

[1] Also some applications mistakenly use af_packet socket bound to ETH_P_ALL only to send packets.
Receive queue is never drained and immediately full.

---
 net/packet/af_packet.c | 6 ++++++
 1 file changed, 6 insertions(+)
net/packet/af_packet.c +6 -0
index 2d499679811af53886ce0c8a1cdd74cd73107eac..860ca3e6abf5198214612e9acc095530b61dac40 100644
@@ -2193,6 +2193,12 @@ static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
 	if (!res)
 		goto drop_n_restore;
 
+	/* If we are flooded, just give up */
+	if (__packet_rcv_has_room(po, skb) == ROOM_NONE) {
+		atomic_inc(&po->tp_drops);
+		goto drop_n_restore;
+	}
+
 	if (skb->ip_summed == CHECKSUM_PARTIAL)
 		status |= TP_STATUS_CSUMNOTREADY;
 	else if (skb->pkt_type != PACKET_OUTGOING &&
Signature · 1 line
2.22.0.rc2.383.gf4fbbf30c2-goog
#7 lore raw

PATCH 7/8 net/packet: remove locking from packet_rcv_has_room()

In reply to Eric Dumazet

__packet_rcv_has_room() can now be run without lock being held.

po->pressure is only a non persistent hint, we can mark all read/write accesses with READ_ONCE()/WRITE_ONCE() to document the fact that the field could be written without any synchronization.

---
 net/packet/af_packet.c | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)
net/packet/af_packet.c +9 -11
index 860ca3e6abf5198214612e9acc095530b61dac40..d409e2fdaa7ee8ddf261354f91b682e403f40e9e 100644
@@ -1260,15 +1260,13 @@ static int __packet_rcv_has_room(const struct packet_sock *po,
 
 static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
 {
-	int ret;
-	bool has_room;
+	int pressure, ret;
 
-	spin_lock_bh(&po->sk.sk_receive_queue.lock);
 	ret = __packet_rcv_has_room(po, skb);
-	has_room = ret == ROOM_NORMAL;
-	if (po->pressure == has_room)
-		po->pressure = !has_room;
-	spin_unlock_bh(&po->sk.sk_receive_queue.lock);
+	pressure = ret != ROOM_NORMAL;
+
+	if (READ_ONCE(po->pressure) != pressure)
+		WRITE_ONCE(po->pressure, pressure);
 
 	return ret;
 }
@@ -1353,7 +1351,7 @@ static unsigned int fanout_demux_rollover(struct packet_fanout *f,
 	i = j = min_t(int, po->rollover->sock, num - 1);
 	do {
 		po_next = pkt_sk(f->arr[i]);
-		if (po_next != po_skip && !po_next->pressure &&
+		if (po_next != po_skip && !READ_ONCE(po_next->pressure) &&
 		    packet_rcv_has_room(po_next, skb) == ROOM_NORMAL) {
 			if (i != j)
 				po->rollover->sock = i;
@@ -3310,7 +3308,7 @@ static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
 	if (skb == NULL)
 		goto out;
 
-	if (pkt_sk(sk)->pressure)
+	if (READ_ONCE(pkt_sk(sk)->pressure))
 		packet_rcv_has_room(pkt_sk(sk), NULL);
 
 	if (pkt_sk(sk)->has_vnet_hdr) {
@@ -4129,8 +4127,8 @@ static __poll_t packet_poll(struct file *file, struct socket *sock,
 			TP_STATUS_KERNEL))
 			mask |= EPOLLIN | EPOLLRDNORM;
 	}
-	if (po->pressure && __packet_rcv_has_room(po, NULL) == ROOM_NORMAL)
-		po->pressure = 0;
+	if (READ_ONCE(po->pressure) && __packet_rcv_has_room(po, NULL) == ROOM_NORMAL)
+		WRITE_ONCE(po->pressure, 0);
 	spin_unlock_bh(&sk->sk_receive_queue.lock);
 	spin_lock_bh(&sk->sk_write_queue.lock);
 	if (po->tx_ring.pg_vec) {
Signature · 1 line
2.22.0.rc2.383.gf4fbbf30c2-goog
#8 lore raw

PATCH 8/8 net/packet: introduce packet_rcv_try_clear_pressure() helper

In reply to Eric Dumazet

There are two places where we want to clear the pressure if possible, add a helper to make it more obvious.

---
 net/packet/af_packet.c | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)
net/packet/af_packet.c +9 -4
index d409e2fdaa7ee8ddf261354f91b682e403f40e9e..8c27e198268ab5148daa8e90aa2f53546623b9ed 100644
@@ -1271,6 +1271,13 @@ static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
 	return ret;
 }
 
+static void packet_rcv_try_clear_pressure(struct packet_sock *po)
+{
+	if (READ_ONCE(po->pressure) &&
+	    __packet_rcv_has_room(po, NULL) == ROOM_NORMAL)
+		WRITE_ONCE(po->pressure,  0);
+}
+
 static void packet_sock_destruct(struct sock *sk)
 {
 	skb_queue_purge(&sk->sk_error_queue);
@@ -3308,8 +3315,7 @@ static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
 	if (skb == NULL)
 		goto out;
 
-	if (READ_ONCE(pkt_sk(sk)->pressure))
-		packet_rcv_has_room(pkt_sk(sk), NULL);
+	packet_rcv_try_clear_pressure(pkt_sk(sk));
 
 	if (pkt_sk(sk)->has_vnet_hdr) {
 		err = packet_rcv_vnet(msg, skb, &len);
@@ -4127,8 +4133,7 @@ static __poll_t packet_poll(struct file *file, struct socket *sock,
 			TP_STATUS_KERNEL))
 			mask |= EPOLLIN | EPOLLRDNORM;
 	}
-	if (READ_ONCE(po->pressure) && __packet_rcv_has_room(po, NULL) == ROOM_NORMAL)
-		WRITE_ONCE(po->pressure, 0);
+	packet_rcv_try_clear_pressure(po);
 	spin_unlock_bh(&sk->sk_receive_queue.lock);
 	spin_lock_bh(&sk->sk_write_queue.lock);
 	if (po->tx_ring.pg_vec) {
Signature · 1 line
2.22.0.rc2.383.gf4fbbf30c2-goog
Acked-by on v1 8/8
#9 lore raw

Re: [PATCH net-next 8/8] net/packet: introduce packet_rcv_try_clear_pressure() helper

Hi,

Quoted 14 lines · Eric Dumazet

There are two places where we want to clear the pressure if possible, add a helper to make it more obvious.

---
 net/packet/af_packet.c | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)
net/packet/af_packet.c +0 -0
index d409e2fdaa7ee8ddf261354f91b682e403f40e9e..8c27e198268ab5148daa8e90aa2f53546623b9ed 100644

net/packet/af_packet.c static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb) · line 1278

Quoted 8 lines
net/packet/af_packet.c +5 -0
@@ -1271,6 +1271,13 @@ static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
 	return ret;
 }
 
+static void packet_rcv_try_clear_pressure(struct packet_sock *po)
+{
+	if (READ_ONCE(po->pressure) &&
+	    __packet_rcv_has_room(po, NULL) == ROOM_NORMAL)
+		WRITE_ONCE(po->pressure,  0);

Just a couple of (microscopical?) nitpicks, double space here and on the commit message of patch 1/8.

Series look good.

Cheers,
--
Vinicius

maintainer Acked-by on the v1 series
#10 lore raw
In reply to Eric Dumazet
Quoted 27 lines · Eric Dumazet

Using tcpdump (or other af_packet user) on a busy host can lead to catastrophic consequences, because suddenly, potentially all cpus are spinning on a contended spinlock.

Both packet_rcv() and tpacket_rcv() grab the spinlock to eventually find there is no room for an additional packet.

This patch series align packet_rcv() and tpacket_rcv() to both check if the queue is full before grabbing the spinlock.

If the queue is full, they both increment a new atomic counter placed on a separate cache line to let readers drain the queue faster.

There is still false sharing on this new atomic counter, we might in the future make it per cpu if there is interest.

Eric Dumazet (8):
net/packet: constify __packet_get_status() argument net/packet: constify packet_lookup_frame() and __tpacket_has_room() net/packet: constify prb_lookup_block() and __tpacket_v3_has_room() net/packet: constify __packet_rcv_has_room()
net/packet: make tp_drops atomic
net/packet: implement shortcut in tpacket_rcv() net/packet: remove locking from packet_rcv_has_room() net/packet: introduce packet_rcv_try_clear_pressure() helper

Quoted 1 line
 net/packet/af_packet.c | 96 ++++++++++++++++++++++++-----------

For the series:

Thanks Eric

maintainer
#11 lore raw
In reply to Eric Dumazet

From: Eric Dumazet <[email protected]>
Date: Wed, 12 Jun 2019 09:52:25 -0700

Quoted 7 lines

Using tcpdump (or other af_packet user) on a busy host can lead to catastrophic consequences, because suddenly, potentially all cpus are spinning on a contended spinlock.

Both packet_rcv() and tpacket_rcv() grab the spinlock to eventually find there is no room for an additional packet.

Quoted 8 lines

This patch series align packet_rcv() and tpacket_rcv() to both check if the queue is full before grabbing the spinlock.

If the queue is full, they both increment a new atomic counter placed on a separate cache line to let readers drain the queue faster.

There is still false sharing on this new atomic counter, we might in the future make it per cpu if there is interest.

Series applied.