Skip to contentSkip to conversation

bpf: sockmap fixes

PATCH Superseded John Fastabend 2 participants 6 messages Jan 29, 2018 – Feb 1, 2018

Subsystems (5)
Networking Network drivers BPF TCP +1 more subsystems

v3 of 4 Superseded Latest is v4 →

lore
Share & tools
lore

John Fastabend

author maintainer
#0 lore raw

v3 cover bpf: sockmap fixes

A set of fixes for sockmap to resolve programs referencing sockmaps and closing without deleting all entries in the map and/or not detaching BPF programs attached to the map. Both leaving entries in the map and not detaching programs may result in the map failing to be removed by BPF infrastructure due to reference counts never reaching zero.

For this we pull in the ULP infrastructure to hook into the close() hook of the sock layer. This seemed natural because we have additional sockmap features (to add support for TX hooks) that will also use the ULP infrastructure. This allows us to cleanup entries in the map when socks are closed() and avoid trying to get the sk_state_change() hook to fire in all cases.

The second issue resolved here occurs when users don't detach programs. The gist is a refcnt issue resolved by implementing the release callback. See patch for details.

For testing I ran both sample/sockmap and selftests bpf/test_maps.c. Dave Watson ran TLS test suite on v1 version of the patches without the put_module error path change.

v3 wrap psock reference in RCU
v2 changes rebased onto bpf-next with the following small updates

Patch 1/3 Dave Watson pointed out that we should do a module put in case future users use this with another ULP other than sockmap. And module_put is a nop when owner = NULL (sockmap case) so no harm here and code is more robust.

Patch 2/3 Be explicit and set user_visible to false just to avoid any reader confusion.

---

John Fastabend (3):
net: add a UID to use for ULP socket assignment
bpf: sockmap, add sock close() hook to remove socks
bpf: sockmap, fix leaking maps with attached but not detached progs

 include/net/tcp.h    |    8 ++
 kernel/bpf/sockmap.c |  175 +++++++++++++++++++++++++++++---------------------
 net/ipv4/tcp_ulp.c   |   57 +++++++++++++++-
 net/tls/tls_main.c   |    2 +
 4 files changed, 165 insertions(+), 77 deletions(-)
author maintainer
#1 lore raw

PATCH v3 1/3 net: add a UID to use for ULP socket assignment

Create a UID field and enum that can be used to assign ULPs to sockets. This saves a set of string comparisons if the ULP id is known.

For sockmap, which is added in the next patches, a ULP is used to hook into TCP sockets close state. In this case the ULP being added is done at map insert time and the ULP is known and done on the kernel side. In this case the named lookup is not needed. Because we don't want to expose psock internals to user space socket options a user visible flag is also added. For TLS this is set for BPF it will be cleared.

Alos remove pr_notice, user gets an error code back and should check that rather than rely on logs.

---
 include/net/tcp.h  |    6 +++++
 net/ipv4/tcp_ulp.c |   57 +++++++++++++++++++++++++++++++++++++++++++++++-----
 net/tls/tls_main.c |    2 ++
 3 files changed, 60 insertions(+), 5 deletions(-)
include/net/tcp.h +6 -0
index 093e967..ba10ca7 100644
@@ -1983,6 +1983,10 @@ static inline void tcp_listendrop(const struct sock *sk)
 #define TCP_ULP_MAX		128
 #define TCP_ULP_BUF_MAX		(TCP_ULP_NAME_MAX*TCP_ULP_MAX)
 
+enum {
+	TCP_ULP_TLS,
+};
+
 struct tcp_ulp_ops {
 	struct list_head	list;
 
@@ -1991,7 +1995,9 @@ struct tcp_ulp_ops {
 	/* cleanup ulp */
 	void (*release)(struct sock *sk);
 
+	int		uid;
 	char		name[TCP_ULP_NAME_MAX];
+	bool		user_visible;
 	struct module	*owner;
 };
 int tcp_register_ulp(struct tcp_ulp_ops *type);
net/ipv4/tcp_ulp.c +52 -5

83 lines not on this page · raw message

net/tls/tls_main.c +2 -0
index 736719c..b0d5fce 100644
@@ -484,6 +484,8 @@ static int tls_init(struct sock *sk)
 
 static struct tcp_ulp_ops tcp_tls_ulp_ops __read_mostly = {
 	.name			= "tls",
+	.uid			= TCP_ULP_TLS,
+	.user_visible		= true,
 	.owner			= THIS_MODULE,
 	.init			= tls_init,
 };
maintainer
#2 lore raw

Re: [bpf-next PATCH v3 1/3] net: add a UID to use for ULP socket assignment

Quoted 71 lines · John Fastabend

Create a UID field and enum that can be used to assign ULPs to sockets. This saves a set of string comparisons if the ULP id is known.

For sockmap, which is added in the next patches, a ULP is used to hook into TCP sockets close state. In this case the ULP being added is done at map insert time and the ULP is known and done on the kernel side. In this case the named lookup is not needed. Because we don't want to expose psock internals to user space socket options a user visible flag is also added. For TLS this is set for BPF it will be cleared.

Alos remove pr_notice, user gets an error code back and should check that rather than rely on logs.

---
 include/net/tcp.h  |    6 +++++
 net/ipv4/tcp_ulp.c |   57 +++++++++++++++++++++++++++++++++++++++++++++++-----
 net/tls/tls_main.c |    2 ++
 3 files changed, 60 insertions(+), 5 deletions(-)
include/net/tcp.h +6 -0
index 093e967..ba10ca7 100644
@@ -1983,6 +1983,10 @@ static inline void tcp_listendrop(const struct sock *sk)
 #define TCP_ULP_MAX		128
 #define TCP_ULP_BUF_MAX		(TCP_ULP_NAME_MAX*TCP_ULP_MAX)
 
+enum {
+	TCP_ULP_TLS,
+};
+
 struct tcp_ulp_ops {
 	struct list_head	list;
 
@@ -1991,7 +1995,9 @@ struct tcp_ulp_ops {
 	/* cleanup ulp */
 	void (*release)(struct sock *sk);
 
+	int		uid;
 	char		name[TCP_ULP_NAME_MAX];
+	bool		user_visible;
 	struct module	*owner;
 };
 int tcp_register_ulp(struct tcp_ulp_ops *type);
net/ipv4/tcp_ulp.c +12 -0
index 6bb9e14..6d87e7a 100644
@@ -29,6 +29,18 @@ static struct tcp_ulp_ops *tcp_ulp_find(const char *name)
 	return NULL;
 }
 
+static struct tcp_ulp_ops *tcp_ulp_find_id(const int ulp)
+{
+	struct tcp_ulp_ops *e;
+
+	list_for_each_entry_rcu(e, &tcp_ulp_list, list) {
+		if (e->uid == ulp)
+			return e;
+	}
+
+	return NULL;
+}
+
 static const struct tcp_ulp_ops *__tcp_ulp_find_autoload(const char *name)
 {
 	const struct tcp_ulp_ops *ulp = NULL;

net/ipv4/tcp_ulp.c static const struct tcp_ulp_ops *__tcp_ulp_find_autoload(const char *name) · line 68

Quoted 6 lines
net/ipv4/tcp_ulp.c +3 -0
@@ -51,6 +63,18 @@ static const struct tcp_ulp_ops *__tcp_ulp_find_autoload(const char *name)
 	return ulp;
 }
 
+static const struct tcp_ulp_ops *__tcp_ulp_lookup(const int uid)
+{
+	const struct tcp_ulp_ops *ulp = NULL;

(Tiny nit: the init to NULL is not needed.)

Quoted 28 lines
net/ipv4/tcp_ulp.c +10 -5
+	rcu_read_lock();
+	ulp = tcp_ulp_find_id(uid);
+	if (!ulp || !try_module_get(ulp->owner))
+		ulp = NULL;
+	rcu_read_unlock();
+	return ulp;
+}
+
 /* Attach new upper layer protocol to the list
  * of available protocols.
  */
@@ -59,13 +83,10 @@ int tcp_register_ulp(struct tcp_ulp_ops *ulp)
 	int ret = 0;
 
 	spin_lock(&tcp_ulp_list_lock);
-	if (tcp_ulp_find(ulp->name)) {
-		pr_notice("%s already registered or non-unique name\n",
-			  ulp->name);
+	if (tcp_ulp_find(ulp->name))
 		ret = -EEXIST;
-	} else {
+	else
 		list_add_tail_rcu(&ulp->list, &tcp_ulp_list);
-	}
 	spin_unlock(&tcp_ulp_list_lock);
 
 	return ret;

net/ipv4/tcp_ulp.c int tcp_set_ulp(struct sock *sk, const char *name) · line 148

Quoted 4 lines
net/ipv4/tcp_ulp.c +1 -0
@@ -124,6 +145,32 @@ int tcp_set_ulp(struct sock *sk, const char *name)
 	if (!ulp_ops)
 		return -ENOENT;
 
+	if (!ulp_ops->user_visible)

In this error path, a module_put(ulp_ops->owner) is still missing. The prior __tcp_ulp_find_autoload() tries to get a ref on the module as well, which we otherwise would leak (although not subject to the two current tls/bpf users, but should still be corrected).

Quoted 42 lines
net/ipv4/tcp_ulp.c +25 -0
+		return -ENOENT;
+
+	err = ulp_ops->init(sk);
+	if (err) {
+		module_put(ulp_ops->owner);
+		return err;
+	}
+
+	icsk->icsk_ulp_ops = ulp_ops;
+	return 0;
+}
+
+int tcp_set_ulp_id(struct sock *sk, int ulp)
+{
+	struct inet_connection_sock *icsk = inet_csk(sk);
+	const struct tcp_ulp_ops *ulp_ops;
+	int err;
+
+	if (icsk->icsk_ulp_ops)
+		return -EEXIST;
+
+	ulp_ops = __tcp_ulp_lookup(ulp);
+	if (!ulp_ops)
+		return -ENOENT;
+
 	err = ulp_ops->init(sk);
 	if (err) {
 		module_put(ulp_ops->owner);
net/tls/tls_main.c +2 -0
index 736719c..b0d5fce 100644
@@ -484,6 +484,8 @@ static int tls_init(struct sock *sk)
 
 static struct tcp_ulp_ops tcp_tls_ulp_ops __read_mostly = {
 	.name			= "tls",
+	.uid			= TCP_ULP_TLS,
+	.user_visible		= true,
 	.owner			= THIS_MODULE,
 	.init			= tls_init,
 };
author maintainer
#3 lore raw

PATCH v3 2/3 bpf: sockmap, add sock close() hook to remove socks

In reply to John Fastabend

The selftests test_maps program was leaving dangling BPF sockmap programs around because not all psock elements were removed from the map. The elements in turn hold a reference on the BPF program they are attached to causing BPF programs to stay open even after test_maps has completed.

The original intent was that sk_state_change() would be called when TCP socks went through TCP_CLOSE state. However, because socks may be in SOCK_DEAD state or the sock may be a listening socket the event is not always triggered.

To resolve this use the ULP infrastructure and register our own proto close() handler. This fixes the above case.

Fixes 174a79ff9515 ("bpf: sockmap with sk redirect support")
---
 include/net/tcp.h    |    2 +
 kernel/bpf/sockmap.c |  156 +++++++++++++++++++++++++++++---------------------
 2 files changed, 91 insertions(+), 67 deletions(-)
include/net/tcp.h +2 -0
index ba10ca7..e082101 100644
@@ -1985,6 +1985,7 @@ static inline void tcp_listendrop(const struct sock *sk)
 
 enum {
 	TCP_ULP_TLS,
+	TCP_ULP_BPF,
 };
 
 struct tcp_ulp_ops {
@@ -2003,6 +2004,7 @@ struct tcp_ulp_ops {
 int tcp_register_ulp(struct tcp_ulp_ops *type);
 void tcp_unregister_ulp(struct tcp_ulp_ops *type);
 int tcp_set_ulp(struct sock *sk, const char *name);
+int tcp_set_ulp_id(struct sock *sk, const int ulp);
 void tcp_get_available_ulp(char *buf, size_t len);
 void tcp_cleanup_ulp(struct sock *sk);
 
kernel/bpf/sockmap.c +89 -67

216 lines not on this page · raw message

maintainer
#4 lore raw

Re: [bpf-next PATCH v3 2/3] bpf: sockmap, add sock close() hook to remove socks

Quoted 6 lines · John Fastabend

The selftests test_maps program was leaving dangling BPF sockmap programs around because not all psock elements were removed from the map. The elements in turn hold a reference on the BPF program they are attached to causing BPF programs to stay open even after test_maps has completed.

Quoted 7 lines

The original intent was that sk_state_change() would be called when TCP socks went through TCP_CLOSE state. However, because socks may be in SOCK_DEAD state or the sock may be a listening socket the event is not always triggered.

To resolve this use the ULP infrastructure and register our own proto close() handler. This fixes the above case.

Great that we can get rid of smap_state_change() and piggy-back on ULP infra instead.

Quoted 3 lines
Fixes 174a79ff9515 ("bpf: sockmap with sk redirect support")

[...]

kernel/bpf/sockmap.c static inline struct smap_psock *smap_psock_sk(const struct sock *sk) · line 140

Quoted 11 lines
kernel/bpf/sockmap.c +11 -0
+void bpf_tcp_close(struct sock *sk, long timeout)
+{
+	void (*close_fun)(struct sock *sk, long timeout);
+	struct smap_psock_map_entry *e, *tmp;
+	struct smap_psock *psock;
+	struct sock *osk;
+
+	rcu_read_lock();
+	psock = smap_psock_sk(sk);
+	if (unlikely(!psock))
+		return sk->sk_prot->close(sk, timeout);

Here, we return with RCU read lock held, rcu_read_unlock() is missing. lockdep should probably notice this imbalance as well.

Quoted 19 lines
kernel/bpf/sockmap.c +19 -0
+	/* The psock may be destroyed anytime after exiting the RCU critial
+	 * section so by the time we use close_fun the psock may no longer
+	 * be valid. However, bpf_tcp_close is called with the sock lock
+	 * held so the close hook and sk are still valid.
+	 */
+	close_fun = psock->save_close;
+
+	write_lock_bh(&sk->sk_callback_lock);
+	list_for_each_entry_safe(e, tmp, &psock->maps, list) {
+		osk = cmpxchg(e->entry, sk, NULL);
+		if (osk == sk) {
+			list_del(&e->list);
+			smap_release_sock(psock, sk);
+		}
+	}
+	write_unlock_bh(&sk->sk_callback_lock);
+	rcu_read_unlock();
+	close_fun(sk, timeout);
+}
author maintainer
#5 lore raw

PATCH v3 3/3 bpf: sockmap, fix leaking maps with attached but not detached progs

In reply to John Fastabend

When a program is attached to a map we increment the program refcnt to ensure that the program is not removed while it is potentially being referenced from sockmap side. However, if this same program also references the map (this is a reasonably common pattern in my programs) then the verifier will also increment the maps refcnt from the verifier. This is to ensure the map doesn't get garbage collected while the program has a reference to it.

So we are left in a state where the map holds the refcnt on the program stopping it from being removed and releasing the map refcnt. And vice versa the program holds a refcnt on the map stopping it from releasing the refcnt on the prog.

All this is fine as long as users detach the program while the map fd is still around. But, if the user omits this detach command we are left with a dangling map we can no longer release.

To resolve this when the map fd is released decrement the program references and remove any reference from the map to the program. This fixes the issue with possibly dangling map and creates a user side API constraint. That is, the map fd must be held open for programs to be attached to a map.

Fixes 174a79ff9515 ("bpf: sockmap with sk redirect support")
---
 kernel/bpf/sockmap.c |   19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)
kernel/bpf/sockmap.c +14 -5
index 6c6c274..c8b25f1 100644
@@ -608,11 +608,6 @@ static void sock_map_free(struct bpf_map *map)
 	}
 	rcu_read_unlock();
 
-	if (stab->bpf_verdict)
-		bpf_prog_put(stab->bpf_verdict);
-	if (stab->bpf_parse)
-		bpf_prog_put(stab->bpf_parse);
-
 	sock_map_remove_complete(stab);
 }
 
@@ -888,6 +883,19 @@ static int sock_map_update_elem(struct bpf_map *map,
 	return err;
 }
 
+static void sock_map_release(struct bpf_map *map, struct file *map_file)
+{
+	struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
+	struct bpf_prog *orig;
+
+	orig = xchg(&stab->bpf_parse, NULL);
+	if (orig)
+		bpf_prog_put(orig);
+	orig = xchg(&stab->bpf_verdict, NULL);
+	if (orig)
+		bpf_prog_put(orig);
+}
+
 const struct bpf_map_ops sock_map_ops = {
 	.map_alloc = sock_map_alloc,
 	.map_free = sock_map_free,
@@ -895,6 +903,7 @@ static int sock_map_update_elem(struct bpf_map *map,
 	.map_get_next_key = sock_map_get_next_key,
 	.map_update_elem = sock_map_update_elem,
 	.map_delete_elem = sock_map_delete_elem,
+	.map_release = sock_map_release,
 };
 
 BPF_CALL_4(bpf_sock_map_update, struct bpf_sock_ops_kern *, bpf_sock,