Skip to contentSkip to conversation

bpf: support cloning sk storage on accept()

PATCH Superseded Stanislav Fomichev 5 participants 14 messages Aug 9, 2019 – Aug 13, 2019

Subsystems (7)

v2 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

v2 cover bpf: support cloning sk storage on accept()

Currently there is no way to propagate sk storage from the listener socket to a newly accepted one. Consider the following use case:

        fd = socket();
        setsockopt(fd, SOL_IP, IP_TOS,...);
        /* ^^^ setsockopt BPF program triggers here and saves something
         * into sk storage of the listener.
         */
        listen(fd, ...);
        while (client = accept(fd)) {
                /* At this point all association between listener
                 * socket and newly accepted one is gone. New
                 * socket will not have any sk storage attached.
                 */
        }

Let's add new BPF_F_CLONE flag that can be specified when creating a socket storage map. This new flag indicates that map contents should be cloned when the socket is cloned.

v2:
* remove spinlocks around selem_link_map/sk (Martin KaFai Lau)
* BPF_F_CLONE on a map, not selem (Martin KaFai Lau)
* hold a map while cloning (Martin KaFai Lau)
* use BTF maps in selftests (Yonghong Song)
* do proper cleanup selftests; don't call close(-1) (Yonghong Song)
* export bpf_map_inc_not_zero

Cc Martin KaFai Lau <[email protected]>
Cc Yonghong Song <[email protected]>

Stanislav Fomichev (4):
bpf: export bpf_map_inc_not_zero
bpf: support cloning sk storage on accept()
bpf: sync bpf.h to tools/
selftests/bpf: add sockopt clone/inheritance test

 include/linux/bpf.h                           |   2 +
 include/net/bpf_sk_storage.h                  |  10 +
 include/uapi/linux/bpf.h                      |   3 +
 kernel/bpf/syscall.c                          |  16 +-
 net/core/bpf_sk_storage.c                     | 100 ++++++-
 net/core/sock.c                               |   9 +-
 tools/include/uapi/linux/bpf.h                |   3 +
 tools/testing/selftests/bpf/.gitignore        |   1 +
 tools/testing/selftests/bpf/Makefile          |   3 +-
 .../selftests/bpf/progs/sockopt_inherit.c     |  97 +++++++
 .../selftests/bpf/test_sockopt_inherit.c      | 253 ++++++++++++++++++
 11 files changed, 487 insertions(+), 10 deletions(-)
 create mode 100644 tools/testing/selftests/bpf/progs/sockopt_inherit.c
 create mode 100644 tools/testing/selftests/bpf/test_sockopt_inherit.c
Signature · 1 line
2.23.0.rc1.153.gdeed80330f-goog
#1 lore raw

PATCH v2 1/4 bpf: export bpf_map_inc_not_zero

Rename existing bpf_map_inc_not_zero to __bpf_map_inc_not_zero to indicate that it's caller's responsibility to do proper locking. Create and export bpf_map_inc_not_zero wrapper that properly locks map_idr_lock. Will be used in the next commit to hold a map while cloning a socket.

Cc Martin KaFai Lau <[email protected]>
Cc Yonghong Song <[email protected]>
---
 include/linux/bpf.h  |  2 ++
 kernel/bpf/syscall.c | 16 +++++++++++++---
 2 files changed, 15 insertions(+), 3 deletions(-)
include/linux/bpf.h +2 -0
index f9a506147c8a..15ae49862b82 100644
@@ -647,6 +647,8 @@ void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock);
 struct bpf_map *bpf_map_get_with_uref(u32 ufd);
 struct bpf_map *__bpf_map_get(struct fd f);
 struct bpf_map * __must_check bpf_map_inc(struct bpf_map *map, bool uref);
+struct bpf_map * __must_check bpf_map_inc_not_zero(struct bpf_map *map,
+						   bool uref);
 void bpf_map_put_with_uref(struct bpf_map *map);
 void bpf_map_put(struct bpf_map *map);
 int bpf_map_charge_memlock(struct bpf_map *map, u32 pages);
kernel/bpf/syscall.c +13 -3
index 5d141f16f6fa..cf8052b016e7 100644
@@ -683,8 +683,8 @@ struct bpf_map *bpf_map_get_with_uref(u32 ufd)
 }
 
 /* map_idr_lock should have been held */
-static struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map,
-					    bool uref)
+static struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map,
+					      bool uref)
 {
 	int refold;
 
@@ -704,6 +704,16 @@ static struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map,
 	return map;
 }
 
+struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map, bool uref)
+{
+	spin_lock_bh(&map_idr_lock);
+	map = __bpf_map_inc_not_zero(map, uref);
+	spin_unlock_bh(&map_idr_lock);
+
+	return map;
+}
+EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero);
+
 int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
 {
 	return -ENOTSUPP;
@@ -2177,7 +2187,7 @@ static int bpf_map_get_fd_by_id(const union bpf_attr *attr)
 	spin_lock_bh(&map_idr_lock);
 	map = idr_find(&map_idr, id);
 	if (map)
-		map = bpf_map_inc_not_zero(map, true);
+		map = __bpf_map_inc_not_zero(map, true);
 	else
 		map = ERR_PTR(-ENOENT);
 	spin_unlock_bh(&map_idr_lock);
Signature · 1 line
2.23.0.rc1.153.gdeed80330f-goog
Acked-by on v2 1/4
#2 lore raw

Re: [PATCH bpf-next v2 1/4] bpf: export bpf_map_inc_not_zero

Quoted 9 lines · Stanislav Fomichev

Rename existing bpf_map_inc_not_zero to __bpf_map_inc_not_zero to indicate that it's caller's responsibility to do proper locking. Create and export bpf_map_inc_not_zero wrapper that properly locks map_idr_lock. Will be used in the next commit to hold a map while cloning a socket.

Cc Martin KaFai Lau <[email protected]>
Cc Yonghong Song <[email protected]>
#3 lore raw

PATCH v2 2/4 bpf: support cloning sk storage on accept()

In reply to Stanislav Fomichev

Add new helper bpf_sk_storage_clone which optionally clones sk storage and call it from sk_clone_lock.

Cc Martin KaFai Lau <[email protected]>
Cc Yonghong Song <[email protected]>
---
 include/net/bpf_sk_storage.h |  10 ++++
 include/uapi/linux/bpf.h     |   3 ++
 net/core/bpf_sk_storage.c    | 100 +++++++++++++++++++++++++++++++++--
 net/core/sock.c              |   9 ++--
 4 files changed, 116 insertions(+), 6 deletions(-)
include/net/bpf_sk_storage.h +10 -0
index b9dcb02e756b..8e4f831d2e52 100644
@@ -10,4 +10,14 @@ void bpf_sk_storage_free(struct sock *sk);
 extern const struct bpf_func_proto bpf_sk_storage_get_proto;
 extern const struct bpf_func_proto bpf_sk_storage_delete_proto;
 
+#ifdef CONFIG_BPF_SYSCALL
+int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk);
+#else
+static inline int bpf_sk_storage_clone(const struct sock *sk,
+				       struct sock *newsk)
+{
+	return 0;
+}
+#endif
+
 #endif /* _BPF_SK_STORAGE_H */
include/uapi/linux/bpf.h +3 -0
index 4393bd4b2419..0ef594ac3899 100644
@@ -337,6 +337,9 @@ enum bpf_attach_type {
 #define BPF_F_RDONLY_PROG	(1U << 7)
 #define BPF_F_WRONLY_PROG	(1U << 8)
 
+/* Clone map from listener for newly accepted socket */
+#define BPF_F_CLONE		(1U << 9)
+
 /* flags for BPF_PROG_QUERY */
 #define BPF_F_QUERY_EFFECTIVE	(1U << 0)
 
net/core/bpf_sk_storage.c +97 -3

136 lines not on this page · raw message

net/core/sock.c +6 -3
index d57b0cc995a0..f5e801a9cea4 100644
@@ -1851,9 +1851,12 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
 			goto out;
 		}
 		RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL);
-#ifdef CONFIG_BPF_SYSCALL
-		RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL);
-#endif
+
+		if (bpf_sk_storage_clone(sk, newsk)) {
+			sk_free_unlock_clone(newsk);
+			newsk = NULL;
+			goto out;
+		}
 
 		newsk->sk_err	   = 0;
 		newsk->sk_err_soft = 0;
Signature · 1 line
2.23.0.rc1.153.gdeed80330f-goog
maintainer
#5 lore raw
In reply to Stanislav Fomichev
Quoted 6 lines · Stanislav Fomichev

Add new helper bpf_sk_storage_clone which optionally clones sk storage and call it from sk_clone_lock.

Cc Martin KaFai Lau <[email protected]>
Cc Yonghong Song <[email protected]>

[...]

Quoted 42 lines
net/core/bpf_sk_storage.c +42 -0
+int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk)
+{
+	struct bpf_sk_storage *new_sk_storage = NULL;
+	struct bpf_sk_storage *sk_storage;
+	struct bpf_sk_storage_elem *selem;
+	int ret;
+
+	RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL);
+
+	rcu_read_lock();
+	sk_storage = rcu_dereference(sk->sk_bpf_storage);
+
+	if (!sk_storage || hlist_empty(&sk_storage->list))
+		goto out;
+
+	hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
+		struct bpf_sk_storage_elem *copy_selem;
+		struct bpf_sk_storage_map *smap;
+		struct bpf_map *map;
+		int refold;
+
+		smap = rcu_dereference(SDATA(selem)->smap);
+		if (!(smap->map.map_flags & BPF_F_CLONE))
+			continue;
+
+		map = bpf_map_inc_not_zero(&smap->map, false);
+		if (IS_ERR(map))
+			continue;
+
+		copy_selem = bpf_sk_storage_clone_elem(newsk, smap, selem);
+		if (!copy_selem) {
+			ret = -ENOMEM;
+			bpf_map_put(map);
+			goto err;
+		}
+
+		if (new_sk_storage) {
+			selem_link_map(smap, copy_selem);
+			__selem_link_sk(new_sk_storage, copy_selem);
+		} else {
+			ret = sk_storage_alloc(newsk, smap, copy_selem);
+			if (ret) {

net/core/bpf_sk_storage.c static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key) · line 822

Quoted 10 lines
net/core/bpf_sk_storage.c +10 -0
+				kfree(copy_selem);
+				atomic_sub(smap->elem_size,
+					   &newsk->sk_omem_alloc);
+				bpf_map_put(map);
+				goto err;
+			}
+
+			new_sk_storage = rcu_dereference(copy_selem->sk_storage);
+		}
+		bpf_map_put(map);

The map get/put combination /under/ RCU read lock seems a bit odd to me, could you exactly describe the race that this would be preventing?

net/core/bpf_sk_storage.c static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key) · line 834

Quoted 12 lines
net/core/bpf_sk_storage.c +12 -0
+	}
+
+out:
+	rcu_read_unlock();
+	return 0;
+
+err:
+	rcu_read_unlock();
+
+	bpf_sk_storage_free(newsk);
+	return ret;
+}

Thanks,
Daniel

#6 lore raw
Quoted 55 lines · Daniel Borkmann
Quoted 6 lines · Stanislav Fomichev

Add new helper bpf_sk_storage_clone which optionally clones sk storage and call it from sk_clone_lock.

Cc Martin KaFai Lau <[email protected]>
Cc Yonghong Song <[email protected]>

[...]

Quoted 47 lines
net/core/bpf_sk_storage.c +47 -0
+int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk)
+{
+	struct bpf_sk_storage *new_sk_storage = NULL;
+	struct bpf_sk_storage *sk_storage;
+	struct bpf_sk_storage_elem *selem;
+	int ret;
+
+	RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL);
+
+	rcu_read_lock();
+	sk_storage = rcu_dereference(sk->sk_bpf_storage);
+
+	if (!sk_storage || hlist_empty(&sk_storage->list))
+		goto out;
+
+	hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
+		struct bpf_sk_storage_elem *copy_selem;
+		struct bpf_sk_storage_map *smap;
+		struct bpf_map *map;
+		int refold;
+
+		smap = rcu_dereference(SDATA(selem)->smap);
+		if (!(smap->map.map_flags & BPF_F_CLONE))
+			continue;
+
+		map = bpf_map_inc_not_zero(&smap->map, false);
+		if (IS_ERR(map))
+			continue;
+
+		copy_selem = bpf_sk_storage_clone_elem(newsk, smap, selem);
+		if (!copy_selem) {
+			ret = -ENOMEM;
+			bpf_map_put(map);
+			goto err;
+		}
+
+		if (new_sk_storage) {
+			selem_link_map(smap, copy_selem);
+			__selem_link_sk(new_sk_storage, copy_selem);
+		} else {
+			ret = sk_storage_alloc(newsk, smap, copy_selem);
+			if (ret) {
+				kfree(copy_selem);
+				atomic_sub(smap->elem_size,
+					   &newsk->sk_omem_alloc);
+				bpf_map_put(map);
+				goto err;
Quoted 8 lines

net/core/bpf_sk_storage.c static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key) · line 822

Quoted 5 lines
net/core/bpf_sk_storage.c +5 -0
+			}
+
+			new_sk_storage = rcu_dereference(copy_selem->sk_storage);
+		}
+		bpf_map_put(map);

The map get/put combination /under/ RCU read lock seems a bit odd to me, could you exactly describe the race that this would be preventing?

There is a race between sk storage release and sk storage clone. bpf_sk_storage_map_free uses synchronize_rcu to wait for all existing users to finish and the new ones are prevented via map's refcnt being zero; we need to do something like that for the clone. Martin suggested to use bpf_map_inc_not_zero/bpf_map_put. If I read everythin correctly, I think without map_inc/map_put we get the following race:

CPU0 CPU1

bpf_map_put
bpf_sk_storage_map_free(smap)

    synchronize_rcu

    // no more users via bpf or
    // syscall, but clone
    // can still happen

    for each (bucket)
      selem_unlink
        selem_unlink_map(smap)

        // adding anything at
        // this point to the
        // bucket will leak

                                       rcu_read_lock
                                       tcp_v4_rcv
                                         tcp_v4_do_rcv
                                           // sk is lockless TCP_LISTEN
                                           tcp_v4_cookie_check
                                             tcp_v4_syn_recv_sock
                                               bpf_sk_storage_clone
                                                 rcu_dereference(sk->sk_bpf_storage)
                                                 selem_link_map(smap, copy)
                                                 // adding new element to the
                                                 // map -> leak
                                       rcu_read_unlock

      selem_unlink_sk
       sk->sk_bpf_storage = NULL

    synchronize_rcu
maintainer
#7 lore raw
Quoted 99 lines · Stanislav Fomichev
Quoted 55 lines · Daniel Borkmann
Quoted 6 lines · Stanislav Fomichev

Add new helper bpf_sk_storage_clone which optionally clones sk storage and call it from sk_clone_lock.

Cc Martin KaFai Lau <[email protected]>
Cc Yonghong Song <[email protected]>

[...]

Quoted 47 lines
net/core/bpf_sk_storage.c +47 -0
+int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk)
+{
+	struct bpf_sk_storage *new_sk_storage = NULL;
+	struct bpf_sk_storage *sk_storage;
+	struct bpf_sk_storage_elem *selem;
+	int ret;
+
+	RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL);
+
+	rcu_read_lock();
+	sk_storage = rcu_dereference(sk->sk_bpf_storage);
+
+	if (!sk_storage || hlist_empty(&sk_storage->list))
+		goto out;
+
+	hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
+		struct bpf_sk_storage_elem *copy_selem;
+		struct bpf_sk_storage_map *smap;
+		struct bpf_map *map;
+		int refold;
+
+		smap = rcu_dereference(SDATA(selem)->smap);
+		if (!(smap->map.map_flags & BPF_F_CLONE))
+			continue;
+
+		map = bpf_map_inc_not_zero(&smap->map, false);
+		if (IS_ERR(map))
+			continue;
+
+		copy_selem = bpf_sk_storage_clone_elem(newsk, smap, selem);
+		if (!copy_selem) {
+			ret = -ENOMEM;
+			bpf_map_put(map);
+			goto err;
+		}
+
+		if (new_sk_storage) {
+			selem_link_map(smap, copy_selem);
+			__selem_link_sk(new_sk_storage, copy_selem);
+		} else {
+			ret = sk_storage_alloc(newsk, smap, copy_selem);
+			if (ret) {
+				kfree(copy_selem);
+				atomic_sub(smap->elem_size,
+					   &newsk->sk_omem_alloc);
+				bpf_map_put(map);
+				goto err;
Quoted 8 lines

net/core/bpf_sk_storage.c static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key) · line 822

Quoted 5 lines
net/core/bpf_sk_storage.c +5 -0
+			}
+
+			new_sk_storage = rcu_dereference(copy_selem->sk_storage);
+		}
+		bpf_map_put(map);

The map get/put combination /under/ RCU read lock seems a bit odd to me, could you exactly describe the race that this would be preventing?

There is a race between sk storage release and sk storage clone. bpf_sk_storage_map_free uses synchronize_rcu to wait for all existing users to finish and the new ones are prevented via map's refcnt being zero; we need to do something like that for the clone. Martin suggested to use bpf_map_inc_not_zero/bpf_map_put. If I read everythin correctly, I think without map_inc/map_put we get the following race:

CPU0 CPU1

bpf_map_put
bpf_sk_storage_map_free(smap)

     synchronize_rcu

     // no more users via bpf or
     // syscall, but clone
     // can still happen

     for each (bucket)
       selem_unlink
         selem_unlink_map(smap)

         // adding anything at
         // this point to the
         // bucket will leak

                                        rcu_read_lock
                                        tcp_v4_rcv
                                          tcp_v4_do_rcv
                                            // sk is lockless TCP_LISTEN
                                            tcp_v4_cookie_check
                                              tcp_v4_syn_recv_sock
                                                bpf_sk_storage_clone
                                                  rcu_dereference(sk->sk_bpf_storage)
                                                  selem_link_map(smap, copy)
Quoted 9 lines
                                                  // adding new element to the
                                                  // map -> leak
                                        rcu_read_unlock

       selem_unlink_sk
        sk->sk_bpf_storage = NULL

     synchronize_rcu

Makes sense, thanks for clarifying. Perhaps a small comment on top of the bpf_map_inc_not_zero() would be great as well, so it's immediately clear also from this location when reading the code why this is done.

Thanks,
Daniel

#8 lore raw
Quoted 107 lines · Daniel Borkmann
Quoted 106 lines · Stanislav Fomichev
Quoted 55 lines · Daniel Borkmann
Quoted 6 lines · Stanislav Fomichev

Add new helper bpf_sk_storage_clone which optionally clones sk storage and call it from sk_clone_lock.

Cc Martin KaFai Lau <[email protected]>
Cc Yonghong Song <[email protected]>

[...]

Quoted 47 lines
net/core/bpf_sk_storage.c +47 -0
+int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk)
+{
+	struct bpf_sk_storage *new_sk_storage = NULL;
+	struct bpf_sk_storage *sk_storage;
+	struct bpf_sk_storage_elem *selem;
+	int ret;
+
+	RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL);
+
+	rcu_read_lock();
+	sk_storage = rcu_dereference(sk->sk_bpf_storage);
+
+	if (!sk_storage || hlist_empty(&sk_storage->list))
+		goto out;
+
+	hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
+		struct bpf_sk_storage_elem *copy_selem;
+		struct bpf_sk_storage_map *smap;
+		struct bpf_map *map;
+		int refold;
+
+		smap = rcu_dereference(SDATA(selem)->smap);
+		if (!(smap->map.map_flags & BPF_F_CLONE))
+			continue;
+
+		map = bpf_map_inc_not_zero(&smap->map, false);
+		if (IS_ERR(map))
+			continue;
+
+		copy_selem = bpf_sk_storage_clone_elem(newsk, smap, selem);
+		if (!copy_selem) {
+			ret = -ENOMEM;
+			bpf_map_put(map);
+			goto err;
+		}
+
+		if (new_sk_storage) {
+			selem_link_map(smap, copy_selem);
+			__selem_link_sk(new_sk_storage, copy_selem);
+		} else {
+			ret = sk_storage_alloc(newsk, smap, copy_selem);
+			if (ret) {
+				kfree(copy_selem);
+				atomic_sub(smap->elem_size,
+					   &newsk->sk_omem_alloc);
+				bpf_map_put(map);
+				goto err;
Quoted 8 lines

net/core/bpf_sk_storage.c static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key) · line 822

Quoted 5 lines
net/core/bpf_sk_storage.c +5 -0
+			}
+
+			new_sk_storage = rcu_dereference(copy_selem->sk_storage);
+		}
+		bpf_map_put(map);

The map get/put combination /under/ RCU read lock seems a bit odd to me, could you exactly describe the race that this would be preventing?

There is a race between sk storage release and sk storage clone. bpf_sk_storage_map_free uses synchronize_rcu to wait for all existing users to finish and the new ones are prevented via map's refcnt being zero; we need to do something like that for the clone. Martin suggested to use bpf_map_inc_not_zero/bpf_map_put. If I read everythin correctly, I think without map_inc/map_put we get the following race:

CPU0 CPU1

bpf_map_put
bpf_sk_storage_map_free(smap)

     synchronize_rcu

     // no more users via bpf or
     // syscall, but clone
     // can still happen

     for each (bucket)
       selem_unlink
         selem_unlink_map(smap)

         // adding anything at
         // this point to the
         // bucket will leak

                                        rcu_read_lock
                                        tcp_v4_rcv
                                          tcp_v4_do_rcv
                                            // sk is lockless TCP_LISTEN
                                            tcp_v4_cookie_check
                                              tcp_v4_syn_recv_sock
                                                bpf_sk_storage_clone
                                                  rcu_dereference(sk->sk_bpf_storage)
                                                  selem_link_map(smap, copy)
                                                  // adding new element to the
                                                  // map -> leak
                                        rcu_read_unlock

       selem_unlink_sk
        sk->sk_bpf_storage = NULL
Quoted 6 lines
Quoted 2 lines

synchronize_rcu

Makes sense, thanks for clarifying. Perhaps a small comment on top of the bpf_map_inc_not_zero() would be great as well, so it's immediately clear also from this location when reading the code why this is done.

Sure, no problem, will have something similar to what I have before synchronize_rcu in bpf_sk_storage_map_free.

Quoted 2 lines

Thanks,
Daniel

#9 lore raw
In reply to Stanislav Fomichev
Quoted 2 lines · Stanislav Fomichev

Add new helper bpf_sk_storage_clone which optionally clones sk storage and call it from sk_clone_lock.

Thanks for v2. Sorry for the delay. I am traveling.

Quoted 76 lines
Cc Martin KaFai Lau <[email protected]>
Cc Yonghong Song <[email protected]>
---
 include/net/bpf_sk_storage.h |  10 ++++
 include/uapi/linux/bpf.h     |   3 ++
 net/core/bpf_sk_storage.c    | 100 +++++++++++++++++++++++++++++++++--
 net/core/sock.c              |   9 ++--
 4 files changed, 116 insertions(+), 6 deletions(-)
include/net/bpf_sk_storage.h +10 -0
index b9dcb02e756b..8e4f831d2e52 100644
@@ -10,4 +10,14 @@ void bpf_sk_storage_free(struct sock *sk);
 extern const struct bpf_func_proto bpf_sk_storage_get_proto;
 extern const struct bpf_func_proto bpf_sk_storage_delete_proto;
 
+#ifdef CONFIG_BPF_SYSCALL
+int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk);
+#else
+static inline int bpf_sk_storage_clone(const struct sock *sk,
+				       struct sock *newsk)
+{
+	return 0;
+}
+#endif
+
 #endif /* _BPF_SK_STORAGE_H */
include/uapi/linux/bpf.h +3 -0
index 4393bd4b2419..0ef594ac3899 100644
@@ -337,6 +337,9 @@ enum bpf_attach_type {
 #define BPF_F_RDONLY_PROG	(1U << 7)
 #define BPF_F_WRONLY_PROG	(1U << 8)
 
+/* Clone map from listener for newly accepted socket */
+#define BPF_F_CLONE		(1U << 9)
+
 /* flags for BPF_PROG_QUERY */
 #define BPF_F_QUERY_EFFECTIVE	(1U << 0)
 
net/core/bpf_sk_storage.c +4 -2
index 94c7f77ecb6b..584e08ee0ca3 100644
@@ -12,6 +12,9 @@
 
 static atomic_t cache_idx;
 
+#define SK_STORAGE_CREATE_FLAG_MASK					\
+	(BPF_F_NO_PREALLOC | BPF_F_CLONE)
+
 struct bucket {
 	struct hlist_head list;
 	raw_spinlock_t lock;
@@ -209,7 +212,6 @@ static void selem_unlink_sk(struct bpf_sk_storage_elem *selem)
 		kfree_rcu(sk_storage, rcu);
 }
 
-/* sk_storage->lock must be held and sk_storage->list cannot be empty */
 static void __selem_link_sk(struct bpf_sk_storage *sk_storage,
 			    struct bpf_sk_storage_elem *selem)
 {
@@ -509,7 +511,7 @@ static int sk_storage_delete(struct sock *sk, struct bpf_map *map)
 	return 0;
 }
 
-/* Called by __sk_destruct() */
+/* Called by __sk_destruct() & bpf_sk_storage_clone() */
 void bpf_sk_storage_free(struct sock *sk)
 {
 	struct bpf_sk_storage_elem *selem;

net/core/bpf_sk_storage.c static void bpf_sk_storage_map_free(struct bpf_map *map) · line 566

Quoted 8 lines
net/core/bpf_sk_storage.c +5 -0
@@ -557,6 +559,11 @@ static void bpf_sk_storage_map_free(struct bpf_map *map)
 
 	smap = (struct bpf_sk_storage_map *)map;
 
+	/* Note that this map might be concurrently cloned from
+	 * bpf_sk_storage_clone. Wait for any existing bpf_sk_storage_clone
+	 * RCU read section to finish before proceeding. New RCU
+	 * read sections should be prevented via bpf_map_inc_not_zero.
+	 */

Thanks!

net/core/bpf_sk_storage.c static void bpf_sk_storage_map_free(struct bpf_map *map) · line 612

Quoted 10 lines
net/core/bpf_sk_storage.c +2 -1
 	synchronize_rcu();
 
 	/* bpf prog and the userspace can no longer access this map
@@ -601,7 +608,8 @@ static void bpf_sk_storage_map_free(struct bpf_map *map)
 
 static int bpf_sk_storage_map_alloc_check(union bpf_attr *attr)
 {
-	if (attr->map_flags != BPF_F_NO_PREALLOC || attr->max_entries ||
+	if (attr->map_flags & ~SK_STORAGE_CREATE_FLAG_MASK ||
+	    attr->max_entries ||

I think "!(attr->map_flags & BPF_F_NO_PREALLOC)" should also be needed.

Quoted 80 lines
net/core/bpf_sk_storage.c +73 -0
 	    attr->key_size != sizeof(int) || !attr->value_size ||
 	    /* Enforce BTF for userspace sk dumping */
 	    !attr->btf_key_type_id || !attr->btf_value_type_id)
@@ -739,6 +747,92 @@ static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key)
 	return err;
 }
 
+static struct bpf_sk_storage_elem *
+bpf_sk_storage_clone_elem(struct sock *newsk,
+			  struct bpf_sk_storage_map *smap,
+			  struct bpf_sk_storage_elem *selem)
+{
+	struct bpf_sk_storage_elem *copy_selem;
+
+	copy_selem = selem_alloc(smap, newsk, NULL, true);
+	if (!copy_selem)
+		return NULL;
+
+	if (map_value_has_spin_lock(&smap->map))
+		copy_map_value_locked(&smap->map, SDATA(copy_selem)->data,
+				      SDATA(selem)->data, true);
+	else
+		copy_map_value(&smap->map, SDATA(copy_selem)->data,
+			       SDATA(selem)->data);
+
+	return copy_selem;
+}
+
+int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk)
+{
+	struct bpf_sk_storage *new_sk_storage = NULL;
+	struct bpf_sk_storage *sk_storage;
+	struct bpf_sk_storage_elem *selem;
+	int ret;
+
+	RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL);
+
+	rcu_read_lock();
+	sk_storage = rcu_dereference(sk->sk_bpf_storage);
+
+	if (!sk_storage || hlist_empty(&sk_storage->list))
+		goto out;
+
+	hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
+		struct bpf_sk_storage_elem *copy_selem;
+		struct bpf_sk_storage_map *smap;
+		struct bpf_map *map;
+		int refold;
+
+		smap = rcu_dereference(SDATA(selem)->smap);
+		if (!(smap->map.map_flags & BPF_F_CLONE))
+			continue;
+
+		map = bpf_map_inc_not_zero(&smap->map, false);
+		if (IS_ERR(map))
+			continue;
+
+		copy_selem = bpf_sk_storage_clone_elem(newsk, smap, selem);
+		if (!copy_selem) {
+			ret = -ENOMEM;
+			bpf_map_put(map);
+			goto err;
+		}
+
+		if (new_sk_storage) {
+			selem_link_map(smap, copy_selem);
+			__selem_link_sk(new_sk_storage, copy_selem);
+		} else {
+			ret = sk_storage_alloc(newsk, smap, copy_selem);
+			if (ret) {
+				kfree(copy_selem);
+				atomic_sub(smap->elem_size,
+					   &newsk->sk_omem_alloc);
+				bpf_map_put(map);
+				goto err;
+			}
+
+			new_sk_storage = rcu_dereference(copy_selem->sk_storage);
+		}
+		bpf_map_put(map);

net/core/bpf_sk_storage.c static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key) · line 832

Quoted 10 lines
net/core/bpf_sk_storage.c +10 -0
+	}
+
+out:
+	rcu_read_unlock();
+	return 0;
+
+err:
+	rcu_read_unlock();
+
+	bpf_sk_storage_free(newsk);

The later sk_free_unlock_clone(newsk) should eventually call bpf_sk_storage_free(newsk) also?

Others LGTM.

Quoted 29 lines
net/core/bpf_sk_storage.c +3 -0
+	return ret;
+}
+
 BPF_CALL_4(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk,
 	   void *, value, u64, flags)
 {
net/core/sock.c +6 -3
index d57b0cc995a0..f5e801a9cea4 100644
@@ -1851,9 +1851,12 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
 			goto out;
 		}
 		RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL);
-#ifdef CONFIG_BPF_SYSCALL
-		RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL);
-#endif
+
+		if (bpf_sk_storage_clone(sk, newsk)) {
+			sk_free_unlock_clone(newsk);
+			newsk = NULL;
+			goto out;
+		}
 
 		newsk->sk_err	   = 0;
 		newsk->sk_err_soft = 0;
Signature · 1 line
2.23.0.rc1.153.gdeed80330f-goog
#10 lore raw
Quoted 4 lines · Martin Lau
Quoted 2 lines · Stanislav Fomichev

Add new helper bpf_sk_storage_clone which optionally clones sk storage and call it from sk_clone_lock.

Thanks for v2. Sorry for the delay. I am traveling.

No worries, take your time, if you're OOO feel free to do another round when you get back, not urgent.

Quoted 91 lines
Quoted 76 lines
Cc Martin KaFai Lau <[email protected]>
Cc Yonghong Song <[email protected]>
---
 include/net/bpf_sk_storage.h |  10 ++++
 include/uapi/linux/bpf.h     |   3 ++
 net/core/bpf_sk_storage.c    | 100 +++++++++++++++++++++++++++++++++--
 net/core/sock.c              |   9 ++--
 4 files changed, 116 insertions(+), 6 deletions(-)
include/net/bpf_sk_storage.h +10 -0
index b9dcb02e756b..8e4f831d2e52 100644
@@ -10,4 +10,14 @@ void bpf_sk_storage_free(struct sock *sk);
 extern const struct bpf_func_proto bpf_sk_storage_get_proto;
 extern const struct bpf_func_proto bpf_sk_storage_delete_proto;
 
+#ifdef CONFIG_BPF_SYSCALL
+int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk);
+#else
+static inline int bpf_sk_storage_clone(const struct sock *sk,
+				       struct sock *newsk)
+{
+	return 0;
+}
+#endif
+
 #endif /* _BPF_SK_STORAGE_H */
include/uapi/linux/bpf.h +3 -0
index 4393bd4b2419..0ef594ac3899 100644
@@ -337,6 +337,9 @@ enum bpf_attach_type {
 #define BPF_F_RDONLY_PROG	(1U << 7)
 #define BPF_F_WRONLY_PROG	(1U << 8)
 
+/* Clone map from listener for newly accepted socket */
+#define BPF_F_CLONE		(1U << 9)
+
 /* flags for BPF_PROG_QUERY */
 #define BPF_F_QUERY_EFFECTIVE	(1U << 0)
 
net/core/bpf_sk_storage.c +4 -2
index 94c7f77ecb6b..584e08ee0ca3 100644
@@ -12,6 +12,9 @@
 
 static atomic_t cache_idx;
 
+#define SK_STORAGE_CREATE_FLAG_MASK					\
+	(BPF_F_NO_PREALLOC | BPF_F_CLONE)
+
 struct bucket {
 	struct hlist_head list;
 	raw_spinlock_t lock;
@@ -209,7 +212,6 @@ static void selem_unlink_sk(struct bpf_sk_storage_elem *selem)
 		kfree_rcu(sk_storage, rcu);
 }
 
-/* sk_storage->lock must be held and sk_storage->list cannot be empty */
 static void __selem_link_sk(struct bpf_sk_storage *sk_storage,
 			    struct bpf_sk_storage_elem *selem)
 {
@@ -509,7 +511,7 @@ static int sk_storage_delete(struct sock *sk, struct bpf_map *map)
 	return 0;
 }
 
-/* Called by __sk_destruct() */
+/* Called by __sk_destruct() & bpf_sk_storage_clone() */
 void bpf_sk_storage_free(struct sock *sk)
 {
 	struct bpf_sk_storage_elem *selem;

net/core/bpf_sk_storage.c static void bpf_sk_storage_map_free(struct bpf_map *map) · line 566

Quoted 8 lines
net/core/bpf_sk_storage.c +5 -0
@@ -557,6 +559,11 @@ static void bpf_sk_storage_map_free(struct bpf_map *map)
 
 	smap = (struct bpf_sk_storage_map *)map;
 
+	/* Note that this map might be concurrently cloned from
+	 * bpf_sk_storage_clone. Wait for any existing bpf_sk_storage_clone
+	 * RCU read section to finish before proceeding. New RCU
+	 * read sections should be prevented via bpf_map_inc_not_zero.
+	 */

Thanks!

Quoted 5 lines
net/core/bpf_sk_storage.c +0 -0
 	synchronize_rcu();
 
 	/* bpf prog and the userspace can no longer access this map
@@ -601,7 +608,8 @@ static void bpf_sk_storage_map_free(struct bpf_map *map)
 
Quoted 6 lines

net/core/bpf_sk_storage.c static void bpf_sk_storage_map_free(struct bpf_map *map) · line 612

Quoted 5 lines
net/core/bpf_sk_storage.c +2 -1
 static int bpf_sk_storage_map_alloc_check(union bpf_attr *attr)
 {
-	if (attr->map_flags != BPF_F_NO_PREALLOC || attr->max_entries ||
+	if (attr->map_flags & ~SK_STORAGE_CREATE_FLAG_MASK ||
+	    attr->max_entries ||

I think "!(attr->map_flags & BPF_F_NO_PREALLOC)" should also be needed.

Makes sense, we always want to have BPF_F_NO_PREALLOC set. Will add, thanks!

Quoted 84 lines
Quoted 84 lines
net/core/bpf_sk_storage.c +77 -0
 	    attr->key_size != sizeof(int) || !attr->value_size ||
 	    /* Enforce BTF for userspace sk dumping */
 	    !attr->btf_key_type_id || !attr->btf_value_type_id)
@@ -739,6 +747,92 @@ static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key)
 	return err;
 }
 
+static struct bpf_sk_storage_elem *
+bpf_sk_storage_clone_elem(struct sock *newsk,
+			  struct bpf_sk_storage_map *smap,
+			  struct bpf_sk_storage_elem *selem)
+{
+	struct bpf_sk_storage_elem *copy_selem;
+
+	copy_selem = selem_alloc(smap, newsk, NULL, true);
+	if (!copy_selem)
+		return NULL;
+
+	if (map_value_has_spin_lock(&smap->map))
+		copy_map_value_locked(&smap->map, SDATA(copy_selem)->data,
+				      SDATA(selem)->data, true);
+	else
+		copy_map_value(&smap->map, SDATA(copy_selem)->data,
+			       SDATA(selem)->data);
+
+	return copy_selem;
+}
+
+int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk)
+{
+	struct bpf_sk_storage *new_sk_storage = NULL;
+	struct bpf_sk_storage *sk_storage;
+	struct bpf_sk_storage_elem *selem;
+	int ret;
+
+	RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL);
+
+	rcu_read_lock();
+	sk_storage = rcu_dereference(sk->sk_bpf_storage);
+
+	if (!sk_storage || hlist_empty(&sk_storage->list))
+		goto out;
+
+	hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
+		struct bpf_sk_storage_elem *copy_selem;
+		struct bpf_sk_storage_map *smap;
+		struct bpf_map *map;
+		int refold;
+
+		smap = rcu_dereference(SDATA(selem)->smap);
+		if (!(smap->map.map_flags & BPF_F_CLONE))
+			continue;
+
+		map = bpf_map_inc_not_zero(&smap->map, false);
+		if (IS_ERR(map))
+			continue;
+
+		copy_selem = bpf_sk_storage_clone_elem(newsk, smap, selem);
+		if (!copy_selem) {
+			ret = -ENOMEM;
+			bpf_map_put(map);
+			goto err;
+		}
+
+		if (new_sk_storage) {
+			selem_link_map(smap, copy_selem);
+			__selem_link_sk(new_sk_storage, copy_selem);
+		} else {
+			ret = sk_storage_alloc(newsk, smap, copy_selem);
+			if (ret) {
+				kfree(copy_selem);
+				atomic_sub(smap->elem_size,
+					   &newsk->sk_omem_alloc);
+				bpf_map_put(map);
+				goto err;
+			}
+
+			new_sk_storage = rcu_dereference(copy_selem->sk_storage);
+		}
+		bpf_map_put(map);
+	}
+
+out:
+	rcu_read_unlock();
Quoted 8 lines

net/core/bpf_sk_storage.c static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key) · line 832

Quoted 6 lines
net/core/bpf_sk_storage.c +6 -0
+	return 0;
+
+err:
+	rcu_read_unlock();
+
+	bpf_sk_storage_free(newsk);

The later sk_free_unlock_clone(newsk) should eventually call bpf_sk_storage_free(newsk) also?

Hm, good point, I can drop it from here and rely on sk_free_unlock_clone to call __sk_destruct/bpf_sk_storage_free. That probably deserves a comment though :-)

Quoted 1 line

Others LGTM.

Thank you for a review!

Quoted 29 lines
Quoted 29 lines
net/core/bpf_sk_storage.c +3 -0
+	return ret;
+}
+
 BPF_CALL_4(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk,
 	   void *, value, u64, flags)
 {
net/core/sock.c +6 -3
index d57b0cc995a0..f5e801a9cea4 100644
@@ -1851,9 +1851,12 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
 			goto out;
 		}
 		RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL);
-#ifdef CONFIG_BPF_SYSCALL
-		RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL);
-#endif
+
+		if (bpf_sk_storage_clone(sk, newsk)) {
+			sk_free_unlock_clone(newsk);
+			newsk = NULL;
+			goto out;
+		}
 
 		newsk->sk_err	   = 0;
 		newsk->sk_err_soft = 0;
Signature · 1 line
2.23.0.rc1.153.gdeed80330f-goog
#11 lore raw

PATCH v2 3/4 bpf: sync bpf.h to tools/

In reply to Stanislav Fomichev

Sync new sk storage clone flag.

---
 tools/include/uapi/linux/bpf.h | 3 +++
 1 file changed, 3 insertions(+)
tools/include/uapi/linux/bpf.h +3 -0
index 4393bd4b2419..0ef594ac3899 100644
@@ -337,6 +337,9 @@ enum bpf_attach_type {
 #define BPF_F_RDONLY_PROG	(1U << 7)
 #define BPF_F_WRONLY_PROG	(1U << 8)
 
+/* Clone map from listener for newly accepted socket */
+#define BPF_F_CLONE		(1U << 9)
+
 /* flags for BPF_PROG_QUERY */
 #define BPF_F_QUERY_EFFECTIVE	(1U << 0)
 
Signature · 1 line
2.23.0.rc1.153.gdeed80330f-goog
#12 lore raw

PATCH v2 4/4 selftests/bpf: add sockopt clone/inheritance test

In reply to Stanislav Fomichev

Add a test that calls setsockopt on the listener socket which triggers BPF program. This BPF program writes to the sk storage and sets clone flag. Make sure that sk storage is cloned for a newly accepted connection.

We have two cloned maps in the tests to make sure we hit both cases in bpf_sk_storage_clone: first element (sk_storage_alloc) and non-first element(s) (selem_link_map).

Cc Martin KaFai Lau <[email protected]>
Cc Yonghong Song <[email protected]>
---
 tools/testing/selftests/bpf/.gitignore        |   1 +
 tools/testing/selftests/bpf/Makefile          |   3 +-
 .../selftests/bpf/progs/sockopt_inherit.c     |  97 +++++++
 .../selftests/bpf/test_sockopt_inherit.c      | 253 ++++++++++++++++++
 4 files changed, 353 insertions(+), 1 deletion(-)
 create mode 100644 tools/testing/selftests/bpf/progs/sockopt_inherit.c
 create mode 100644 tools/testing/selftests/bpf/test_sockopt_inherit.c
tools/testing/selftests/bpf/.gitignore +1 -0
index 90f70d2c7c22..60c9338cd9b4 100644
@@ -42,4 +42,5 @@ xdping
 test_sockopt
 test_sockopt_sk
 test_sockopt_multi
+test_sockopt_inherit
 test_tcp_rtt
tools/testing/selftests/bpf/Makefile +2 -1
index 3bd0f4a0336a..c875763a851a 100644
@@ -29,7 +29,7 @@ TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_lpm_map test
 	test_cgroup_storage test_select_reuseport test_section_names \
 	test_netcnt test_tcpnotify_user test_sock_fields test_sysctl test_hashmap \
 	test_btf_dump test_cgroup_attach xdping test_sockopt test_sockopt_sk \
-	test_sockopt_multi test_tcp_rtt
+	test_sockopt_multi test_sockopt_inherit test_tcp_rtt
 
 BPF_OBJ_FILES = $(patsubst %.c,%.o, $(notdir $(wildcard progs/*.c)))
 TEST_GEN_FILES = $(BPF_OBJ_FILES)
@@ -110,6 +110,7 @@ $(OUTPUT)/test_cgroup_attach: cgroup_helpers.c
 $(OUTPUT)/test_sockopt: cgroup_helpers.c
 $(OUTPUT)/test_sockopt_sk: cgroup_helpers.c
 $(OUTPUT)/test_sockopt_multi: cgroup_helpers.c
+$(OUTPUT)/test_sockopt_inherit: cgroup_helpers.c
 $(OUTPUT)/test_tcp_rtt: cgroup_helpers.c
 
 .PHONY: force
tools/testing/selftests/bpf/progs/sockopt_inherit.c new file +97 -0

97 lines not on this page · raw message

tools/testing/selftests/bpf/test_sockopt_inherit.c new file +253 -0

253 lines not on this page · raw message

Signature · 1 line
2.23.0.rc1.153.gdeed80330f-goog
Acked-by on v2 4/4
#13 lore raw

Re: [PATCH bpf-next v2 4/4] selftests/bpf: add sockopt clone/inheritance test

Quoted 5 lines · Stanislav Fomichev

Add a test that calls setsockopt on the listener socket which triggers BPF program. This BPF program writes to the sk storage and sets clone flag. Make sure that sk storage is cloned for a newly accepted connection.

Quoted 7 lines

We have two cloned maps in the tests to make sure we hit both cases in bpf_sk_storage_clone: first element (sk_storage_alloc) and non-first element(s) (selem_link_map).

Cc Martin KaFai Lau <[email protected]>
Cc Yonghong Song <[email protected]>