Skip to contentSkip to conversation

Single-cpu updates for per-cpu maps

PATCH In review Paul Chaignon 4 participants 8 messages Dec 18, 2019 – Dec 19, 2019

Subsystems (7)
lore
Share & tools
lore
#0 lore raw

cover Single-cpu updates for per-cpu maps

Currently, userspace programs have to update the values of all CPUs at once when updating per-cpu maps. This limitation prevents the update of a single CPU's value without the risk of missing concurrent updates on other CPU's values.

The first patch allows userspace to update the value of a specific CPU in per-cpu maps. The second and last patches add test cases and support in bpftool respectively.

Paul Chaignon (3):
bpf: Single-cpu updates for per-cpu maps selftests/bpf: Tests for single-cpu updates of per-cpu maps
bpftool: Support single-cpu updates for per-cpu maps

 include/uapi/linux/bpf.h                      |  4 ++
 kernel/bpf/arraymap.c                         | 19 +++--
 kernel/bpf/hashtab.c                          | 49 +++++++------
 kernel/bpf/local_storage.c                    | 16 +++--
 kernel/bpf/syscall.c                          | 17 +++--
 .../bpf/bpftool/Documentation/bpftool-map.rst | 13 ++--
 tools/bpf/bpftool/bash-completion/bpftool     |  2 +-
 tools/bpf/bpftool/map.c                       | 70 ++++++++++++++-----
 tools/include/uapi/linux/bpf.h                |  4 ++
 tools/testing/selftests/bpf/test_maps.c       | 34 ++++++++-
 10 files changed, 168 insertions(+), 60 deletions(-)
Signature · 1 line
2.24.0
#1 lore raw

PATCH 1/3 bpf: Single-cpu updates for per-cpu maps

Currently, userspace programs have to update the values of all CPUs at once when updating per-cpu maps. This limitation prevents the update of a single CPU's value without the risk of missing concurrent updates on other CPU's values.

This patch allows userspace to update the value of a specific CPU in per-cpu maps. The CPU whose value should be updated is encoded in the 32 upper-bits of the flags argument, as follows. The new BPF_CPU flag can be combined with existing flags.

bpf_map_update_elem(..., cpuid << 32 | BPF_CPU)

---
 include/uapi/linux/bpf.h       |  4 +++
 kernel/bpf/arraymap.c          | 19 ++++++++-----
 kernel/bpf/hashtab.c           | 49 ++++++++++++++++++++--------------
 kernel/bpf/local_storage.c     | 16 +++++++----
 kernel/bpf/syscall.c           | 17 +++++++++---
 tools/include/uapi/linux/bpf.h |  4 +++
 6 files changed, 74 insertions(+), 35 deletions(-)
include/uapi/linux/bpf.h +4 -0
index dbbcf0b02970..2efb17d2c77a 100644
@@ -316,6 +316,10 @@ enum bpf_attach_type {
 #define BPF_NOEXIST	1 /* create new element if it didn't exist */
 #define BPF_EXIST	2 /* update existing element */
 #define BPF_F_LOCK	4 /* spin_lock-ed map_lookup/map_update */
+#define BPF_CPU		8 /* single-cpu update for per-cpu maps */
+
+/* CPU mask for single-cpu updates */
+#define BPF_CPU_MASK	0xFFFFFFFF00000000ULL
 
 /* flags for BPF_MAP_CREATE command */
 #define BPF_F_NO_PREALLOC	(1U << 0)
kernel/bpf/arraymap.c +13 -6
index f0d19bbb9211..a96e94696819 100644
@@ -302,7 +302,8 @@ static int array_map_update_elem(struct bpf_map *map, void *key, void *value,
 	u32 index = *(u32 *)key;
 	char *val;
 
-	if (unlikely((map_flags & ~BPF_F_LOCK) > BPF_EXIST))
+	if (unlikely((map_flags & ~BPF_CPU_MASK & ~BPF_F_LOCK &
+				  ~BPF_CPU) > BPF_EXIST))
 		/* unknown flags */
 		return -EINVAL;
 
@@ -341,7 +342,7 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
 	int cpu, off = 0;
 	u32 size;
 
-	if (unlikely(map_flags > BPF_EXIST))
+	if (unlikely((map_flags & ~BPF_CPU_MASK & ~BPF_CPU) > BPF_EXIST))
 		/* unknown flags */
 		return -EINVAL;
 
@@ -349,7 +350,7 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
 		/* all elements were pre-allocated, cannot insert a new one */
 		return -E2BIG;
 
-	if (unlikely(map_flags == BPF_NOEXIST))
+	if (unlikely(map_flags & BPF_NOEXIST))
 		/* all elements already exist */
 		return -EEXIST;
 
@@ -362,9 +363,15 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
 	size = round_up(map->value_size, 8);
 	rcu_read_lock();
 	pptr = array->pptrs[index & array->index_mask];
-	for_each_possible_cpu(cpu) {
-		bpf_long_memcpy(per_cpu_ptr(pptr, cpu), value + off, size);
-		off += size;
+	if (map_flags & BPF_CPU) {
+		bpf_long_memcpy(per_cpu_ptr(pptr, map_flags >> 32), value,
+				size);
+	} else {
+		for_each_possible_cpu(cpu) {
+			bpf_long_memcpy(per_cpu_ptr(pptr, cpu), value + off,
+					size);
+			off += size;
+		}
 	}
 	rcu_read_unlock();
 	return 0;
kernel/bpf/hashtab.c +29 -20

156 lines not on this page · raw message

kernel/bpf/local_storage.c +11 -5
index 2ba750725cb2..2f93d75fc74b 100644
@@ -206,7 +206,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *_map, void *_key,
 	int cpu, off = 0;
 	u32 size;
 
-	if (map_flags != BPF_ANY && map_flags != BPF_EXIST)
+	if (map_flags & ~BPF_CPU_MASK & ~BPF_CPU & ~BPF_EXIST)
 		return -EINVAL;
 
 	rcu_read_lock();
@@ -223,10 +223,16 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *_map, void *_key,
 	 * so no kernel data leaks possible
 	 */
 	size = round_up(_map->value_size, 8);
-	for_each_possible_cpu(cpu) {
-		bpf_long_memcpy(per_cpu_ptr(storage->percpu_buf, cpu),
-				value + off, size);
-		off += size;
+	if (map_flags & BPF_CPU) {
+		cpu = map_flags >> 32;
+		bpf_long_memcpy(per_cpu_ptr(storage->percpu_buf, cpu), value,
+				size);
+	} else {
+		for_each_possible_cpu(cpu) {
+			bpf_long_memcpy(per_cpu_ptr(storage->percpu_buf, cpu),
+					value + off, size);
+			off += size;
+		}
 	}
 	rcu_read_unlock();
 	return 0;
kernel/bpf/syscall.c +13 -4
index b08c362f4e02..83e190929f2e 100644
@@ -953,7 +953,9 @@ static int map_update_elem(union bpf_attr *attr)
 	struct bpf_map *map;
 	void *key, *value;
 	u32 value_size;
+	bool per_cpu;
 	struct fd f;
+	int cpuid;
 	int err;
 
 	if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
@@ -974,16 +976,23 @@ static int map_update_elem(union bpf_attr *attr)
 		goto err_put;
 	}
 
+	per_cpu = map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
+		  map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
+		  map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
+		  map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE;
+	if ((attr->flags & BPF_CPU) &&
+	    (!per_cpu || (attr->flags >> 32) >= num_possible_cpus())) {
+		err = -EINVAL;
+		goto err_put;
+	}
+
 	key = __bpf_copy_key(ukey, map->key_size);
 	if (IS_ERR(key)) {
 		err = PTR_ERR(key);
 		goto err_put;
 	}
 
-	if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
-	    map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
-	    map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
-	    map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
+	if (per_cpu && !(attr->flags & BPF_CPU))
 		value_size = round_up(map->value_size, 8) * num_possible_cpus();
 	else
 		value_size = map->value_size;
tools/include/uapi/linux/bpf.h +4 -0
index dbbcf0b02970..2efb17d2c77a 100644
@@ -316,6 +316,10 @@ enum bpf_attach_type {
 #define BPF_NOEXIST	1 /* create new element if it didn't exist */
 #define BPF_EXIST	2 /* update existing element */
 #define BPF_F_LOCK	4 /* spin_lock-ed map_lookup/map_update */
+#define BPF_CPU		8 /* single-cpu update for per-cpu maps */
+
+/* CPU mask for single-cpu updates */
+#define BPF_CPU_MASK	0xFFFFFFFF00000000ULL
 
 /* flags for BPF_MAP_CREATE command */
 #define BPF_F_NO_PREALLOC	(1U << 0)
Signature · 1 line
2.24.0
#2 lore raw

Re: [PATCH bpf-next 1/3] bpf: Single-cpu updates for per-cpu maps

Quoted 9 lines · Paul Chaignon

Currently, userspace programs have to update the values of all CPUs at once when updating per-cpu maps. This limitation prevents the update of a single CPU's value without the risk of missing concurrent updates on other CPU's values.

This patch allows userspace to update the value of a specific CPU in per-cpu maps. The CPU whose value should be updated is encoded in the 32 upper-bits of the flags argument, as follows. The new BPF_CPU flag can be combined with existing flags.

In general makes sense. Could you elaborate more on concrete issue?

Quoted 17 lines

bpf_map_update_elem(..., cpuid << 32 | BPF_CPU)

---
 include/uapi/linux/bpf.h       |  4 +++
 kernel/bpf/arraymap.c          | 19 ++++++++-----
 kernel/bpf/hashtab.c           | 49 ++++++++++++++++++++--------------
 kernel/bpf/local_storage.c     | 16 +++++++----
 kernel/bpf/syscall.c           | 17 +++++++++---
 tools/include/uapi/linux/bpf.h |  4 +++
 6 files changed, 74 insertions(+), 35 deletions(-)
include/uapi/linux/bpf.h +0 -0
index dbbcf0b02970..2efb17d2c77a 100644

include/uapi/linux/bpf.h enum bpf_attach_type { · line 319

Quoted 4 lines
include/uapi/linux/bpf.h +1 -0
@@ -316,6 +316,10 @@ enum bpf_attach_type {
 #define BPF_NOEXIST	1 /* create new element if it didn't exist */
 #define BPF_EXIST	2 /* update existing element */
 #define BPF_F_LOCK	4 /* spin_lock-ed map_lookup/map_update */
+#define BPF_CPU		8 /* single-cpu update for per-cpu maps */

BPF_F_CPU would be more consistent with the rest of flags.

Can BPF_F_CURRENT_CPU be supported as well?

And for consistency support this flag in map_lookup_elem too?

include/uapi/linux/bpf.h enum bpf_attach_type { · line 322

Quoted 3 lines
include/uapi/linux/bpf.h +3 -0
+
+/* CPU mask for single-cpu updates */
+#define BPF_CPU_MASK	0xFFFFFFFF00000000ULL

what is the reason to expose this in uapi?

kernel/bpf/arraymap.c static int array_map_update_elem(struct bpf_map *map, void *key, void *value, · line 306

Quoted 13 lines
include/uapi/linux/bpf.h +0 -0
 /* flags for BPF_MAP_CREATE command */
 #define BPF_F_NO_PREALLOC	(1U << 0)
kernel/bpf/arraymap.c +2 -1
index f0d19bbb9211..a96e94696819 100644
@@ -302,7 +302,8 @@ static int array_map_update_elem(struct bpf_map *map, void *key, void *value,
 	u32 index = *(u32 *)key;
 	char *val;
 
-	if (unlikely((map_flags & ~BPF_F_LOCK) > BPF_EXIST))
+	if (unlikely((map_flags & ~BPF_CPU_MASK & ~BPF_F_LOCK &
+				  ~BPF_CPU) > BPF_EXIST))

that reads odd.
More traditional would be ~ (A | B | C)

#3 lore raw

Re: [PATCH bpf-next 1/3] bpf: Single-cpu updates for per-cpu maps

Thanks everyone for the reviews and suggestions!

Quoted 6 lines · Alexei Starovoitov
Quoted 5 lines · Paul Chaignon

Currently, userspace programs have to update the values of all CPUs at once when updating per-cpu maps. This limitation prevents the update of a single CPU's value without the risk of missing concurrent updates on other CPU's values.

Quoted 6 lines
Quoted 4 lines

This patch allows userspace to update the value of a specific CPU in per-cpu maps. The CPU whose value should be updated is encoded in the 32 upper-bits of the flags argument, as follows. The new BPF_CPU flag can be combined with existing flags.

In general makes sense. Could you elaborate more on concrete issue?

Sure. I have a BPF program that matches incoming packets against packet-filtering rules, with a NIC that steers some flows (correspond to different tenants) to specific CPUs. Rules are stored in a per-cpu hashmap with a counter (counting matching packets) and an off/on switch. To enable a rule on a new CPU, I lookup the value for that rule, switch the on/off variable in the value corresponding to the given CPU and update the map. However, the counters corresponding to other CPUs for that same rule might have been updated between the lookup and the update.

Other BPF users have requested the same feature before on the bcc repository [1]. They probably have different (tracing-related?) use cases.

1 - https://github.com/iovisor/bcc/issues/1886

Quoted 25 lines
Quoted 18 lines

bpf_map_update_elem(..., cpuid << 32 | BPF_CPU)

---
 include/uapi/linux/bpf.h       |  4 +++
 kernel/bpf/arraymap.c          | 19 ++++++++-----
 kernel/bpf/hashtab.c           | 49 ++++++++++++++++++++--------------
 kernel/bpf/local_storage.c     | 16 +++++++----
 kernel/bpf/syscall.c           | 17 +++++++++---
 tools/include/uapi/linux/bpf.h |  4 +++
 6 files changed, 74 insertions(+), 35 deletions(-)
include/uapi/linux/bpf.h +0 -0
index dbbcf0b02970..2efb17d2c77a 100644

include/uapi/linux/bpf.h enum bpf_attach_type { · line 319

Quoted 4 lines
include/uapi/linux/bpf.h +1 -0
@@ -316,6 +316,10 @@ enum bpf_attach_type {
 #define BPF_NOEXIST	1 /* create new element if it didn't exist */
 #define BPF_EXIST	2 /* update existing element */
 #define BPF_F_LOCK	4 /* spin_lock-ed map_lookup/map_update */
+#define BPF_CPU		8 /* single-cpu update for per-cpu maps */

BPF_F_CPU would be more consistent with the rest of flags.

Quoted 1 line

Can BPF_F_CURRENT_CPU be supported as well?

You mean to update the value corresponding to the CPU on which the userspace program is running? BPF_F_CURRENT_CPU is a mask on the lower 32 bits so it would clash with existing flags, but there's probably another way to implement the same. Not sure I see the use case though; userspace programs can easily update the value for the current CPU with BPF_F_CPU.

Quoted 2 lines

And for consistency support this flag in map_lookup_elem too?

Sure, I'll add it to the v2.

Quoted 6 lines

include/uapi/linux/bpf.h enum bpf_attach_type { · line 322

Quoted 4 lines
include/uapi/linux/bpf.h +3 -0
+
+/* CPU mask for single-cpu updates */
+#define BPF_CPU_MASK	0xFFFFFFFF00000000ULL

what is the reason to expose this in uapi?

No reason; that's a mistake.

Quoted 18 lines
Quoted 8 lines
include/uapi/linux/bpf.h +0 -0
 /* flags for BPF_MAP_CREATE command */
 #define BPF_F_NO_PREALLOC	(1U << 0)
kernel/bpf/arraymap.c +0 -0
index f0d19bbb9211..a96e94696819 100644

kernel/bpf/arraymap.c static int array_map_update_elem(struct bpf_map *map, void *key, void *value, · line 306

Quoted 6 lines
kernel/bpf/arraymap.c +2 -1
@@ -302,7 +302,8 @@ static int array_map_update_elem(struct bpf_map *map, void *key, void *value,
 	u32 index = *(u32 *)key;
 	char *val;
 
-	if (unlikely((map_flags & ~BPF_F_LOCK) > BPF_EXIST))
+	if (unlikely((map_flags & ~BPF_CPU_MASK & ~BPF_F_LOCK &
+				  ~BPF_CPU) > BPF_EXIST))

that reads odd.
More traditional would be ~ (A | B | C)

#4 lore raw

Re: [PATCH bpf-next 1/3] bpf: Single-cpu updates for per-cpu maps

In reply to Paul Chaignon
Quoted 11 lines · Paul Chaignon

Currently, userspace programs have to update the values of all CPUs at once when updating per-cpu maps. This limitation prevents the update of a single CPU's value without the risk of missing concurrent updates on other CPU's values.

This patch allows userspace to update the value of a specific CPU in per-cpu maps. The CPU whose value should be updated is encoded in the 32 upper-bits of the flags argument, as follows. The new BPF_CPU flag can be combined with existing flags.

   bpf_map_update_elem(..., cpuid << 32 | BPF_CPU)

Some additional comments beyond Alexei's one.

Quoted 16 lines
---
  include/uapi/linux/bpf.h       |  4 +++
  kernel/bpf/arraymap.c          | 19 ++++++++-----
  kernel/bpf/hashtab.c           | 49 ++++++++++++++++++++--------------
  kernel/bpf/local_storage.c     | 16 +++++++----
  kernel/bpf/syscall.c           | 17 +++++++++---
  tools/include/uapi/linux/bpf.h |  4 +++
  6 files changed, 74 insertions(+), 35 deletions(-)
include/uapi/linux/bpf.h +0 -0
index dbbcf0b02970..2efb17d2c77a 100644

include/uapi/linux/bpf.h enum bpf_attach_type { · line 322

Quoted 7 lines
include/uapi/linux/bpf.h +4 -0
@@ -316,6 +316,10 @@ enum bpf_attach_type {
  #define BPF_NOEXIST	1 /* create new element if it didn't exist */
  #define BPF_EXIST	2 /* update existing element */
  #define BPF_F_LOCK	4 /* spin_lock-ed map_lookup/map_update */
+#define BPF_CPU		8 /* single-cpu update for per-cpu maps */
+
+/* CPU mask for single-cpu updates */
+#define BPF_CPU_MASK	0xFFFFFFFF00000000ULL

BPF_F_CPU_MASK?

Quoted 8 lines
include/uapi/linux/bpf.h +0 -0
  /* flags for BPF_MAP_CREATE command */
  #define BPF_F_NO_PREALLOC	(1U << 0)
kernel/bpf/arraymap.c +0 -0
index f0d19bbb9211..a96e94696819 100644

kernel/bpf/arraymap.c static int array_map_update_elem(struct bpf_map *map, void *key, void *value, · line 306

Quoted 6 lines
kernel/bpf/arraymap.c +2 -1
@@ -302,7 +302,8 @@ static int array_map_update_elem(struct bpf_map *map, void *key, void *value,
  	u32 index = *(u32 *)key;
  	char *val;
 
-	if (unlikely((map_flags & ~BPF_F_LOCK) > BPF_EXIST))
+	if (unlikely((map_flags & ~BPF_CPU_MASK & ~BPF_F_LOCK &
+				  ~BPF_CPU) > BPF_EXIST))

Maybe create a macro ARRAY_UPDATE_FLAG_MASK similar to existing ARRAY_CREATE_FLAG_MASK? This will make a little easier to follow, esp. we got three individual flags here.
There are possibly some other places as well below can be done in a similar way.

kernel/bpf/arraymap.c int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, · line 345

Quoted 9 lines
kernel/bpf/arraymap.c +1 -1
  		/* unknown flags */
  		return -EINVAL;
 
@@ -341,7 +342,7 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
  	int cpu, off = 0;
  	u32 size;
 
-	if (unlikely(map_flags > BPF_EXIST))
+	if (unlikely((map_flags & ~BPF_CPU_MASK & ~BPF_CPU) > BPF_EXIST))

~(BPF_F_CPU_MASK | BPF_F_CPU) or create a macro for like ARRAY_UPDATE_CPU_MASK for (BPF_F_CPU_MASK | BPF_F_CPU)?

Quoted 36 lines
kernel/bpf/arraymap.c +10 -4
  		/* unknown flags */
  		return -EINVAL;
 
@@ -349,7 +350,7 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
  		/* all elements were pre-allocated, cannot insert a new one */
  		return -E2BIG;
 
-	if (unlikely(map_flags == BPF_NOEXIST))
+	if (unlikely(map_flags & BPF_NOEXIST))
  		/* all elements already exist */
  		return -EEXIST;
 
@@ -362,9 +363,15 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
  	size = round_up(map->value_size, 8);
  	rcu_read_lock();
  	pptr = array->pptrs[index & array->index_mask];
-	for_each_possible_cpu(cpu) {
-		bpf_long_memcpy(per_cpu_ptr(pptr, cpu), value + off, size);
-		off += size;
+	if (map_flags & BPF_CPU) {
+		bpf_long_memcpy(per_cpu_ptr(pptr, map_flags >> 32), value,
+				size);
+	} else {
+		for_each_possible_cpu(cpu) {
+			bpf_long_memcpy(per_cpu_ptr(pptr, cpu), value + off,
+					size);
+			off += size;
+		}
  	}
  	rcu_read_unlock();
  	return 0;
kernel/bpf/hashtab.c +0 -0
index 22066a62c8c9..be45c7c4509f 100644

kernel/bpf/hashtab.c static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l) · line 700

Quoted 8 lines
kernel/bpf/hashtab.c +2 -2
@@ -695,12 +695,12 @@ static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l)
  }
 
  static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr,
-			    void *value, bool onallcpus)
+			    void *value, int cpuid)
  {
-	if (!onallcpus) {
+	if (cpuid == -1) {

Magic number -1 and -2 should be macros?

Quoted 8 lines
kernel/bpf/hashtab.c +1 -1
  		/* copy true value_size bytes */
  		memcpy(this_cpu_ptr(pptr), value, htab->map.value_size);
-	} else {
+	} else if (cpuid == -2) {
  		u32 size = round_up(htab->map.value_size, 8);
  		int off = 0, cpu;
 
@@ -709,6 +709,10 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr,

[...]

#5 lore raw

Re: [PATCH bpf-next 1/3] bpf: Single-cpu updates for per-cpu maps

Quoted 29 lines · Yonghong Song
Quoted 11 lines · Paul Chaignon

Currently, userspace programs have to update the values of all CPUs at once when updating per-cpu maps. This limitation prevents the update of a single CPU's value without the risk of missing concurrent updates on other CPU's values.

This patch allows userspace to update the value of a specific CPU in per-cpu maps. The CPU whose value should be updated is encoded in the 32 upper-bits of the flags argument, as follows. The new BPF_CPU flag can be combined with existing flags.

   bpf_map_update_elem(..., cpuid << 32 | BPF_CPU)

Some additional comments beyond Alexei's one.

Quoted 11 lines
---
  include/uapi/linux/bpf.h       |  4 +++
  kernel/bpf/arraymap.c          | 19 ++++++++-----
  kernel/bpf/hashtab.c           | 49 ++++++++++++++++++++--------------
  kernel/bpf/local_storage.c     | 16 +++++++----
  kernel/bpf/syscall.c           | 17 +++++++++---
  tools/include/uapi/linux/bpf.h |  4 +++
  6 files changed, 74 insertions(+), 35 deletions(-)
Quoted 13 lines

include/uapi/linux/bpf.h enum bpf_attach_type { · line 322

Quoted 12 lines
include/uapi/linux/bpf.h +4 -0
index dbbcf0b02970..2efb17d2c77a 100644
@@ -316,6 +316,10 @@ enum bpf_attach_type {
  #define BPF_NOEXIST 1 /* create new element if it didn't exist */
  #define BPF_EXIST   2 /* update existing element */
  #define BPF_F_LOCK  4 /* spin_lock-ed map_lookup/map_update */
+#define BPF_CPU              8 /* single-cpu update for per-cpu maps */
+
+/* CPU mask for single-cpu updates */
+#define BPF_CPU_MASK 0xFFFFFFFF00000000ULL

BPF_F_CPU_MASK?

Maybe even define this as a function-like macro:

#define BPF_F_CPU_NR(cpu) ((cpu) << 32)

so that it can be easily combined with other flags in code like so:

bpf_map_update_element(...., BPF_F_LOCK | BPF_F_CPU_NR(10))

BPF_F_CPU_NR can automatically set BPF_F_CPU flag as well, btw.

[...]

#6 lore raw

PATCH 2/3 selftests/bpf: Tests for single-cpu updates of per-cpu maps

In reply to Paul Chaignon

This patch adds test cases for single-cpu updates of per-cpu maps. It updates values for the first and last CPUs and checks that only their values were updated.

---
 tools/testing/selftests/bpf/test_maps.c | 34 ++++++++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)
tools/testing/selftests/bpf/test_maps.c +33 -1
index 02eae1e864c2..e3d3b15e1b91 100644
@@ -150,6 +150,7 @@ static void test_hashmap_percpu(unsigned int task, void *data)
 	BPF_DECLARE_PERCPU(long, value);
 	long long key, next_key, first_key;
 	int expected_key_mask = 0;
+	long new_value = 42;
 	int fd, i;
 
 	fd = bpf_create_map(BPF_MAP_TYPE_PERCPU_HASH, sizeof(key),
@@ -232,6 +233,23 @@ static void test_hashmap_percpu(unsigned int task, void *data)
 	key = 1;
 	assert(bpf_map_update_elem(fd, &key, value, BPF_EXIST) == 0);
 
+	/* Update value for a single CPU. */
+	assert(bpf_map_update_elem(fd, &key, &new_value,
+				   BPF_NOEXIST | BPF_CPU) == -1 &&
+	       errno == EEXIST);
+	assert(bpf_map_update_elem(fd, &key, &new_value,
+				   ((__u64)nr_cpus) << 32 | BPF_CPU) == -1 &&
+	       errno == EINVAL);
+	assert(bpf_map_update_elem(fd, &key, &new_value,
+				   BPF_EXIST | BPF_CPU) == 0);
+	assert(bpf_map_update_elem(fd, &key, &new_value,
+				   (nr_cpus - 1ULL) << 32 | BPF_CPU) == 0);
+	assert(bpf_map_lookup_elem(fd, &key, value) == 0);
+	assert(bpf_percpu(value, 0) == new_value);
+	for (i = 1; i < nr_cpus - 1; i++)
+		assert(bpf_percpu(value, i) == i + 100);
+	assert(bpf_percpu(value, nr_cpus - 1) == new_value);
+
 	/* Delete both elements. */
 	key = 1;
 	assert(bpf_map_delete_elem(fd, &key) == 0);
@@ -402,6 +420,7 @@ static void test_arraymap_percpu(unsigned int task, void *data)
 	unsigned int nr_cpus = bpf_num_possible_cpus();
 	BPF_DECLARE_PERCPU(long, values);
 	int key, next_key, fd, i;
+	long new_value = 42;
 
 	fd = bpf_create_map(BPF_MAP_TYPE_PERCPU_ARRAY, sizeof(key),
 			    sizeof(bpf_percpu(values, 0)), 2, 0);
@@ -449,8 +468,21 @@ static void test_arraymap_percpu(unsigned int task, void *data)
 	assert(bpf_map_get_next_key(fd, &next_key, &next_key) == -1 &&
 	       errno == ENOENT);
 
-	/* Delete shouldn't succeed. */
+	/* Update value for a single CPU */
 	key = 1;
+	assert(bpf_map_update_elem(fd, &key, &new_value,
+				   ((__u64)nr_cpus) << 32 | BPF_CPU) == -1 &&
+	       errno == EINVAL);
+	assert(bpf_map_update_elem(fd, &key, &new_value, BPF_CPU) == 0);
+	assert(bpf_map_update_elem(fd, &key, &new_value,
+				   (nr_cpus - 1ULL) << 32 | BPF_CPU) == 0);
+	assert(bpf_map_lookup_elem(fd, &key, values) == 0);
+	assert(bpf_percpu(values, 0) == new_value);
+	for (i = 1; i < nr_cpus - 1; i++)
+		assert(bpf_percpu(values, i) == i + 100);
+	assert(bpf_percpu(values, nr_cpus - 1) == new_value);
+
+	/* Delete shouldn't succeed. */
 	assert(bpf_map_delete_elem(fd, &key) == -1 && errno == EINVAL);
 
 	close(fd);
Signature · 1 line
2.24.0
#7 lore raw

PATCH 3/3 bpftool: Support single-cpu updates for per-cpu maps

In reply to Paul Chaignon

This patch adds support for the new BPF_CPU flag in bpftool, to enable single-cpu updates of per-cpu maps. It can be combined with existing flags; for example, to update the value for key 0 on CPU 9 only if it doesn't already exist:

bpftool map update key 0 0 0 0 value 1 0 0 0 noexist cpu 9

---
 .../bpf/bpftool/Documentation/bpftool-map.rst | 13 ++--
 tools/bpf/bpftool/bash-completion/bpftool     |  2 +-
 tools/bpf/bpftool/map.c                       | 70 ++++++++++++++-----
 3 files changed, 61 insertions(+), 24 deletions(-)
tools/bpf/bpftool/Documentation/bpftool-map.rst +8 -5
index cdeae8ae90ba..72aa9b72f08f 100644
@@ -25,7 +25,7 @@ MAP COMMANDS
 |	**bpftool** **map create**     *FILE* **type** *TYPE* **key** *KEY_SIZE* **value** *VALUE_SIZE* \
 |		**entries** *MAX_ENTRIES* **name** *NAME* [**flags** *FLAGS*] [**dev** *NAME*]
 |	**bpftool** **map dump**       *MAP*
-|	**bpftool** **map update**     *MAP* [**key** *DATA*] [**value** *VALUE*] [*UPDATE_FLAGS*]
+|	**bpftool** **map update**     *MAP* [**key** *DATA*] [**value** *VALUE*] [*UPDATE_FLAGS*]...
 |	**bpftool** **map lookup**     *MAP* [**key** *DATA*]
 |	**bpftool** **map getnext**    *MAP* [**key** *DATA*]
 |	**bpftool** **map delete**     *MAP*  **key** *DATA*
@@ -43,7 +43,7 @@ MAP COMMANDS
 |	*DATA* := { [**hex**] *BYTES* }
 |	*PROG* := { **id** *PROG_ID* | **pinned** *FILE* | **tag** *PROG_TAG* | **name** *PROG_NAME* }
 |	*VALUE* := { *DATA* | *MAP* | *PROG* }
-|	*UPDATE_FLAGS* := { **any** | **exist** | **noexist** }
+|	*UPDATE_FLAGS* := { **any** | **exist** | **noexist** | **cpu** *CPU_ID* }
 |	*TYPE* := { **hash** | **array** | **prog_array** | **perf_event_array** | **percpu_hash**
 |		| **percpu_array** | **stack_trace** | **cgroup_array** | **lru_hash**
 |		| **lru_percpu_hash** | **lpm_trie** | **array_of_maps** | **hash_of_maps**
@@ -73,9 +73,12 @@ DESCRIPTION
 	**bpftool map update**  *MAP* [**key** *DATA*] [**value** *VALUE*] [*UPDATE_FLAGS*]
 		  Update map entry for a given *KEY*.
 
-		  *UPDATE_FLAGS* can be one of: **any** update existing entry
-		  or add if doesn't exit; **exist** update only if entry already
-		  exists; **noexist** update only if entry doesn't exist.
+		  *UPDATE_FLAGS* can be: **any** update existing entry or add
+		  if doesn't exit; **exist** update only if entry already
+		  exists; **noexist** update only if entry doesn't exist;
+		  **cpu** update only value for given *CPU_ID* in per-CPU map.
+		  Only one of **any**, **exist**, and **noexist** can be
+		  specified.
 
 		  If the **hex** keyword is provided in front of the bytes
 		  sequence, the bytes are parsed as hexadeximal values, even if
tools/bpf/bpftool/bash-completion/bpftool +1 -1
index 754d8395e451..116bf3dffb47 100644
@@ -689,7 +689,7 @@ _bpftool()
                             esac
 
                             _bpftool_once_attr 'key'
-                            local UPDATE_FLAGS='any exist noexist'
+                            local UPDATE_FLAGS='any exist noexist cpu'
                             for (( idx=3; idx < ${#words[@]}-1; idx++ )); do
                                 if [[ ${words[idx]} == 'value' ]]; then
                                     # 'value' is present, but is not the last
tools/bpf/bpftool/map.c +52 -18

113 lines not on this page · raw message

Signature · 1 line
2.24.0