Skip to contentSkip to conversation

tools/sched_ext: Add example C schedulers

PATCH Superseded Emil Tsalapatis 2 participants 6 messages Jan 13 – Jan 21 Scheduler - sched_ext

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

v2 cover tools/sched_ext: Add example C schedulers

Add to the tree several C sched-ext schedulers developed and maintained out-of-tree by the sched-ext project over the years. These schedulers demonstrate sched-ext's feature set and provide a starting point for scheduler development. Place the schedulers together with the existing examples in tools/sched_ext.

Emil Tsalapatis (3):
tools/sched_ext: add scx_userland scheduler
tools/sched_ext: add scx_pair scheduler
tools/sched_ext: add arena based scheduler

 tools/sched_ext/Makefile           |   2 +-
 tools/sched_ext/scx_pair.bpf.c     | 610 +++++++++++++++++++++++++
 tools/sched_ext/scx_pair.c         | 180 ++++++++
 tools/sched_ext/scx_pair.h         |   9 +
 tools/sched_ext/scx_sdt.bpf.c      | 710 +++++++++++++++++++++++++++++
 tools/sched_ext/scx_sdt.c          | 101 ++++
 tools/sched_ext/scx_sdt.h          | 113 +++++
 tools/sched_ext/scx_userland.bpf.c | 344 ++++++++++++++
 tools/sched_ext/scx_userland.c     | 437 ++++++++++++++++++
 tools/sched_ext/scx_userland.h     |  17 +
 10 files changed, 2522 insertions(+), 1 deletion(-)
 create mode 100644 tools/sched_ext/scx_pair.bpf.c
 create mode 100644 tools/sched_ext/scx_pair.c
 create mode 100644 tools/sched_ext/scx_pair.h
 create mode 100644 tools/sched_ext/scx_sdt.bpf.c
 create mode 100644 tools/sched_ext/scx_sdt.c
 create mode 100644 tools/sched_ext/scx_sdt.h
 create mode 100644 tools/sched_ext/scx_userland.bpf.c
 create mode 100644 tools/sched_ext/scx_userland.c
 create mode 100644 tools/sched_ext/scx_userland.h
Signature · 1 line
2.49.0
#1 lore raw

PATCH v2 1/3 tools/sched_ext: add scx_userland scheduler

Add in the scx_userland scheduler that does vruntime-based scheduling in userspace code and communicates scheduling decisions to BPF by accessing and modifying globals through the skeleton.

---
 tools/sched_ext/Makefile           |   2 +-
 tools/sched_ext/scx_userland.bpf.c | 344 +++++++++++++++++++++++
 tools/sched_ext/scx_userland.c     | 437 +++++++++++++++++++++++++++++
 tools/sched_ext/scx_userland.h     |  17 ++
 4 files changed, 799 insertions(+), 1 deletion(-)
 create mode 100644 tools/sched_ext/scx_userland.bpf.c
 create mode 100644 tools/sched_ext/scx_userland.c
 create mode 100644 tools/sched_ext/scx_userland.h
tools/sched_ext/Makefile +1 -1
index e4bda2474060..12043a82a1a9 100644
@@ -189,7 +189,7 @@ $(INCLUDE_DIR)/%.bpf.skel.h: $(SCXOBJ_DIR)/%.bpf.o $(INCLUDE_DIR)/vmlinux.h $(BP
 
 SCX_COMMON_DEPS := include/scx/common.h include/scx/user_exit_info.h | $(BINDIR)
 
-c-sched-targets = scx_simple scx_cpu0 scx_qmap scx_central scx_flatcg
+c-sched-targets = scx_simple scx_cpu0 scx_qmap scx_central scx_flatcg scx_userland
 
 $(addprefix $(BINDIR)/,$(c-sched-targets)): \
 	$(BINDIR)/%: \
tools/sched_ext/scx_userland.bpf.c new file +344 -0

344 lines not on this page · raw message

tools/sched_ext/scx_userland.c new file +437 -0

437 lines not on this page · raw message

tools/sched_ext/scx_userland.h new file +17 -0
new file mode 100644
index 000000000000..684fb2dd5de9
@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2022 Meta, Inc */
+
+#ifndef __SCX_USERLAND_COMMON_H
+#define __SCX_USERLAND_COMMON_H
+
+/*
+ * An instance of a task that has been enqueued by the kernel for consumption
+ * by a user space global scheduler thread.
+ */
+struct scx_userland_enqueued_task {
+	__s32 pid;
+	u64 sum_exec_runtime;
+	u64 weight;
+};
+
+#endif  // __SCX_USERLAND_COMMON_H
Signature · 1 line
2.49.0
#2 lore raw

PATCH v2 2/3 tools/sched_ext: add scx_pair scheduler

In reply to Emil Tsalapatis

Add the scx_pair cgroup-based core scheduler.

---
 tools/sched_ext/Makefile       |   2 +-
 tools/sched_ext/scx_pair.bpf.c | 610 +++++++++++++++++++++++++++++++++
 tools/sched_ext/scx_pair.c     | 180 ++++++++++
 tools/sched_ext/scx_pair.h     |   9 +
 4 files changed, 800 insertions(+), 1 deletion(-)
 create mode 100644 tools/sched_ext/scx_pair.bpf.c
 create mode 100644 tools/sched_ext/scx_pair.c
 create mode 100644 tools/sched_ext/scx_pair.h
tools/sched_ext/Makefile +1 -1
index 12043a82a1a9..208e8f8fe4d8 100644
@@ -189,7 +189,7 @@ $(INCLUDE_DIR)/%.bpf.skel.h: $(SCXOBJ_DIR)/%.bpf.o $(INCLUDE_DIR)/vmlinux.h $(BP
 
 SCX_COMMON_DEPS := include/scx/common.h include/scx/user_exit_info.h | $(BINDIR)
 
-c-sched-targets = scx_simple scx_cpu0 scx_qmap scx_central scx_flatcg scx_userland
+c-sched-targets = scx_simple scx_cpu0 scx_qmap scx_central scx_flatcg scx_userland scx_pair
 
 $(addprefix $(BINDIR)/,$(c-sched-targets)): \
 	$(BINDIR)/%: \
tools/sched_ext/scx_pair.bpf.c new file +610 -0

610 lines not on this page · raw message

tools/sched_ext/scx_pair.c new file +180 -0

180 lines not on this page · raw message

tools/sched_ext/scx_pair.h new file +9 -0
new file mode 100644
index 000000000000..d9666a447d3f
@@ -0,0 +1,9 @@
+#ifndef __SCX_EXAMPLE_PAIR_H
+#define __SCX_EXAMPLE_PAIR_H
+
+enum {
+	MAX_QUEUED		= 4096,
+	MAX_CGRPS		= 4096,
+};
+
+#endif /* __SCX_EXAMPLE_PAIR_H */
Signature · 1 line
2.49.0
#3 lore raw

PATCH v2 3/3 tools/sched_ext: add arena based scheduler

In reply to Emil Tsalapatis

Add a scheduler that uses BPF arenas to manage task context data.

---
 tools/sched_ext/Makefile      |   2 +-
 tools/sched_ext/scx_sdt.bpf.c | 710 ++++++++++++++++++++++++++++++++++
 tools/sched_ext/scx_sdt.c     | 101 +++++
 tools/sched_ext/scx_sdt.h     | 113 ++++++
 4 files changed, 925 insertions(+), 1 deletion(-)
 create mode 100644 tools/sched_ext/scx_sdt.bpf.c
 create mode 100644 tools/sched_ext/scx_sdt.c
 create mode 100644 tools/sched_ext/scx_sdt.h
tools/sched_ext/Makefile +1 -1
index 208e8f8fe4d8..47ad7444677e 100644
@@ -189,7 +189,7 @@ $(INCLUDE_DIR)/%.bpf.skel.h: $(SCXOBJ_DIR)/%.bpf.o $(INCLUDE_DIR)/vmlinux.h $(BP
 
 SCX_COMMON_DEPS := include/scx/common.h include/scx/user_exit_info.h | $(BINDIR)
 
-c-sched-targets = scx_simple scx_cpu0 scx_qmap scx_central scx_flatcg scx_userland scx_pair
+c-sched-targets = scx_simple scx_cpu0 scx_qmap scx_central scx_flatcg scx_userland scx_pair scx_sdt
 
 $(addprefix $(BINDIR)/,$(c-sched-targets)): \
 	$(BINDIR)/%: \
tools/sched_ext/scx_sdt.bpf.c new file +710 -0

710 lines not on this page · raw message

tools/sched_ext/scx_sdt.c new file +101 -0

101 lines not on this page · raw message

tools/sched_ext/scx_sdt.h new file +113 -0

113 lines not on this page · raw message

Signature · 1 line
2.49.0
reviewer
#4 lore raw

Re: [PATCH v2 3/3] tools/sched_ext: add arena based scheduler

Hello,

On Tue, Jan 13, 2026 at 11:48:18AM -0500, Emil Tsalapatis wrote:
...

Quoted 42 lines
tools/sched_ext/scx_sdt.bpf.c new file +42 -0
+__hidden
+int scx_alloc_free_idx(struct scx_allocator *alloc, __u64 idx)
+{
+	const __u64 mask = (1 << SDT_TASK_ENTS_PER_PAGE_SHIFT) - 1;
+	sdt_desc_t *lv_desc[SDT_TASK_LEVELS];
+	sdt_desc_t * __arena *desc_children;
+	struct sdt_chunk __arena *chunk;
+	sdt_desc_t *desc;
+	struct sdt_data __arena *data;
+	__u64 level, shift, pos;
+	__u64 lv_pos[SDT_TASK_LEVELS];
+	int ret;
+	int i;
+
+	if (!alloc)
+		return 0;
+
+	desc = alloc->root;
+	if (unlikely(!desc))
+		return -EINVAL;
+
+	/* To appease the verifier. */
+	for (level = zero; level < SDT_TASK_LEVELS && can_loop; level++) {
+		lv_desc[level] = NULL;
+		lv_pos[level] = 0;
+	}
+
+	/* Find the leaf node containing the index. */
+	for (level = zero; level < SDT_TASK_LEVELS && can_loop; level++) {
+		shift = (SDT_TASK_LEVELS - 1 - level) * SDT_TASK_ENTS_PER_PAGE_SHIFT;
+		pos = (idx >> shift) & mask;
+
+		lv_desc[level] = desc;
+		lv_pos[level] = pos;
+
+		if (level == SDT_TASK_LEVELS - 1)
+			break;
+
+		chunk = desc->chunk;
+
+		desc_children = (sdt_desc_t * __arena *)chunk->descs;
+		desc = desc_children[pos];

tools/sched_ext/scx_sdt.bpf.c line 314

Quoted 10 lines
tools/sched_ext/scx_sdt.bpf.c new file +10 -0
+
+		if (unlikely(!desc))
+			return -EINVAL;
+	}
+
+	chunk = desc->chunk;
+
+	pos = idx & mask;
+	data = chunk->data[pos];
+	if (likely(!data)) {

AI review flagged this. Should this be "if (likely(data)) {"?

tools/sched_ext/scx_sdt.bpf.c line 323

Quoted 9 lines
tools/sched_ext/scx_sdt.bpf.c new file +9 -0
+		data[pos] = (struct sdt_data) {
+			.tid.genn = data->tid.genn + 1,
+		};
+
+		/* Zero out one word at a time. */
+		for (i = zero; i < alloc->pool.elem_size / 8 && can_loop; i++) {
+			data->payload[i] = 0;
+		}
+	}

Thanks.

Signature · 1 line
tejun
#5 lore raw

Re: [PATCH v2 3/3] tools/sched_ext: add arena based scheduler

Quoted 50 lines · Tejun Heo

Hello,

On Tue, Jan 13, 2026 at 11:48:18AM -0500, Emil Tsalapatis wrote:
...

Quoted 46 lines
tools/sched_ext/scx_sdt.bpf.c new file +46 -0
+__hidden
+int scx_alloc_free_idx(struct scx_allocator *alloc, __u64 idx)
+{
+	const __u64 mask = (1 << SDT_TASK_ENTS_PER_PAGE_SHIFT) - 1;
+	sdt_desc_t *lv_desc[SDT_TASK_LEVELS];
+	sdt_desc_t * __arena *desc_children;
+	struct sdt_chunk __arena *chunk;
+	sdt_desc_t *desc;
+	struct sdt_data __arena *data;
+	__u64 level, shift, pos;
+	__u64 lv_pos[SDT_TASK_LEVELS];
+	int ret;
+	int i;
+
+	if (!alloc)
+		return 0;
+
+	desc = alloc->root;
+	if (unlikely(!desc))
+		return -EINVAL;
+
+	/* To appease the verifier. */
+	for (level = zero; level < SDT_TASK_LEVELS && can_loop; level++) {
+		lv_desc[level] = NULL;
+		lv_pos[level] = 0;
+	}
+
+	/* Find the leaf node containing the index. */
+	for (level = zero; level < SDT_TASK_LEVELS && can_loop; level++) {
+		shift = (SDT_TASK_LEVELS - 1 - level) * SDT_TASK_ENTS_PER_PAGE_SHIFT;
+		pos = (idx >> shift) & mask;
+
+		lv_desc[level] = desc;
+		lv_pos[level] = pos;
+
+		if (level == SDT_TASK_LEVELS - 1)
+			break;
+
+		chunk = desc->chunk;
+
+		desc_children = (sdt_desc_t * __arena *)chunk->descs;
+		desc = desc_children[pos];
+
+		if (unlikely(!desc))
+			return -EINVAL;
+	}
Quoted 9 lines

tools/sched_ext/scx_sdt.bpf.c line 314

Quoted 6 lines
tools/sched_ext/scx_sdt.bpf.c new file +6 -0
+
+	chunk = desc->chunk;
+
+	pos = idx & mask;
+	data = chunk->data[pos];
+	if (likely(!data)) {

AI review flagged this. Should this be "if (likely(data)) {"?

It should, I will send an updated version.

Quoted 11 lines

tools/sched_ext/scx_sdt.bpf.c line 323

Quoted 9 lines
tools/sched_ext/scx_sdt.bpf.c new file +9 -0
+		data[pos] = (struct sdt_data) {
+			.tid.genn = data->tid.genn + 1,
+		};
+
+		/* Zero out one word at a time. */
+		for (i = zero; i < alloc->pool.elem_size / 8 && can_loop; i++) {
+			data->payload[i] = 0;
+		}
+	}

Thanks.