tools/sched_ext: Receive updates from SCX repo
Andrea Righi
reviewerPATCH 1/1 tools/sched_ext: Receive updates from SCX repo
Receive tools/sched_ext updates form https://github.com/sched-ext/scx to sync userspace bits:
- basic BPF arena allocator abstractions,
- additional process flags definitions,
- fixed is_migration_disabled() helper,
- separate out user_exit_info BPF and user space code.
This also fixes the following warning when building the selftests:
tools/sched_ext/include/scx/common.bpf.h:550:9: warning: 'likely' macro redefined [-Wmacro-redefined]
550 | #define likely(x) __builtin_expect(!!(x), 1)
| ^
--- .../include/scx/bpf_arena_common.bpf.h | 175 ++++++++++++++++++ .../sched_ext/include/scx/bpf_arena_common.h | 33 ++++ tools/sched_ext/include/scx/common.bpf.h | 102 +++++++++- tools/sched_ext/include/scx/common.h | 5 +- tools/sched_ext/include/scx/compat.bpf.h | 5 + .../include/scx/user_exit_info.bpf.h | 40 ++++ tools/sched_ext/include/scx/user_exit_info.h | 49 +---- .../include/scx/user_exit_info_common.h | 30 +++ tools/sched_ext/scx_central.bpf.c | 2 +- tools/sched_ext/scx_central.c | 1 + tools/sched_ext/scx_flatcg.bpf.c | 2 +- tools/sched_ext/scx_flatcg.c | 2 + tools/sched_ext/scx_simple.c | 2 + 13 files changed, 388 insertions(+), 60 deletions(-) create mode 100644 tools/sched_ext/include/scx/bpf_arena_common.bpf.h create mode 100644 tools/sched_ext/include/scx/bpf_arena_common.h create mode 100644 tools/sched_ext/include/scx/user_exit_info.bpf.h create mode 100644 tools/sched_ext/include/scx/user_exit_info_common.h
ChangeLog:
- Import only the bare minimum changes required by the scheduler examples and the selftests
- Link to the previous patch:
https://lore.kernel.org/all/[email protected]/
tools/sched_ext/include/scx/bpf_arena_common.bpf.h new file +175 -0
@@ -0,0 +1,175 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */ +#pragma once + +#ifndef PAGE_SIZE +#define PAGE_SIZE __PAGE_SIZE +/* +* for older kernels try sizeof(struct genradix_node) +* or flexible: +* static inline long __bpf_page_size(void) { +* return bpf_core_enum_value(enum page_size_enum___l, __PAGE_SIZE___l) ?: sizeof(struct genradix_node); +* } +* but generated code is not great. +*/ +#endif + +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) && !defined(BPF_ARENA_FORCE_ASM ) +#define __arena __attribute__((address_space(1))) +#define __arena_global __attribute__((address_space(1))) +#define cast_kern (ptr)/* nop for bpf prog. emitted by LLVM */ +#define cast_user (ptr)/* nop for bpf prog. emitted by LLVM */ +#else + +/* emit instruction: +* rX = rX .off = BPF_ADDR_SPACE_CAST .imm32 = (dst_as << 16) | src_as +* +* This is a workaround for LLVM compiler versions without +* __BPF_FEATURE_ADDR_SPACE_CAST that do not automatically cast between arena +* pointers and native kernel/userspace ones. In this case we explicitly do so +* with cast_kern() and cast_user(). E.g., in the Linux kernel tree, +* tools/testing/selftests/bpf includes tests that use these macros to implement +* linked lists and hashtables backed by arena memory. In sched_ext, we use +* cast_kern() and cast_user() for compatibility with older LLVM toolchains. +*/ +#ifndef bpf_addr_space_cast +#define bpf_addr_space_cast (var, dst_as, src_as)\ + asm volatile(".byte 0xBF; \ + .ifc %[reg], r0; \ + .byte 0x00; \ + .endif; \ + .ifc %[reg], r1; \ + .byte 0x11; \ + .endif; \ + .ifc %[reg], r2; \ + .byte 0x22; \ + .endif; \ + .ifc %[reg], r3; \ + .byte 0x33; \ + .endif; \ + .ifc %[reg], r4; \ + .byte 0x44; \ + .endif; \ + .ifc %[reg], r5; \ + .byte 0x55; \ + .endif; \ + .ifc %[reg], r6; \ + .byte 0x66; \ + .endif; \ + .ifc %[reg], r7; \ + .byte 0x77; \ + .endif; \ + .ifc %[reg], r8; \ + .byte 0x88; \ + .endif; \ + .ifc %[reg], r9; \ + .byte 0x99; \ + .endif; \ + .short %[off]; \ + .long %[as]" \ + : [reg]"+r"(var) \ + : [off]"i"(BPF_ADDR_SPACE_CAST) \ + , [as]"i"((dst_as << 16) | src_as)); +#endif + +#define __arena +#define __arena_global SEC(".addr_space.1") +#define cast_kern (ptr) bpf_addr_space_cast(ptr, 0, 1) +#define cast_user (ptr) bpf_addr_space_cast(ptr, 1, 0) +#endif + +void __arena* bpf_arena_alloc_pages(void *map, void __arena *addr, __u32 page_cnt, +int node_id,__u64 flags) __ksym__weak ; +void bpf_arena_free_pages(void *map, void __arena *ptr, __u32 page_cnt) __ksym __weak; + +/* +* Note that cond_break can only be portably used in the body of a breakable +* construct, whereas can_loop can be used anywhere. +*/ +#ifdef TEST +#define can_loop true +#define __cond_break (expr) expr +#else +#ifdef __BPF_FEATURE_MAY_GOTO +#define can_loop \ + ({ __label__ l_break, l_continue; \ + bool ret = true; \ + asm volatile goto("may_goto %l[l_break]" \ + :::: l_break); \ + goto l_continue; \ + l_break: ret = false; \ + l_continue:; \ + ret; \ + }) + +#define __cond_break (expr) \ + ({ __label__ l_break, l_continue; \ + asm volatile goto("may_goto %l[l_break]" \ + :::: l_break); \ + goto l_continue; \ + l_break: expr; \ + l_continue:; \ + }) +#else +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define can_loop \ + ({ __label__ l_break, l_continue; \ + bool ret = true; \ + asm volatile goto("1:.byte 0xe5; \ + .byte 0; \ + .long ((%l[l_break] - 1b - 8) / 8) & 0xffff; \ + .short 0" \ + :::: l_break); \ + goto l_continue; \ + l_break: ret = false; \ + l_continue:; \ + ret; \ + }) + +#define __cond_break (expr) \ + ({ __label__ l_break, l_continue; \ + asm volatile goto("1:.byte 0xe5; \ + .byte 0; \ + .long ((%l[l_break] - 1b - 8) / 8) & 0xffff; \ + .short 0" \ + :::: l_break); \ + goto l_continue; \ + l_break: expr; \ + l_continue:; \ + }) +#else +#define can_loop \ + ({ __label__ l_break, l_continue; \ + bool ret = true; \ + asm volatile goto("1:.byte 0xe5; \ + .byte 0; \ + .long (((%l[l_break] - 1b - 8) / 8) & 0xffff) << 16; \ + .short 0" \ + :::: l_break); \ + goto l_continue; \ + l_break: ret = false; \ + l_continue:; \ + ret; \ + }) + +#define __cond_break (expr) \ + ({ __label__ l_break, l_continue; \ + asm volatile goto("1:.byte 0xe5; \ + .byte 0; \ + .long (((%l[l_break] - 1b - 8) / 8) & 0xffff) << 16; \ + .short 0" \ + :::: l_break); \ + goto l_continue; \ + l_break: expr; \ + l_continue:; \ + }) +#endif /* __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ */ +#endif /* __BPF_FEATURE_MAY_GOTO */ +#endif /* TEST */ + +#define cond_break __cond_break(break) +#define cond_break_label (label) __cond_break(goto label) + + +void bpf_preempt_disable(void) __weak __ksym; +void bpf_preempt_enable(void) __weak __ksym;
tools/sched_ext/include/scx/bpf_arena_common.h new file +33 -0
@@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */ +#pragma once + +#ifndef arena_container_of +#define arena_container_of (ptr, type, member) \ + ({ \ + void __arena *__mptr = (void __arena *)(ptr); \ + ((type *)(__mptr - offsetof(type, member))); \ + }) +#endif + +/* Provide the definition of PAGE_SIZE. */ +#include <sys/user.h> + +#define __arena +#define __arg_arena +#define cast_kern (ptr)/* nop for user space */ +#define cast_user (ptr)/* nop for user space */ +char __attribute__((weak)) arena[1 ]; + +#ifndef offsetof +#define offsetof (type, member) ((unsigned long)&((type *)0)->member) +#endif + +static inline void __arena* bpf_arena_alloc_pages(void *map, void *addr, __u32 page_cnt, +int node_id,__u64 flags) +{ +return NULL ; +} +static inline void bpf_arena_free_pages(void *map, void __arena *ptr, __u32 page_cnt) +{ +}
tools/sched_ext/include/scx/common.bpf.h +93 -9
194 lines not on this page · raw message
tools/sched_ext/include/scx/common.h +3 -2
@@ -75,8 +75,9 @@ typedef int64_t s64;#include "enums.h" /* not available when building kernel tools/sched_ext */ -#if __has_include(<lib/sdt_task.h>) -#include <lib/sdt_task.h> +#if __has_include(<lib/sdt_task_defs.h>) +#include "bpf_arena_common.h" +#include <lib/sdt_task_defs.h> #endif #endif /* __SCHED_EXT_COMMON_H */
tools/sched_ext/include/scx/compat.bpf.h +5 -0
@@ -38,6 +38,7 @@ void scx_bpf_dispatch_from_dsq_set_slice___compat(struct bpf_iter_scx_dsq *it__i void scx_bpf_dispatch_from_dsq_set_vtime___compat(struct bpf_iter_scx_dsq *it__iter, u64 vtime) __ksym __weak; bool scx_bpf_dispatch_from_dsq___compat(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak; bool scx_bpf_dispatch_vtime_from_dsq___compat(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak; +int bpf_cpumask_populate(struct cpumask *dst, void *src, size_t src__sz) __ksym __weak;#define scx_bpf_dsq_insert (p, dsq_id, slice, enq_flags) \ (bpf_ksym_exists(scx_bpf_dsq_insert) ? \
@@ -82,6 +83,10 @@ bool scx_bpf_dispatch_vtime_from_dsq___compat(struct bpf_iter_scx_dsq *it__iter, scx_bpf_dispatch_vtime_from_dsq___compat((it__iter), (p), (dsq_id), (enq_flags)) : \ false)) +#define __COMPAT_bpf_cpumask_populate (cpumask, src, size__sz) \ + (bpf_ksym_exists(bpf_cpumask_populate) ? \ + (bpf_cpumask_populate(cpumask, src, size__sz)) : -EOPNOTSUPP) +#define scx_bpf_dispatch (p, dsq_id, slice, enq_flags) \ _Static_assert(false, "scx_bpf_dispatch() renamed to scx_bpf_dsq_insert()")
tools/sched_ext/include/scx/user_exit_info.bpf.h new file +40 -0
@@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* +* Define struct user_exit_info which is shared between BPF and userspace parts +* to communicate exit status and other information. +* +* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. +* Copyright (c) 2022 Tejun Heo <[email protected]> +* Copyright (c) 2022 David Vernet <[email protected]> +*/ + +#ifndef __USER_EXIT_INFO_BPF_H +#define __USER_EXIT_INFO_BPF_H + +#ifndef LSP +#include "vmlinux.h" +#endif +#include <bpf/bpf_core_read.h> + +#include "user_exit_info_common.h" + +#define UEI_DEFINE (__name) \ + char RESIZABLE_ARRAY(data, __name##_dump); \ + const volatile u32 __name##_dump_len; \ + struct user_exit_info __name SEC(".data") + +#define UEI_RECORD (__uei_name, __ei) ({ \ + bpf_probe_read_kernel_str(__uei_name.reason, \ + sizeof(__uei_name.reason), (__ei)->reason); \ + bpf_probe_read_kernel_str(__uei_name.msg, \ + sizeof(__uei_name.msg), (__ei)->msg); \ + bpf_probe_read_kernel_str(__uei_name##_dump, \ + __uei_name##_dump_len, (__ei)->dump); \ + if (bpf_core_field_exists((__ei)->exit_code)) \ + __uei_name.exit_code = (__ei)->exit_code; \ +/* use __sync to force memory barrier */ \ +__sync_val_compare_and_swap (&__uei_name.kind, __uei_name.kind, \ + (__ei)->kind); \ +}) + +#endif /* __USER_EXIT_INFO_BPF_H */
tools/sched_ext/include/scx/user_exit_info.h +2 -47
62 lines not on this page · raw message
tools/sched_ext/include/scx/user_exit_info_common.h new file +30 -0
@@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* +* Define struct user_exit_info which is shared between BPF and userspace parts +* to communicate exit status and other information. +* +* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. +* Copyright (c) 2022 Tejun Heo <[email protected]> +* Copyright (c) 2022 David Vernet <[email protected]> +*/ +#ifndef __USER_EXIT_INFO_COMMON_H +#define __USER_EXIT_INFO_COMMON_H + +#ifdef LSP +#include "../vmlinux.h" +#endif + +enum uei_sizes { +UEI_REASON_LEN =128 , +UEI_MSG_LEN =1024 , +UEI_DUMP_DFL_LEN =32768 , +}; + +struct user_exit_info { +int kind; +s64 exit_code; +char reason[UEI_REASON_LEN ]; +char msg[UEI_MSG_LEN ]; +}; + +#endif /* __USER_EXIT_INFO_COMMON_H */
tools/sched_ext/scx_central.bpf.c +1 -1
@@ -1,6 +1,6 @@/* SPDX-License-Identifier: GPL-2.0 */ /* -* A central FIFO sched_ext scheduler which demonstrates the followings: +* A central FIFO sched_ext scheduler which demonstrates the following: * * a. Making all scheduling decisions from one CPU: *
tools/sched_ext/scx_central.c +1 -0
@@ -61,6 +61,7 @@ int main(int argc, char **argv) skel->rodata->nr_cpu_ids =libbpf_num_possible_cpus (); skel->rodata->slice_ns =__COMPAT_ENUM_OR_ZERO ("scx_public_consts" ,"SCX_SLICE_DFL" ); +assert (skel->rodata->nr_cpu_ids >0 );assert (skel->rodata->nr_cpu_ids <=INT32_MAX );while ((opt =getopt (argc, argv,"s:c:pvh" )) !=-1 ) {
tools/sched_ext/scx_flatcg.bpf.c +1 -1
@@ -950,5 +950,5 @@ SCX_OPS_DEFINE(flatcg_ops, .cgroup_move = (void *)fcg_cgroup_move, .init = (void *)fcg_init, .exit = (void *)fcg_exit, - .flags = SCX_OPS_ENQ_EXITING, + .flags = SCX_OPS_HAS_CGROUP_WEIGHT | SCX_OPS_ENQ_EXITING, .name = "flatcg");
tools/sched_ext/scx_flatcg.c +2 -0
@@ -6,6 +6,7 @@*/ #include <stdio.h> #include <signal.h> +#include <assert.h> #include <unistd.h> #include <libgen.h> #include <limits.h>
@@ -137,6 +138,7 @@ int main(int argc, char **argv) skel =SCX_OPS_OPEN (flatcg_ops, scx_flatcg); skel->rodata->nr_cpus =libbpf_num_possible_cpus (); +assert (skel->rodata->nr_cpus >0 ); skel->rodata->cgrp_slice_ns =__COMPAT_ENUM_OR_ZERO ("scx_public_consts" ,"SCX_SLICE_DFL" );while ((opt =getopt (argc, argv,"s:i:dfvh" )) !=-1 ) {
tools/sched_ext/scx_simple.c +2 -0
@@ -7,6 +7,7 @@#include <stdio.h> #include <unistd.h> #include <signal.h> +#include <assert.h> #include <libgen.h> #include <bpf/bpf.h> #include <scx/common.h>
@@ -41,6 +42,7 @@ static void sigint_handler(int simple)static void read_stats (struct scx_simple *skel,__u64 *stats) {int nr_cpus =libbpf_num_possible_cpus (); +assert (nr_cpus >0 );__u64 cnts[2 ][nr_cpus];__u32 idx;
Signature · 1 line
2.50.1
Quoted 11 lines · Andrea Righi
Receive
tools/sched_extupdates form https://github.com/sched-ext/scx to sync userspace bits:- basic BPF arena allocator abstractions,
- additional process flags definitions,
- fixed
is_migration_disabled()helper,- separate out
user_exit_infoBPF and user space code.
Quoted 8 lines
This also fixes the following warning when building the selftests:
tools/sched_ext/include/scx/common.bpf.h:550:9: warning: 'likely' macro redefined [-Wmacro-redefined]
550 | #define likely(x)__builtin_expect(!!(x), 1)
| ^Co-developed-by Cheng-Yang Chou <[email protected]>Signed-off-by Andrea Righi <[email protected]>
Applied to sched_ext/for-6.18.
Thanks.
Signature · 1 line
tejun