sys_ringbuffer
Subsystems (9)
cover sys_ringbuffer
New syscall for mapping generic ringbuffers for arbitary (supported) file descriptors.
Ringbuffers can be created either when requested or at file open time, and can be mapped into multiple address spaces (naturally, since files can be shared as well).
Initial motivation is for fuse, but I plan on adding support to pipes and possibly sockets as well - pipes are a particularly interesting use case, because if both the sender and receiver of a pipe opt in to the new ringbuffer interface, we can make them the _same_ ringbuffer for true zero copy IO, while being backwards compatible with existing pipes.
the ringbuffer_wait and ringbuffer_wakeup syscalls are probably going away in a future iteration, in favor of just using futexes.
In my testing, reading/writing from the ringbuffer 16 bytes at a time is ~7x faster than using read/write syscalls - and I was testing with mitigations off, real world benefit will be even higher.
Kent Overstreet (5):
darray: lift from bcachefs
darray: Fix darray_for_each_reverse() when darray is empty
fs: sys_ringbuffer
ringbuffer: Test device
ringbuffer: Userspace test helper
MAINTAINERS | 7 +
arch/x86/entry/syscalls/syscall_32.tbl | 3 +
arch/x86/entry/syscalls/syscall_64.tbl | 3 +
fs/Makefile | 2 +
fs/bcachefs/Makefile | 1 -
fs/bcachefs/btree_types.h | 2 +-
fs/bcachefs/btree_update.c | 2 +
fs/bcachefs/btree_write_buffer_types.h | 2 +-
fs/bcachefs/fsck.c | 2 +-
fs/bcachefs/journal_io.h | 2 +-
fs/bcachefs/journal_sb.c | 2 +-
fs/bcachefs/sb-downgrade.c | 3 +-
fs/bcachefs/sb-errors_types.h | 2 +-
fs/bcachefs/sb-members.h | 3 +-
fs/bcachefs/subvolume.h | 1 -
fs/bcachefs/subvolume_types.h | 2 +-
fs/bcachefs/thread_with_file_types.h | 2 +-
fs/bcachefs/util.h | 28 +-
fs/ringbuffer.c | 474 ++++++++++++++++++++++++
fs/ringbuffer_test.c | 209 +++++++++++
{fs/bcachefs => include/linux}/darray.h | 61 +--
include/linux/darray_types.h | 22 ++
include/linux/fs.h | 2 +
include/linux/mm_types.h | 4 +
include/linux/ringbuffer_sys.h | 18 +
include/uapi/linux/futex.h | 1 +
include/uapi/linux/ringbuffer_sys.h | 40 ++
init/Kconfig | 9 +
kernel/fork.c | 2 +
lib/Kconfig.debug | 5 +
lib/Makefile | 2 +-
{fs/bcachefs => lib}/darray.c | 12 +-
tools/ringbuffer/Makefile | 3 +
tools/ringbuffer/ringbuffer-test.c | 254 +++++++++++++
34 files changed, 1125 insertions(+), 62 deletions(-)
create mode 100644 fs/ringbuffer.c
create mode 100644 fs/ringbuffer_test.c
rename {fs/bcachefs => include/linux}/darray.h (63%)
create mode 100644 include/linux/darray_types.h
create mode 100644 include/linux/ringbuffer_sys.h
create mode 100644 include/uapi/linux/ringbuffer_sys.h
rename {fs/bcachefs => lib}/darray.c (56%)
create mode 100644 tools/ringbuffer/Makefile
create mode 100644 tools/ringbuffer/ringbuffer-test.cSignature · 1 line
2.45.1
PATCH 1/5 darray: lift from bcachefs
dynamic arrays - inspired from CCAN darrays, basically c++ stl vectors.
Used by thread_with_stdio, which is also being lifted from bcachefs for xfs.
---
MAINTAINERS | 7 +++
fs/bcachefs/Makefile | 1 -
fs/bcachefs/btree_types.h | 2 +-
fs/bcachefs/btree_update.c | 2 +
fs/bcachefs/btree_write_buffer_types.h | 2 +-
fs/bcachefs/fsck.c | 2 +-
fs/bcachefs/journal_io.h | 2 +-
fs/bcachefs/journal_sb.c | 2 +-
fs/bcachefs/sb-downgrade.c | 3 +-
fs/bcachefs/sb-errors_types.h | 2 +-
fs/bcachefs/sb-members.h | 3 +-
fs/bcachefs/subvolume.h | 1 -
fs/bcachefs/subvolume_types.h | 2 +-
fs/bcachefs/thread_with_file_types.h | 2 +-
fs/bcachefs/util.h | 28 +-----------
{fs/bcachefs => include/linux}/darray.h | 59 ++++++++++++++++---------
include/linux/darray_types.h | 22 +++++++++
lib/Makefile | 2 +-
{fs/bcachefs => lib}/darray.c | 12 ++++-
19 files changed, 95 insertions(+), 61 deletions(-)
rename {fs/bcachefs => include/linux}/darray.h (66%)
create mode 100644 include/linux/darray_types.h
rename {fs/bcachefs => lib}/darray.c (56%)MAINTAINERS +7 -0
@@ -6010,6 +6010,13 @@ F: net/ax25/ax25_out.c F: net/ax25/ax25_timer.c F: net/ax25/sysctl_net_ax25.c +DARRAY +M: Kent Overstreet <[email protected]> +L: [email protected] +S: Maintained +F: include/linux/darray.h +F: include/linux/darray_types.h + DATA ACCESS MONITOR M: SeongJae Park <[email protected]> L: [email protected]
fs/bcachefs/Makefile +0 -1
@@ -28,7 +28,6 @@ bcachefs-y := \ checksum.o \ clock.o \ compress.o \ - darray.o \ debug.o \ dirent.o \ disk_groups.o \
fs/bcachefs/btree_types.h +1 -1
@@ -2,13 +2,13 @@#ifndef _BCACHEFS_BTREE_TYPES_H#define _BCACHEFS_BTREE_TYPES_H +#include <linux/darray_types.h> #include <linux/list.h> #include <linux/rhashtable.h> #include "bbpos_types.h" #include "btree_key_cache_types.h" #include "buckets_types.h" -#include "darray.h" #include "errcode.h" #include "journal_types.h" #include "replicas_types.h"
fs/bcachefs/btree_update.c +2 -0
@@ -14,6 +14,8 @@#include "snapshot.h" #include "trace.h" +#include <linux/darray.h> +static inline int btree_insert_entry_cmp (const struct btree_insert_entry *l,const struct btree_insert_entry *r) {
fs/bcachefs/btree_write_buffer_types.h +1 -1
@@ -2,7 +2,7 @@#ifndef _BCACHEFS_BTREE_WRITE_BUFFER_TYPES_H#define _BCACHEFS_BTREE_WRITE_BUFFER_TYPES_H -#include "darray.h" +#include <linux/darray_types.h> #include "journal_types.h" #define BTREE_WRITE_BUFERED_VAL_U64s_MAX 4
fs/bcachefs/fsck.c +1 -1
@@ -5,7 +5,6 @@#include "btree_cache.h" #include "btree_update.h" #include "buckets.h" -#include "darray.h" #include "dirent.h" #include "error.h" #include "fs-common.h"
@@ -18,6 +17,7 @@#include "xattr.h" #include <linux/bsearch.h> +#include <linux/darray.h> #include <linux/dcache.h> /* struct qstr */ /*
fs/bcachefs/journal_io.h +1 -1
@@ -2,7 +2,7 @@#ifndef _BCACHEFS_JOURNAL_IO_H#define _BCACHEFS_JOURNAL_IO_H -#include "darray.h" +#include <linux/darray_types.h> void bch2_journal_pos_from_member_info_set (struct bch_fs *);void bch2_journal_pos_from_member_info_resume (struct bch_fs *);
fs/bcachefs/journal_sb.c +1 -1
@@ -2,8 +2,8 @@#include "bcachefs.h" #include "journal_sb.h" -#include "darray.h" +#include <linux/darray.h> #include <linux/sort.h> /* BCH_SB_FIELD_journal: */
fs/bcachefs/sb-downgrade.c +2 -1
@@ -6,12 +6,13 @@*/ #include "bcachefs.h" -#include "darray.h" #include "recovery_passes.h" #include "sb-downgrade.h" #include "sb-errors.h" #include "super-io.h" +#include <linux/darray.h> +#define RECOVERY_PASS_ALL_FSCK BIT_ULL(63)/*
fs/bcachefs/sb-errors_types.h +1 -1
@@ -2,7 +2,7 @@#ifndef _BCACHEFS_SB_ERRORS_TYPES_H#define _BCACHEFS_SB_ERRORS_TYPES_H -#include "darray.h" +#include <linux/darray_types.h> #define BCH_SB_ERRS () \ x(clean_but_journal_not_empty, 0) \
fs/bcachefs/sb-members.h +2 -1
@@ -2,9 +2,10 @@#ifndef _BCACHEFS_SB_MEMBERS_H#define _BCACHEFS_SB_MEMBERS_H -#include "darray.h" #include "bkey_types.h" +#include <linux/darray.h> +extern char *const bch2_member_error_strs[]; static inline struct bch_member *
fs/bcachefs/subvolume.h +0 -1
@@ -2,7 +2,6 @@#ifndef _BCACHEFS_SUBVOLUME_H#define _BCACHEFS_SUBVOLUME_H -#include "darray.h" #include "subvolume_types.h" enum bch_validate_flags ;
fs/bcachefs/subvolume_types.h +1 -1
@@ -2,7 +2,7 @@#ifndef _BCACHEFS_SUBVOLUME_TYPES_H#define _BCACHEFS_SUBVOLUME_TYPES_H -#include "darray.h" +#include <linux/darray_types.h> typedef DARRAY (u32)snapshot_id_list ;
fs/bcachefs/thread_with_file_types.h +1 -1
@@ -2,7 +2,7 @@ #ifndef _BCACHEFS_THREAD_WITH_FILE_TYPES_H #define _BCACHEFS_THREAD_WITH_FILE_TYPES_H -#include "darray.h" +#include <linux/darray_types.h> struct stdio_buf { spinlock_t lock;
fs/bcachefs/util.h +2 -26
@@ -5,22 +5,22 @@#include <linux/bio.h> #include <linux/blkdev.h> #include <linux/closure.h> +#include <linux/darray.h> #include <linux/errno.h> #include <linux/freezer.h> #include <linux/kernel.h> -#include <linux/sched/clock.h> #include <linux/llist.h> #include <linux/log2.h> #include <linux/percpu.h> #include <linux/preempt.h> #include <linux/ratelimit.h> +#include <linux/sched/clock.h> #include <linux/slab.h> #include <linux/vmalloc.h> #include <linux/workqueue.h> #include "mean_and_variance.h" -#include "darray.h" #include "time_stats.h" struct closure ;
@@ -626,30 +626,6 @@ static inline void memset_u64s_tail(void *s, int c, unsigned bytes)memset (s + bytes, c, rem); } -/* just the memmove, doesn't update @_nr */ -#define __array_insert_item (_array, _nr, _pos) \ - memmove(&(_array)[(_pos) + 1], \ - &(_array)[(_pos)], \ - sizeof((_array)[0]) * ((_nr) - (_pos))) - -#define array_insert_item (_array, _nr, _pos, _new_item) \ -do { \ - __array_insert_item(_array, _nr, _pos); \ - (_nr)++; \ - (_array)[(_pos)] = (_new_item); \ -} while (0) - -#define array_remove_items (_array, _nr, _pos, _nr_to_remove) \ -do { \ - (_nr) -= (_nr_to_remove); \ - memmove(&(_array)[(_pos)], \ - &(_array)[(_pos) + (_nr_to_remove)], \ - sizeof((_array)[0]) * ((_nr) - (_pos))); \ -} while (0) - -#define array_remove_item (_array, _nr, _pos) \ - array_remove_items(_array, _nr, _pos, 1) -static inline void __move_gap (void *array,size_t element_size,size_t nr,size_t size,size_t old_gap,size_t new_gap)
include/linux/darray.h renamed from fs/bcachefs/darray.h +39 -20
93 lines not on this page · raw message
include/linux/darray_types.h new file +22 -0
@@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* +* (C) 2022-2024 Kent Overstreet <[email protected]> +*/ +#ifndef _LINUX_DARRAY_TYpES_H +#define _LINUX_DARRAY_TYpES_H + +#include <linux/types.h> + +#define DARRAY_PREALLOCATED (_type, _nr) \ +struct { \ + size_t nr, size; \ + _type *data; \ + _type preallocated[_nr]; \ +} + +#define DARRAY (_type) DARRAY_PREALLOCATED(_type, 0) + +typedef DARRAY (char )darray_char ; +typedef DARRAY (char *)darray_str ; + +#endif /* _LINUX_DARRAY_TYpES_H */
lib/Makefile +1 -1
@@ -48,7 +48,7 @@ obj-y += bcd.o sort.o parser.o debug_locks.o random32.o \ bsearch.o find_bit.o llist.o lwq.o memweight.o kfifo.o \ percpu-refcount.o rhashtable.o base64.o \ once.o refcount.o rcuref.o usercopy.o errseq.o bucket_locks.o \ - generic-radix-tree.o bitmap-str.o + generic-radix-tree.o bitmap-str.o darray.o obj-$(CONFIG_STRING_KUNIT_TEST) += string_kunit.o obj-y += string_helpers.o obj-$(CONFIG_STRING_HELPERS_KUNIT_TEST) += string_helpers_kunit.o
lib/darray.c renamed from fs/bcachefs/darray.c +10 -2
@@ -1,10 +1,14 @@// SPDX-License-Identifier: GPL-2.0 +/* +* (C) 2022-2024 Kent Overstreet <[email protected]> +*/ +#include <linux/darray.h> #include <linux/log2.h> +#include <linux/module.h> #include <linux/slab.h> -#include "darray.h" -int __bch2_darray_resize (darray_char *d,size_t element_size,size_t new_size,gfp_t gfp) +int __darray_resize_slowpath (darray_char *d,size_t element_size,size_t new_size,gfp_t gfp) {if (new_size > d->size) { new_size =roundup_pow_of_two (new_size);
@@ -22,3 +26,7 @@ int __bch2_darray_resize(darray_char *d, size_t element_size, size_t new_size, greturn 0 ; } +EXPORT_SYMBOL_GPL (__darray_resize_slowpath); + +MODULE_AUTHOR ("Kent Overstreet" ); +MODULE_LICENSE ("GPL" );
Signature · 1 line
2.45.1
kernel test robot
build botRe: [PATCH 1/5] darray: lift from bcachefs
kernel test robot noticed the following build errors:
Hi Kent,
kernel test robot noticed the following build errors:
[auto build test ERROR on tip/locking/core]
[also build test ERROR on linus/master v6.10-rc2]
[cannot apply to akpm-mm/mm-nonmm-unstable tip/x86/asm akpm-mm/mm-everything next-20240531] [If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Kent-Overstreet/darray-lift-from-bcachefs/20240603-083536 base: tip/locking/core patch link: https://lore.kernel.org/r/20240603003306.2030491-2-kent.overstreet%40linux.dev patch subject: [PATCH 1/5] darray: lift from bcachefs config: i386-buildonly-randconfig-004-20240603 (https://download.01.org/0day-ci/archive/20240603/[email protected]/config) compiler: gcc-10 (Ubuntu 10.5.0-1ubuntu1) 10.5.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240603/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All errors (new ones prefixed by >>):
In file included from fs/bcachefs/bcachefs.h:461,
from fs/bcachefs/reflink.c:2:
>> fs/bcachefs/btree_node_scan_types.h:5:10: fatal error: darray.h: No such file or directory
5 | #include "darray.h"
| ^~~~~~~~~~compilation terminated.
vim +5 fs/bcachefs/btree_node_scan_types.h
4409b8081d1624 Kent Overstreet 2024-03-11 4 4409b8081d1624 Kent Overstreet 2024-03-11 @5 #include "darray.h" 4409b8081d1624 Kent Overstreet 2024-03-11 6
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
PATCH 2/5 darray: Fix darray_for_each_reverse() when darray is empty
In reply to Kent Overstreet--- include/linux/darray.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/darray.h +1 -1
@@ -108,7 +108,7 @@ do { \ for (typeof(&(_d).data[0]) _i = (_d).data; _i < (_d).data + (_d).nr; _i++)#define darray_for_each_reverse (_d, _i) \ - for (typeof(&(_d).data[0]) _i = (_d).data + (_d).nr - 1; _i >= (_d).data; --_i) + for (typeof(&(_d).data[0]) _i = (_d).data + (_d).nr - 1; (_d).data && _i >= (_d).data; --_i)#define darray_init (_d) \ do { \
Signature · 1 line
2.45.1
PATCH 3/5 fs: sys_ringbuffer
In reply to Kent OverstreetAdd new syscalls for generic ringbuffers that can be attached to arbitrary (supporting) file descriptors.
A ringbuffer consists of:
- a single page for head/tail pointers, size/mask, and other ancilliary metadata, described by 'struct ringbuffer_ptrs'
- a data buffer, consisting of one or more pages mapped at 'ringbuffer_ptrs.data_offset' above the address of 'ringbuffer_ptrs'
The data buffer is always a power of two size. Head and tail pointers are u32 byte offsets, and they are stored unmasked (i.e., they use the full 32 bit range) - they must be masked for reading.
- ringbuffer(int fd, int rw, u32 size, ulong *addr)
Create or get address of an existing ringbuffer for either reads or writes, of at least size bytes, and attach it to the given file descriptor; the address of the ringbuffer is returned via addr.
Since files can be shared between processes in different address spaces a ringbuffer may be mapped into multiple address spaces via this syscall.
- ringbuffer_wait(int fd, int rw)
Wait for space to be availaable (on a ringbuffer for writing), or data to be available (on a ringbuffer for writing).
todo: add parameters for timeout, minimum amount of data/space to wait for
- ringbuffer_wakeup(int fd, int rw)
Required after writing to a previously empty ringbuffer, or reading from a previously full ringbuffer to notify waiters on the other end
todo - investigate integrating with futexes?
todo - add extra fields to ringbuffer_ptrs for waiting on a minimum amount of data/space, i.e. to signal when a wakeup is required
Kernel interfaces:
- To indicate that ringbuffers are supported on a file, set FOP_RINGBUFFER_READ and/or FOP_RINGBUFFER_WRITE in your file_operations.
- To read or write to a file's associated ringbuffers (file->f_ringbuffer), use ringbuffer_read() or ringbuffer_write().
--- arch/x86/entry/syscalls/syscall_32.tbl | 3 + arch/x86/entry/syscalls/syscall_64.tbl | 3 + fs/Makefile | 1 + fs/ringbuffer.c | 474 +++++++++++++++++++++++++ include/linux/fs.h | 2 + include/linux/mm_types.h | 4 + include/linux/ringbuffer_sys.h | 18 + include/uapi/linux/futex.h | 1 + include/uapi/linux/ringbuffer_sys.h | 40 +++ init/Kconfig | 9 + kernel/fork.c | 2 + 11 files changed, 557 insertions(+) create mode 100644 fs/ringbuffer.c create mode 100644 include/linux/ringbuffer_sys.h create mode 100644 include/uapi/linux/ringbuffer_sys.h
arch/x86/entry/syscalls/syscall_32.tbl +3 -0
@@ -467,3 +467,6 @@ 460 i386 lsm_set_self_attr sys_lsm_set_self_attr 461 i386 lsm_list_modules sys_lsm_list_modules 462 i386 mseal sys_mseal +463 i386 ringbuffer sys_ringbuffer +464 i386 ringbuffer_wait sys_ringbuffer_wait +465 i386 ringbuffer_wakeup sys_ringbuffer_wakeup
arch/x86/entry/syscalls/syscall_64.tbl +3 -0
@@ -384,6 +384,9 @@ 460 common lsm_set_self_attr sys_lsm_set_self_attr 461 common lsm_list_modules sys_lsm_list_modules 462 common mseal sys_mseal +463 common ringbuffer sys_ringbuffer +464 common ringbuffer_wait sys_ringbuffer_wait +465 common ringbuffer_wakeup sys_ringbuffer_wakeup # # Due to a historical design error, certain syscalls are numbered differently
fs/Makefile +1 -0
@@ -28,6 +28,7 @@ obj-$(CONFIG_TIMERFD) += timerfd.o obj-$(CONFIG_EVENTFD) += eventfd.o obj-$(CONFIG_USERFAULTFD) += userfaultfd.o obj-$(CONFIG_AIO) += aio.o +obj-$(CONFIG_RINGBUFFER) += ringbuffer.o obj-$(CONFIG_FS_DAX) += dax.o obj-$(CONFIG_FS_ENCRYPTION) += crypto/ obj-$(CONFIG_FS_VERITY) += verity/
fs/ringbuffer.c new file +474 -0
474 lines not on this page · raw message
include/linux/fs.h +2 -0
@@ -1996,6 +1996,7 @@ struct offset_ctx;typedef unsigned int __bitwise fop_flags_t ; +struct ringbuffer ;struct file_operations {struct module *owner;fop_flags_t fop_flags;
@@ -2004,6 +2005,7 @@ struct file_operations {ssize_t (*write) (struct file *,const char __user *,size_t ,loff_t *);ssize_t (*read_iter) (struct kiocb *,struct iov_iter *);ssize_t (*write_iter) (struct kiocb *,struct iov_iter *); +struct ringbuffer *(*ringbuffer)(struct file *,int );int (*iopoll)(struct kiocb *kiocb,struct io_comp_batch *,unsigned int flags);int (*iterate_shared) (struct file *,struct dir_context *);
include/linux/mm_types.h +4 -0
@@ -5,6 +5,7 @@#include <linux/mm_types_task.h> #include <linux/auxvec.h> +#include <linux/darray_types.h> #include <linux/kref.h> #include <linux/list.h> #include <linux/spinlock.h>
@@ -911,6 +912,9 @@ struct mm_struct {spinlock_t ioctx_lock;struct kioctx_table __rcu *ioctx_table;#endif +#ifdef CONFIG_RINGBUFFER +DARRAY (struct ringbuffer *) ringbuffers; +#endif #ifdef CONFIG_MEMCG /* * "owner" points to a task that is regarded as the canonical
include/linux/ringbuffer_sys.h new file +18 -0
@@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_RINGBUFFER_SYS_H +#define _LINUX_RINGBUFFER_SYS_H + +#include <linux/darray_types.h> +#include <linux/spinlock_types.h> +#include <uapi/linux/ringbuffer_sys.h> + +struct mm_struct ; +void ringbuffer_mm_exit (struct mm_struct *mm); + +void ringbuffer_free (struct ringbuffer *rb); +struct ringbuffer *ringbuffer_alloc (u32 size); + +ssize_t ringbuffer_read_iter (struct ringbuffer *rb,struct iov_iter *iter,bool nonblock); +ssize_t ringbuffer_write_iter (struct ringbuffer *rb,struct iov_iter *iter,bool nonblock); + +#endif /* _LINUX_RINGBUFFER_SYS_H */
include/uapi/linux/futex.h +1 -0
@@ -22,6 +22,7 @@#define FUTEX_WAIT_REQUEUE_PI 11#define FUTEX_CMP_REQUEUE_PI 12#define FUTEX_LOCK_PI2 13 +#define FUTEX_WAIT_GE 14#define FUTEX_PRIVATE_FLAG 128#define FUTEX_CLOCK_REALTIME 256
include/uapi/linux/ringbuffer_sys.h new file +40 -0
@@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_LINUX_RINGBUFFER_SYS_H +#define _UAPI_LINUX_RINGBUFFER_SYS_H + +#include <uapi/linux/types.h> + +/* +* ringbuffer_ptrs - head and tail pointers for a ringbuffer, mappped to +* userspace: +*/ +struct ringbuffer_ptrs { +/* +* We use u32s because this type is shared between the kernel and +* userspace - ulong/size_t won't work here, we might be 32bit userland +* and 64 bit kernel, and u64 would be preferable (reduced probability +* of ABA) but not all architectures can atomically read/write to a u64; +* we need to avoid torn reads/writes. +* +* head and tail pointers are incremented and stored without masking; +* this is to avoid ABA and differentiate between a full and empty +* buffer - they must be masked with @mask to get an actual offset into +* the data buffer. +* +* All units are in bytes. +* +* Data is emitted at head, consumed from tail. +*/ +__u32 head; +__u32 tail; +__u32 size;/* always a power of two */ +__u32 mask;/* size - 1 */ + +/* +* Starting offset of data buffer, from the start of this struct - will +* always be PAGE_SIZE. +*/ +__u32 data_offset; +}; + +#endif /* _UAPI_LINUX_RINGBUFFER_SYS_H */
init/Kconfig +9 -0
@@ -1673,6 +1673,15 @@ config IO_URING applications to submit and complete IO through submission and completion rings that are shared between the kernel and application. +config RINGBUFFER + bool "Enable ringbuffer() syscall" if EXPERT + select XARRAY_MULTI + default y + help + This option adds support for generic ringbuffers, which can be + attached to any (supported) file descriptor, allowing for reading and + writing without syscall overhead. + config ADVISE_SYSCALLS bool "Enable madvise/fadvise syscalls" if EXPERT default y
kernel/fork.c +2 -0
@@ -103,6 +103,7 @@#include <linux/rseq.h> #include <uapi/linux/pidfd.h> #include <linux/pidfs.h> +#include <linux/ringbuffer_sys.h> #include <asm/pgalloc.h> #include <linux/uaccess.h>
@@ -1340,6 +1341,7 @@ static inline void __mmput(struct mm_struct *mm)VM_BUG_ON (atomic_read (&mm->mm_users));uprobe_clear_state (mm); +ringbuffer_mm_exit (mm);exit_aio (mm);ksm_exit (mm);khugepaged_exit (mm);/* must run before exit_mmap */
Signature · 1 line
2.45.1
kernel test robot
build botRe: [PATCH 3/5] fs: sys_ringbuffer
kernel test robot noticed the following build warnings:
Hi Kent,
kernel test robot noticed the following build warnings:
[auto build test WARNING on tip/locking/core]
[also build test WARNING on linus/master v6.10-rc2]
[cannot apply to akpm-mm/mm-nonmm-unstable tip/x86/asm akpm-mm/mm-everything next-20240531] [If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Kent-Overstreet/darray-lift-from-bcachefs/20240603-083536 base: tip/locking/core patch link: https://lore.kernel.org/r/20240603003306.2030491-4-kent.overstreet%40linux.dev patch subject: [PATCH 3/5] fs: sys_ringbuffer config: arm-allnoconfig (https://download.01.org/0day-ci/archive/20240603/[email protected]/config) compiler: clang version 19.0.0git (https://github.com/llvm/llvm-project bafda89a0944d947fc4b3b5663185e07a397ac30) reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240603/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All warnings (new ones prefixed by >>):
>> <stdin>:1603:2: warning: syscall ringbuffer not implemented [-W#warnings]
1603 | #warning syscall ringbuffer not implemented
| ^
>> <stdin>:1606:2: warning: syscall ringbuffer_wait not implemented [-W#warnings]
1606 | #warning syscall ringbuffer_wait not implemented
| ^
>> <stdin>:1609:2: warning: syscall ringbuffer_wakeup not implemented [-W#warnings] 1609 | #warning syscall ringbuffer_wakeup not implemented
| ^
3 warnings generated.
--
In file included from arch/arm/kernel/asm-offsets.c:12:
In file included from include/linux/mm.h:2253:
include/linux/vmstat.h:514:36: warning: arithmetic between different enumeration types ('enum node_stat_item' and 'enum lru_list') [-Wenum-enum-conversion]
514 | return node_stat_name(NR_LRU_BASE + lru) + 3; // skip "nr_"
| ~~~~~~~~~~~ ^ ~~~
1 warning generated.
>> <stdin>:1603:2: warning: syscall ringbuffer not implemented [-W#warnings]
1603 | #warning syscall ringbuffer not implemented
| ^
>> <stdin>:1606:2: warning: syscall ringbuffer_wait not implemented [-W#warnings]
1606 | #warning syscall ringbuffer_wait not implemented
| ^
>> <stdin>:1609:2: warning: syscall ringbuffer_wakeup not implemented [-W#warnings] 1609 | #warning syscall ringbuffer_wakeup not implemented
| ^
3 warnings generated.
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
kernel test robot
build botRe: [PATCH 3/5] fs: sys_ringbuffer
In reply to Kent Overstreetkernel test robot noticed the following build errors:
Hi Kent,
kernel test robot noticed the following build errors:
[auto build test ERROR on tip/locking/core]
[also build test ERROR on linus/master v6.10-rc2]
[cannot apply to akpm-mm/mm-nonmm-unstable tip/x86/asm akpm-mm/mm-everything next-20240531] [If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Kent-Overstreet/darray-lift-from-bcachefs/20240603-083536 base: tip/locking/core patch link: https://lore.kernel.org/r/20240603003306.2030491-4-kent.overstreet%40linux.dev patch subject: [PATCH 3/5] fs: sys_ringbuffer config: i386-buildonly-randconfig-002-20240603 (https://download.01.org/0day-ci/archive/20240603/[email protected]/config) compiler: clang version 18.1.5 (https://github.com/llvm/llvm-project 617a15a9eac96088ae5e9134248d8236e34b91b1) reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240603/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All errors (new ones prefixed by >>):
In file included from <built-in>:1:
>> ./usr/include/linux/ringbuffer_sys.h:5:10: fatal error: 'uapi/linux/types.h' file not found
5 | #include <uapi/linux/types.h>
| ^~~~~~~~~~~~~~~~~~~~
1 error generated.
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
Re: [PATCH 3/5] fs: sys_ringbuffer
In reply to Kent OverstreetKent!
include/uapi/linux/ringbuffer_sys.h line 11
Quoted 5 lines · Kent Overstreet
include/uapi/linux/ringbuffer_sys.h new file +5
-0+/* +* ringbuffer_ptrs - head and tail pointers for a ringbuffer, mappped to +* userspace: +*/ +struct ringbuffer_ptrs {
The naming is confusing. ringbuffer_ctrl or something like that would be more clear because it's more than just the pointers, which are in fact positions. You have size, mask ... too, no?
include/uapi/linux/ringbuffer_sys.h line 17
Quoted 6 lines
include/uapi/linux/ringbuffer_sys.h new file +6
-0+/* +* We use u32s because this type is shared between the kernel and +* userspace - ulong/size_t won't work here, we might be 32bit userland +* and 64 bit kernel, and u64 would be preferable (reduced probability +* of ABA) but not all architectures can atomically read/write to a u64; +* we need to avoid torn reads/writes.
union rbmagic {
u64 __val64;
struct {
// TOOTIRED: Add big/little endian voodoo
u32 __val32;
u32 __unused;
};
};Plus a bunch of accessors which depend on BITS_PER_LONG, no?
Thanks,
tglx
Re: [PATCH 3/5] fs: sys_ringbuffer
Quoted 12 lines · Thomas Gleixner
Kent!
include/uapi/linux/ringbuffer_sys.h line 11
Quoted 5 lines · Kent Overstreet
include/uapi/linux/ringbuffer_sys.h new file +5
-0+/* +* ringbuffer_ptrs - head and tail pointers for a ringbuffer, mappped to +* userspace: +*/ +struct ringbuffer_ptrs {The naming is confusing.
ringbuffer_ctrlor something like that would be more clear because it's more than just the pointers, which are in fact positions. You have size, mask ... too, no?
I like that name, yeah.
Quoted 16 lines
include/uapi/linux/ringbuffer_sys.h line 17
Quoted 6 lines
include/uapi/linux/ringbuffer_sys.h new file +6
-0+/* +* We use u32s because this type is shared between the kernel and +* userspace - ulong/size_t won't work here, we might be 32bit userland +* and 64 bit kernel, and u64 would be preferable (reduced probability +* of ABA) but not all architectures can atomically read/write to a u64; +* we need to avoid torn reads/writes. union rbmagic { u64 __val64; struct { // TOOTIRED: Add big/little endian voodoo u32 __val32; u32 __unused; }; };
Quoted 1 line
Plus a bunch of accessors which depend on
BITS_PER_LONG, no?
Not sure I follow?
I know biendian machines exist, but I've never heard of both big and little endian being used at the same time. Nor why we'd care about BITS_PER_LONG? This just uses fixed size integer types.
Re: [PATCH 3/5] fs: sys_ringbuffer
Kent!
Quoted 17 lines · Kent Overstreet
Quoted 16 lines · Thomas Gleixner
include/uapi/linux/ringbuffer_sys.h line 17
Quoted 6 lines
include/uapi/linux/ringbuffer_sys.h new file +6
-0+/* +* We use u32s because this type is shared between the kernel and +* userspace - ulong/size_t won't work here, we might be 32bit userland +* and 64 bit kernel, and u64 would be preferable (reduced probability +* of ABA) but not all architectures can atomically read/write to a u64; +* we need to avoid torn reads/writes. union rbmagic { u64 __val64; struct { // TOOTIRED: Add big/little endian voodoo u32 __val32; u32 __unused; }; };
Quoted 7 lines
Quoted 1 line
Plus a bunch of accessors which depend on
BITS_PER_LONG, no?Not sure I follow?
I know biendian machines exist, but I've never heard of both big and little endian being used at the same time. Nor why we'd care about
BITS_PER_LONG? This just uses fixed size integer types.
Read your comment above. Ideally you want to use u64, right?
The problem is that you can't do this unconditionally because of 32-bit systems which do not support 64-bit atomics.
So a binary which is compiled for 32-bit might unconditionally want the 32-bit accessors. Ditto for 32-bit kernels.
The 64bit kernel where it runs on wants to utilize u64, right?
That's fortunately a unidirectional problem as 64-bit user space cannot run on a 32-bit kernel ever.
struct ringbuffer_ctrl {
union rbmagic head;...
};
#ifdef __BITS_PER_LONG == 64
static __always_inline u64 read_head(struct ringbuffer_ctrl *rb)
{
return rb->head.__val64;
}static __always_inline void write_head(struct ringbuffer_ctrl *rb, u64 val)
{
rb->head.__val64 = val;
}
#else
static __always_inline u64 read_head(struct ringbuffer_ctrl *rb)
{
return rb->head.__val32;
}static __always_inline void write_head(struct ringbuffer_ctrl *rb, u64 val)
{
rb->head.__val32 = (u32)val;
}#endif
A 64-bit kernel uses u64 while a 32-bit kernel uses u32. Same for user space.
The ABA concern for 32-bit does not go away, but for 64-bit userspace you get what you want, no?
Now why do you have to care about endianess?
union rbmagic {
u64 __val64;
struct {
u32 __val32;
u32 __unused;
};
};works only correctly for LE. But it does not work for BE because BE obviously requires the u32 members to be in reverse order:
union rbmagic {
u64 __val64;
struct {
u32 __unused;
u32 __val32;
};
};That's a compile time decision. You can't run a BE binary on a LE kernel or the other way around.
So they have to agree on the endianess, but BE has the reverse byte order. That's why you need to have another #ifdef there.
Thanks,
tglx
Re: [PATCH 3/5] fs: sys_ringbuffer
Quoted 33 lines · Thomas Gleixner
Kent!
Quoted 17 lines · Kent Overstreet
Quoted 16 lines · Thomas Gleixner
include/uapi/linux/ringbuffer_sys.h line 17
Quoted 6 lines
include/uapi/linux/ringbuffer_sys.h new file +6
-0+/* +* We use u32s because this type is shared between the kernel and +* userspace - ulong/size_t won't work here, we might be 32bit userland +* and 64 bit kernel, and u64 would be preferable (reduced probability +* of ABA) but not all architectures can atomically read/write to a u64; +* we need to avoid torn reads/writes. union rbmagic { u64 __val64; struct { // TOOTIRED: Add big/little endian voodoo u32 __val32; u32 __unused; }; };Quoted 7 lines
Quoted 1 line
Plus a bunch of accessors which depend on
BITS_PER_LONG, no?Not sure I follow?
I know biendian machines exist, but I've never heard of both big and little endian being used at the same time. Nor why we'd care about
BITS_PER_LONG? This just uses fixed size integer types.Read your comment above. Ideally you want to use u64, right?
The problem is that you can't do this unconditionally because of 32-bit systems which do not support 64-bit atomics.
Quoted 7 lines
So a binary which is compiled for 32-bit might unconditionally want the 32-bit accessors. Ditto for 32-bit kernels.
The 64bit kernel where it runs on wants to utilize u64, right?
That's fortunately a unidirectional problem as 64-bit user space cannot run on a 32-bit kernel ever.
Ah! Yeah, that's slick.
Your code doesn't quite work though; with this scheme, we can't just subtract the 64 bit head and tail to get the curretly used space in the ringbuffer, that'll give the wrong answer if the other end is 32 bit.
But we just need to mask off the high 32 bits (after the subtract, even), and we can still use the full 64 bits for ABA avoidance.
I think I like it...
PATCH 4/5 ringbuffer: Test device
In reply to Kent OverstreetThis adds /dev/ringbuffer-test, which supports reading and writing a sequence of integers, to test performance and correctness.
--- fs/Makefile | 1 + fs/ringbuffer_test.c | 209 +++++++++++++++++++++++++++++++++++++++++++ lib/Kconfig.debug | 5 ++ 3 files changed, 215 insertions(+) create mode 100644 fs/ringbuffer_test.c
fs/Makefile +1 -0
@@ -29,6 +29,7 @@ obj-$(CONFIG_EVENTFD) += eventfd.o obj-$(CONFIG_USERFAULTFD) += userfaultfd.o obj-$(CONFIG_AIO) += aio.o obj-$(CONFIG_RINGBUFFER) += ringbuffer.o +obj-$(CONFIG_RINGBUFFER_TEST) += ringbuffer_test.o obj-$(CONFIG_FS_DAX) += dax.o obj-$(CONFIG_FS_ENCRYPTION) += crypto/ obj-$(CONFIG_FS_VERITY) += verity/
fs/ringbuffer_test.c new file +209 -0
209 lines not on this page · raw message
lib/Kconfig.debug +5 -0
@@ -2957,6 +2957,11 @@ config TEST_OBJPOOL If unsure, say N. +config RINGBUFFER_TEST + bool "Test driver for sys_ringbuffer" + default n + depends on RINGBUFFER + endif # RUNTIME_TESTING_MENU config ARCH_USE_MEMTEST
Signature · 1 line
2.45.1
PATCH 5/5 ringbuffer: Userspace test helper
In reply to Kent OverstreetThis adds a helper for testing the new ringbuffer syscall using /dev/ringbuffer-test; it can do performance testing of both normal reads and writes, and reads and writes via the ringbuffer interface.
--- tools/ringbuffer/Makefile | 3 + tools/ringbuffer/ringbuffer-test.c | 254 +++++++++++++++++++++++++++++ 2 files changed, 257 insertions(+) create mode 100644 tools/ringbuffer/Makefile create mode 100644 tools/ringbuffer/ringbuffer-test.c
tools/ringbuffer/Makefile new file +3 -0
@@ -0,0 +1,3 @@ +CFLAGS=-g -O2 -Wall -Werror -I../../include + +all: ringbuffer-test
tools/ringbuffer/ringbuffer-test.c new file +254 -0
254 lines not on this page · raw message
Signature · 1 line
2.45.1
Quoted 7 lines · Kent Overstreet
New syscall for mapping generic ringbuffers for arbitary (supported) file descriptors.
Ringbuffers can be created either when requested or at file open time, and can be mapped into multiple address spaces (naturally, since files can be shared as well).
Quoted 5 lines
Initial motivation is for fuse, but I plan on adding support to pipes and possibly sockets as well - pipes are a particularly interesting use case, because if both the sender and receiver of a pipe opt in to the new ringbuffer interface, we can make them the _same_ ringbuffer for true zero copy IO, while being backwards compatible with existing pipes.
Hi Kent,
I recently came across a similar use case where the ability to "upgrade" an fd into a more efficient interface would be useful like in this pipe scenario you are describing.
My use case is when you have a block device using the ublk driver. ublk lets userspace servers implement block devices. ublk is great when compatibility is required with applications that expect block device fds, but when an application is willing to implement a shared memory interface to communicate directly with the ublk server then going through a block device is inefficient.
In my case the application is QEMU, where the virtual machine runs a virtio-blk driver that could talk directly to the ublk server via vhost-user-blk. vhost-user-blk is a protocol that allows the virtual machine to talk directly to the ublk server via shared memory without going through QEMU or the host kernel block layer.
QEMU would need a way to upgrade from a ublk block device file to a vhost-user socket. Just like in your pipe example, this approach relies on being able to go from a "compatibility" fd to a more efficient interface gracefully when both sides support this feature.
The generic ringbuffer approach in this series would not work for the vhost-user protocol because the client must be able to provide its own memory and file descriptor passing is needed in general. The protocol spec is here:
https://gitlab.com/qemu-project/qemu/-/blob/master/docs/interop/vhost-user.rst
A different way to approach the fd upgrading problem is to treat this as an AF_UNIX connectivity feature rather than a new ring buffer API. Imagine adding a new address type to AF_UNIX for looking up connections in a struct file (e.g. the pipe fd) instead of on the file system (or the other AF_UNIX address types).
The first program creates the pipe and also an AF_UNIX socket. It calls bind(2) on the socket with the sockaddr_un path "/dev/self/fd/<fd>/<discriminator>" where fd is a pipe fd and discriminator is a string like "ring-buffer" that describes the service/protocol. The AF_UNIX kernel code parses this special path and stores an association with the pipe file for future connect(2) calls. The program listens on the AF_UNIX socket and then continues doing its stuff.
The second program runs and inherits the pipe fd on stdin. It creates an AF_UNIX socket and attempts to connect(2) to "/dev/self/fd/0/ring-buffer". The AF_UNIX kernel code parses this special path and establishes a connection between the connecting and listening sockets inside the pipe fd's struct file. If connect(2) fails then the second program knows that this is an ordinary pipe that does not support upgrading to ring buffer operation.
Now the AF_UNIX socket can be used to pass shared memory for the ring buffer and futexes. This AF_UNIX approach also works for my ublk block device to vhost-user-blk upgrade use case. It does not require a new ring buffer API but instead involves extending AF_UNIX.
You have more use cases than just the pipe scenario, maybe my half-baked idea won't cover all of them, but I wanted to see what you think.
Stefan
Quoted 62 lines
the
ringbuffer_waitandringbuffer_wakeupsyscalls are probably going away in a future iteration, in favor of just using futexes.In my testing, reading/writing from the ringbuffer 16 bytes at a time is ~7x faster than using read/write syscalls - and I was testing with mitigations off, real world benefit will be even higher.
Kent Overstreet (5):
darray: lift from bcachefs
darray: Fixdarray_for_each_reverse()when darray is empty
fs:sys_ringbuffer
ringbuffer: Test device
ringbuffer: Userspace test helperMAINTAINERS | 7 + arch/x86/entry/syscalls/syscall_32.tbl | 3 + arch/x86/entry/syscalls/syscall_64.tbl | 3 + fs/Makefile | 2 + fs/bcachefs/Makefile | 1 - fs/bcachefs/btree_types.h | 2 +- fs/bcachefs/btree_update.c | 2 + fs/bcachefs/btree_write_buffer_types.h | 2 +- fs/bcachefs/fsck.c | 2 +- fs/bcachefs/journal_io.h | 2 +- fs/bcachefs/journal_sb.c | 2 +- fs/bcachefs/sb-downgrade.c | 3 +- fs/bcachefs/sb-errors_types.h | 2 +- fs/bcachefs/sb-members.h | 3 +- fs/bcachefs/subvolume.h | 1 - fs/bcachefs/subvolume_types.h | 2 +- fs/bcachefs/thread_with_file_types.h | 2 +- fs/bcachefs/util.h | 28 +- fs/ringbuffer.c | 474 ++++++++++++++++++++++++ fs/ringbuffer_test.c | 209 +++++++++++ {fs/bcachefs => include/linux}/darray.h | 61 +-- include/linux/darray_types.h | 22 ++ include/linux/fs.h | 2 + include/linux/mm_types.h | 4 + include/linux/ringbuffer_sys.h | 18 + include/uapi/linux/futex.h | 1 + include/uapi/linux/ringbuffer_sys.h | 40 ++ init/Kconfig | 9 + kernel/fork.c | 2 + lib/Kconfig.debug | 5 + lib/Makefile | 2 +- {fs/bcachefs => lib}/darray.c | 12 +- tools/ringbuffer/Makefile | 3 + tools/ringbuffer/ringbuffer-test.c | 254 +++++++++++++ 34 files changed, 1125 insertions(+), 62 deletions(-) create mode 100644 fs/ringbuffer.c create mode 100644 fs/ringbuffer_test.c rename {fs/bcachefs => include/linux}/darray.h (63%) create mode 100644 include/linux/darray_types.h create mode 100644 include/linux/ringbuffer_sys.h create mode 100644 include/uapi/linux/ringbuffer_sys.h rename {fs/bcachefs => lib}/darray.c (56%) create mode 100644 tools/ringbuffer/Makefile create mode 100644 tools/ringbuffer/ringbuffer-test.cSignature · 1 line
2.45.1