Add udmabuf misc device
I am by no means a kernel expert, but Tomeu asked whether I could review this. Generally the patch looks okay, it applied (on top of 4.18.5-gentoo), and compiled without problems.
However, running the experimental qemu branch I get a kernel bug:
BUG: unable to handle kernel NULL pointer dereference at 0000000000000038 [12100.605908] PGD 0 P4D 0 [12100.605918] Oops: 0002 [#1] SMP NOPTI [12100.605926] CPU: 5 PID: 7247 Comm: qemu-system-x86 Not tainted 4.18.5-gentoo #1 [12100.605930] Hardware name: To be filled by O.E.M. To be filled by O.E.M./M5A97 R2.0, BIOS 1302 11/14/2012 [12100.605942] RIP: 0010:fput+0x5/0x80 [12100.605945] Code: e4 ff 48 8b 3d b4 23 10 01 48 8b 34 24 48 83 c4 08 e9 6f 6c fd ff 0f 1f 44 00 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 <f0> 48 ff 4f 38 75 51 53 48 89 fb 65 8b 05 e9 fe 9c 5f 65 48 8b 3c [12100.606003] RSP: 0018:ffffbf00c94ffd90 EFLAGS: 00010246 [12100.606007] RAX: 0000000000000000 RBX: fffffffffffffff8 RCX: ffff98336e000b00 [12100.606011] RDX: 0000000000000000 RSI: 0000000000004000 RDI: 0000000000000000 [12100.606014] RBP: 0000000000000000 R08: ffff983366504bc0 R09: 00000000ffffffff [12100.606018] R10: 0000000000000000 R11: 0000000000000000 R12: ffffffffffffffea [12100.606021] R13: 0000000000000000 R14: ffff983100c5ca20 R15: 0000000000000000 [12100.606026] FS: 00007fe49ea8fbc0(0000) GS:ffff98338ed40000(0000) knlGS:0000000000000000 [12100.606029] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [12100.606032] CR2: 0000000000000038 CR3: 00000003df65e000 CR4: 00000000000406e0 [12100.606036] Call Trace: [12100.606046] udmabuf_create+0x220/0x310 [12100.606057] udmabuf_ioctl+0xf3/0x120 [12100.606064] do_vfs_ioctl+0xb0/0x630 [12100.606071] ? handle_mm_fault+0x108/0x200 [12100.606077] ksys_ioctl+0x92/0xa0 [12100.606082] __x64_sys_ioctl+0x16/0x20 [12100.606089] do_syscall_64+0x4f/0xe0 [12100.606096] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [12100.606103] RIP: 0033:0x7fe4974b43b7
The offending code is in the error exit (0x220 = 544):
0xffffffff816085db <+539>: callq 0xffffffff81240350 <fput> 0xffffffff816085e0 <+544>: mov 0x8(%r14),%rdi 0xffffffff816085e4 <+548>: callq 0xffffffff81217ad0 <kfree> 0xffffffff816085e9 <+553>: mov %r14,%rdi 0xffffffff816085ec <+556>: callq 0xffffffff81217ad0 <kfree> 0xffffffff816085f1 <+561>: mov 0x50(%rsp),%rbx 0xffffffff816085f6 <+566>: xor %gs:0x28,%rbx 0xffffffff816085ff <+575>: mov %r12,%rax 0xffffffff81608602 <+578>: jne 0xffffffff816086c8
<udmabuf_create+776>
err_free_ubuf:
fput(memfd); kfree(ubuf->pages); kfree(ubuf); return ret;
further commenst see below.
+
Quoted 48 lines
(quoted diff) +24
-0+static long udmabuf_create(struct udmabuf_create_list *head, + struct udmabuf_create_item *list) +{ + DEFINE_DMA_BUF_EXPORT_INFO(exp_info); + struct file *memfd = NULL; + struct udmabuf *ubuf; + struct dma_buf *buf; + pgoff_t pgoff, pgcnt, pgidx, pgbuf; + struct page *page; + int seals, ret = -EINVAL; + u32 i, flags; + + ubuf = kzalloc(sizeof(struct udmabuf), GFP_KERNEL); + if (!ubuf) + return -ENOMEM; + + for (i = 0; i < head->count; i++) { + if (!IS_ALIGNED(list[i].offset, PAGE_SIZE)) + goto err_free_ubuf; + if (!IS_ALIGNED(list[i].size, PAGE_SIZE)) + goto err_free_ubuf; + ubuf->pagecount += list[i].size >> PAGE_SHIFT; + } + ubuf->pages = kmalloc_array(ubuf->pagecount, sizeof(structpage *),
(quoted diff) +23
-0+ GFP_KERNEL); + if (!ubuf->pages) { + ret = -ENOMEM; + goto err_free_ubuf; + } + + pgbuf = 0; + for (i = 0; i < head->count; i++) { + memfd = fget(list[i].memfd); + if (!memfd) + goto err_put_pages; + if (!shmem_mapping(file_inode(memfd)->i_mapping)) + goto err_put_pages; + seals = memfd_fcntl(memfd, F_GET_SEALS, 0); + if (seals == -EINVAL || + (seals & SEALS_WANTED) != SEALS_WANTED || + (seals & SEALS_DENIED) != 0) + goto err_put_pages; + pgoff = list[i].offset >> PAGE_SHIFT; + pgcnt = list[i].size >> PAGE_SHIFT; + for (pgidx = 0; pgidx < pgcnt; pgidx++) { + page = shmem_read_mapping_page( + file_inode(memfd)->i_mapping, pgoff
Review
Quoted 10 lines
(quoted diff) +10
-0+ pgidx); + if (IS_ERR(page)) { + ret = PTR_ERR(page); + goto err_put_pages; + } + ubuf->pages[pgbuf++] = page; + } + fput(memfd); + } + memfd = NULL;
Now memfd is NULL
Review
Quoted 9 lines
(quoted diff) +9
-0+ + exp_info.ops = &udmabuf_ops; + exp_info.size = ubuf->pagecount << PAGE_SHIFT; + exp_info.priv = ubuf; + + buf = dma_buf_export(&exp_info); + if (IS_ERR(buf)) { + ret = PTR_ERR(buf); + goto err_put_pages;
Assume an error occured
Review
Quoted 12 lines
(quoted diff) +12
-0+ } + + flags = 0; + if (head->flags & UDMABUF_FLAGS_CLOEXEC) + flags |= O_CLOEXEC; + return dma_buf_fd(buf, flags); + +err_put_pages: + while (pgbuf > 0) + put_page(ubuf->pages[--pgbuf]); +err_free_ubuf: + fput(memfd);
Now fput it called with NULL, and in fput this is used in
0xffffffff81240350 <+0>: callq 0xffffffff81a015c0 <__fentry__>0xffffffff81240355 <+5>: lock decq 0x38(%rdi)
where the bug is signalled, so I guess fput doesn't accept a null pointer.
I'm not sure why you set memfd to NULL at all, because on the following non-error path it is not used.
The other question is of course, why did dma_buf_export fail for me ...
Best,
Gert
PS: Please CC me since I'm not on lkml,
Quoted 274 lines
(quoted diff) +6
-0+ kfree(ubuf->pages); + kfree(ubuf); + return ret; +} + +static long udmabuf_ioctl_create(struct file *filp, unsigned longarg)
(quoted diff) +19
-0+{ + struct udmabuf_create create; + struct udmabuf_create_list head; + struct udmabuf_create_item list; + + if (copy_from_user(&create, (void __user *)arg, + sizeof(struct udmabuf_create))) + return -EFAULT; + + head.flags = create.flags; + head.count = 1; + list.memfd = create.memfd; + list.offset = create.offset; + list.size = create.size; + + return udmabuf_create(&head, &list); +} + +static long udmabuf_ioctl_create_list(struct file *filp, unsignedlong arg)
(quoted diff) +12
-0+{ + struct udmabuf_create_list head; + struct udmabuf_create_item *list; + int ret = -EINVAL; + u32 lsize; + + if (copy_from_user(&head, (void __user *)arg, sizeof(head))) + return -EFAULT; + if (head.count > 1024) + return -EINVAL; + lsize = sizeof(struct udmabuf_create_item) * head.count; + list = memdup_user((void __user *)(arg + sizeof(head)),lsize);
(quoted diff) +53
-0+ if (IS_ERR(list)) + return PTR_ERR(list); + + ret = udmabuf_create(&head, list); + kfree(list); + return ret; +} + +static long udmabuf_ioctl(struct file *filp, unsigned int ioctl, + unsigned long arg) +{ + long ret; + + switch (ioctl) { + case UDMABUF_CREATE: + ret = udmabuf_ioctl_create(filp, arg); + break; + case UDMABUF_CREATE_LIST: + ret = udmabuf_ioctl_create_list(filp, arg); + break; + default: + ret = -EINVAL; + break; + } + return ret; +} + +static const struct file_operations udmabuf_fops = { + .owner = THIS_MODULE, + .unlocked_ioctl = udmabuf_ioctl, +}; + +static struct miscdevice udmabuf_misc = { + .minor = MISC_DYNAMIC_MINOR, + .name = "udmabuf", + .fops = &udmabuf_fops, +}; + +static int __init udmabuf_dev_init(void) +{ + return misc_register(&udmabuf_misc); +} + +static void __exit udmabuf_dev_exit(void) +{ + misc_deregister(&udmabuf_misc); +} + +module_init(udmabuf_dev_init) +module_exit(udmabuf_dev_exit) + +MODULE_AUTHOR("Gerd Hoffmann <[email protected]>"); +MODULE_LICENSE("GPL v2");tools/testing/selftests/drivers/dma-buf/udmabuf.c new file +96
-0@@ -0,0 +1,96 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +#include <errno.h> +#include <fcntl.h> +#include <malloc.h> + +#include <sys/ioctl.h> +#include <sys/syscall.h> +#include <linux/memfd.h> +#include <linux/udmabuf.h> + +#define TEST_PREFIX "drivers/dma-buf/udmabuf" +#define NUM_PAGES 4 + +static int memfd_create (const char *name,unsigned int flags) +{ +return syscall (__NR_memfd_create, name, flags); +} + +int main (int argc,char *argv[]) +{ +struct udmabuf_create create; +int devfd, memfd, buf, ret; +off_t size; +void *mem; + + devfd =open ("/dev/udmabuf" ,O_RDWR ); +if (devfd <0 ) { +printf ("%s: [skip,no-udmabuf]\n" ,TEST_PREFIX ); +exit (77 ); + } + + memfd =memfd_create ("udmabuf-test" ,MFD_CLOEXEC ); +if (memfd <0 ) { +printf ("%s: [skip,no-memfd]\n" ,TEST_PREFIX ); +exit (77 ); + } + + size =getpagesize () *NUM_PAGES ; + ret =ftruncate (memfd, size); +if (ret ==-1 ) { +printf ("%s: [FAIL,memfd-truncate]\n" ,TEST_PREFIX ); +exit (1 ); + } + +memset (&create,0 ,sizeof (create)); + +/* should fail (offset not page aligned) */ + create.memfd = memfd; + create.offset =getpagesize ()/2 ; + create.size =getpagesize (); + buf =ioctl (devfd,UDMABUF_CREATE , &create); +if (buf >=0 ) { +printf ("%s: [FAIL,test-1]\n" ,TEST_PREFIX ); +exit (1 ); + } + +/* should fail (size not multiple of page) */ + create.memfd = memfd; + create.offset =0 ; + create.size =getpagesize ()/2 ; + buf =ioctl (devfd,UDMABUF_CREATE , &create); +if (buf >=0 ) { +printf ("%s: [FAIL,test-2]\n" ,TEST_PREFIX ); +exit (1 ); + } + +/* should fail (not memfd) */ + create.memfd =0 ;/* stdin */ + create.offset =0 ; + create.size = size; + buf =ioctl (devfd,UDMABUF_CREATE , &create); +if (buf >=0 ) { +printf ("%s: [FAIL,test-3]\n" ,TEST_PREFIX ); +exit (1 ); + } + +/* should work */ + create.memfd = memfd; + create.offset =0 ; + create.size = size; + buf =ioctl (devfd,UDMABUF_CREATE , &create); +if (buf <0 ) { +printf ("%s: [FAIL,test-4]\n" ,TEST_PREFIX ); +exit (1 ); + } + +fprintf (stderr,"%s: ok\n" ,TEST_PREFIX ); +close (buf); +close (memfd); +close (devfd); +return 0 ; +}MAINTAINERS +16
-0@@ -14934,6 +14934,14 @@ S: Maintained F: Documentation/filesystems/udf.txt F: fs/udf/ +UDMABUF DRIVER +M: Gerd Hoffmann <[email protected]> +L: [email protected] +S: Maintained +F: drivers/dma-buf/udmabuf.c +F: include/uapi/linux/udmabuf.h +F: tools/testing/selftests/drivers/dma-buf/udmabuf.c + UDRAW TABLET M: Bastien Nocera <[email protected]> L: [email protected]@@ -15343,6 +15351,14 @@ F: arch/x86/um/ F: fs/hostfs/ F: fs/hppfs/ +USERSPACE DMA BUFFER DRIVER +M: Gerd Hoffmann <[email protected]> +S: Maintained +L: [email protected] +F: drivers/dma-buf/udmabuf.c +F: include/uapi/linux/udmabuf.h +T: git git://anongit.freedesktop.org/drm/drm-misc + USERSPACE I/O (UIO) M: Greg Kroah-Hartman <[email protected]> S: Maintaineddrivers/dma-buf/Kconfig +0
-0@@ -30,4 +30,12 @@ config SW_SYNC WARNING: improper use of this can result in deadlockingkernel
drivers/dma-buf/Kconfig +6
-0drivers from userspace. Intended for test and debug only. +config UDMABUF + bool "userspace dmabuf misc driver" + default n + depends on DMA_SHARED_BUFFER + help + A driver to let userspace turn memfd regions into dma-bufs.
drivers/dma-buf/Kconfig +1
-0+ Qemu can use this to create host dmabufs for guestframebuffers.
drivers/dma-buf/Kconfig +1
-0+ endmenudrivers/dma-buf/Makefile +0
-0@@ -1,3 +1,4 @@ obj-y := dma-buf.o dma-fence.o dma-fence-array.o reservation.oseqno-fence.o
obj-$(CONFIG_SYNC_FILE) += sync_file.o
obj-$(CONFIG_SW_SYNC) += sw_sync.o sync_debug.odrivers/dma-buf/Makefile +1
-0+obj-$(CONFIG_UDMABUF) += udmabuf.otools/testing/selftests/drivers/dma-buf/Makefile new file +5
-0@@ -0,0 +1,5 @@ +CFLAGS += -I../../../../../usr/include/ + +TEST_GEN_PROGS := udmabuf + +include ../../lib.mkSignature · 1 line
2.9.3
Hi,
Quoted 18 lines
Review
Quoted 3 lines
(quoted diff) +3
-0+ fput(memfd); + } + memfd = NULL;Now memfd is NULL
Review
Quoted 4 lines
(quoted diff) +4
-0+ buf = dma_buf_export(&exp_info); + if (IS_ERR(buf)) { + ret = PTR_ERR(buf); + goto err_put_pages;Assume an error occured
Review
Quoted 5 lines
(quoted diff) +5
-0+err_put_pages: + while (pgbuf > 0) + put_page(ubuf->pages[--pgbuf]); +err_free_ubuf: + fput(memfd);Now fput it called with NULL, and in fput this is used in
Quoted 2 lines
where the bug is signalled, so I guess fput doesn't accept a null pointer.
Indeed. Already fixed in drm-misc-next.
Quoted 2 lines
I'm not sure why you set memfd to NULL at all, because on the following non-error path it is not used.
And in the error path it should not be user either, at least when the error happens *after* exiting the loop, otherwise we would fput twice in the error case.
Quoted 1 line
The other question is of course, why did
dma_buf_exportfail for me ...
What exactly did you try?
cheers,
Gerd
Am Montag, den 10.09.2018, 10:37 +0200 schrieb Gerd Hoffmann:
...
Quoted 4 lines
Quoted 2 lines
The other question is of course, why did
dma_buf_exportfail for me
...What exactly did you try?
I ran
qemu-system-x86_64 -enable-kvm -smp 5 -M q35 -m 8G \
-drive format=raw,file=ubuntu.raw,if=virtio \
-net nic,model=virtio -net user,hostfwd=tcp::2224-:22 \
-vga virtioubuntu.raw = Ubuntu 18.04 (bionic) + 4.17 kernel (from Ubuntu cosmic).
The VM boots up to a certain point, the last line from the boot log is varying, as far as I can see it is a few lines after virtio_blk detects the disks.
When I add '-display sdl' the Xsession freezes. I was still able to log in remotely and kill -9 qemu, then X11 became usable again.
In all cases I get on the host:
qemu-system-x86_64: info: udmabuf: init ok
and there it stays, certainly because of the kernel bug reported.
Best,
Gert
Quoted 6 lines · Gert Wollny
Am Montag, den 10.09.2018, 10:37 +0200 schrieb Gerd Hoffmann:
...Quoted 3 lines
Quoted 2 lines
The other question is of course, why did
dma_buf_exportfail for me
...
Quoted 8 lines
Quoted 1 line
What exactly did you try?
I ran
qemu-system-x86_64 -enable-kvm -smp 5 -M q35 -m 8G \ -drive format=raw,file=ubuntu.raw,if=virtio \ -net nic,model=virtio -net user,hostfwd=tcp::2224-:22 \ -vga virtio
By default qemu doesn't use memfd for backing storage, you have to explicitly configure qemu that way (see qemu commit log of the test branch):
qemu-system-x86_64 -m 2G
-object memory-backend-memfd,id=ram,size=2G
-numa node,memdev=ram"HTH,
Gerd
Am Montag, den 10.09.2018, 12:53 +0200 schrieb Gerd Hoffmann:
Quoted 8 lines
By default qemu doesn't use memfd for backing storage, you have to explicitly configure qemu that way (see qemu commit log of the test branch):
qemu-system-x86_64 -m 2G -object memory-backend-memfd,id=ram,size=2G -numa node,memdev=ram"
Thanks, but that doesn't seem to help. Are there any host kernel configuration parameters or features that I should set (apart from enabling udmabuf)?
Best,
Gert
Quoted 11 lines · Gert Wollny
Am Montag, den 10.09.2018, 12:53 +0200 schrieb Gerd Hoffmann:
Quoted 8 lines
By default qemu doesn't use memfd for backing storage, you have to explicitly configure qemu that way (see qemu commit log of the test branch):
qemu-system-x86_64 -m 2G -object memory-backend-memfd,id=ram,size=2G -numa node,memdev=ram"Thanks, but that doesn't seem to help.
Note the memory is specified twice, once with -m, once for the memory-backend-memfd object. The two must match of course (or, to be exact, the sum of all memory backends must match the total memory, but unless you want create a virtual numa machine there is no reason to have multiple memory-backends).
Quoted 3 lines
Are there any host kernel
configuration parameters or features that I should set (apart from enabling udmabuf)?
I'm not aware of any unusual parameters required. My config is based on rhel-7, then stripped down with localmodconfig, then enabled the bits I want in my test kernel.
Oh, wait, CONFIG_MEMFD_CREATE could be the one. I guess I should add a Kconfig dependency. Even though udmabuf will build just fine without that (because memfd is basically shmem with some features on top) it doesn't buy you much if the memfd_create syscall is not available ...
cheers,
Gerd
Am Montag, den 10.09.2018, 15:30 +0200 schrieb Gerd Hoffmann:
Quoted 13 lines · Gert Wollny
Quoted 12 lines
Am Montag, den 10.09.2018, 12:53 +0200 schrieb Gerd Hoffmann:
Quoted 10 lines
By default qemu doesn't use memfd for backing storage, you have to
explicitly configure qemu that way (see qemu commit log of the test
branch):qemu-system-x86_64 -m 2G -object memory-backend-memfd,id=ram,size=2G -numa node,memdev=ram"
Quoted 7 lines
Quoted 1 line
Thanks, but that doesn't seem to help.
Note the memory is specified twice, once with -m, once for the memory-backend-memfd object. The two must match of course (or, to be exact, the sum of all memory backends must match the total memory, but unless you want create a virtual numa machine there is no reason to have multiple memory-backends).
Qemu actually complained when it was not the same.
Quoted 7 lines
Quoted 3 lines
Are there any host kernel configuration parameters or features that I should set (apart from enabling udmabuf)?
I'm not aware of any unusual parameters required. My config is based on rhel-7, then stripped down with localmodconfig, then enabled the bits I want in my test kernel.
I use Gentoo and my kernel is rather stripped down ...
Quoted 3 lines
Oh, wait,
CONFIG_MEMFD_CREATEcould be the one. I guess I should add aKconfigdependency.
This was enabled, so not sure what else could be missing, I guess I'll have to add some logging to get an idea.
Best,
Gert
Am Montag, den 10.09.2018, 15:30 +0200 schrieb Gerd Hoffmann:
Quoted 9 lines · Gert Wollny
Quoted 8 lines
Am Montag, den 10.09.2018, 12:53 +0200 schrieb Gerd Hoffmann:
Quoted 7 lines
By default qemu doesn't use memfd for backing storage, you have to
explicitly configure qemu that way (see qemu commit log of the test
branch):
Quoted 6 lines
Quoted 5 lines
Quoted 3 lines
qemu-system-x86_64 -m 2G -object memory-backend-memfd,id=ram,size=2G -numa node,memdev=ram"Thanks, but that doesn't seem to help.
With some more debugging I found
WARNING: CPU: 2 PID: 14658 at drivers/dma-buf/dma-buf.c:410 dma_buf_export+0x85/0x270 which refers to dma_buf_export:414
if (WARN_ON(!exp_info->priv
|| !exp_info->ops
|| !exp_info->ops->map_dma_buf
|| !exp_info->ops->unmap_dma_buf
|| !exp_info->ops->release
|| !exp_info->ops->map_atomic
|| !exp_info->ops->map
|| !exp_info->ops->mmap)) {
return ERR_PTR(-EINVAL);
}Specifically, map_atomic is NULL, but as I can see from the git version, this was removed some time ago, going to test the 4.19 rc2 kernel now,
Best,
Gert
Gert Wollny
Reviewed-byUsing the kernel 4.19.0-rc2 it works now, so With the fix for not calling fput when memfd == NULL the patch is
best,
Gert
Am Montag, den 10.09.2018, 15:30 +0200 schrieb Gerd Hoffmann:
Quoted 42 lines · Gert Wollny
Quoted 8 lines
Am Montag, den 10.09.2018, 12:53 +0200 schrieb Gerd Hoffmann:
Quoted 7 lines
By default qemu doesn't use memfd for backing storage, you have to
explicitly configure qemu that way (see qemu commit log of the test
branch):Quoted 5 lines
Quoted 3 lines
qemu-system-x86_64 -m 2G -object memory-backend-memfd,id=ram,size=2G -numa node,memdev=ram"Thanks, but that doesn't seem to help.
Note the memory is specified twice, once with -m, once for the memory-backend-memfd object. The two must match of course (or, to be exact, the sum of all memory backends must match the total memory, but
unless you want create a virtual numa machine there is no reason to have
multiple memory-backends).Quoted 3 lines
Are there any host kernel
configuration parameters or features that I should set (apart from enabling udmabuf)?I'm not aware of any unusual parameters required. My config is based on
rhel-7, then stripped down with localmodconfig, then enabled the bits I
want in my test kernel.Oh, wait,
CONFIG_MEMFD_CREATEcould be the one. I guess I should add aKconfigdependency. Even though udmabuf will build just fine without that (because memfd is basically shmem with some features on top) it doesn't buy you much if thememfd_createsyscall is not available ...cheers,
Gerd