Skip to contentSkip to conversation

FUSE/CUSE: implement direct mmap support

PATCH In review Tejun Heo 4 participants 37 messages Feb 9, 2010 – Feb 12, 2010 FUSE filesystem core

lore
Share & tools
lore

Tejun Heo

author
#0 lore raw

PATCH 1/1 FUSE/CUSE: implement direct mmap support

Implement FUSE direct mmap support. The server can redirect client mmap requests to any SHMLBA aligned offset in the custom address space attached to the fuse channel. The address space is managed by the server using mmap/munmap(2). The SHMLBA alignment requirement is necessary to avoid cache aliasing issues on archs with virtually indexed caches as FUSE direct mmaps are basically shared memory between clients and the server.

The direct mmap address space is backed by pinned kernel pages which are allocated on the first fault either from a client or the server. If used carelessly, this can easily waste and drain memory. Currently, a server must have CAP_SYS_ADMIN to manage dmmap regions by mmapping and munmapping the channel fd.

---
Here's the long overdue FUSE direct mmap support. I tried several things but this simplistic custom address space implementation turns out to be the cleanest. It also shouldn't be too difficult to extend it with userland ->fault() handler if such thing ever becomes necessary. I'll post the library part soon.

Thanks.

 fs/fuse/cuse.c       |    1 +
 fs/fuse/dev.c        |  294 ++++++++++++++++++++++++++++++++++++++++++++++++++
 fs/fuse/file.c       |  209 +++++++++++++++++++++++++++++++++--
 fs/fuse/fuse_i.h     |   19 ++++
 fs/fuse/inode.c      |    1 +
 include/linux/fuse.h |   32 ++++++
 6 files changed, 545 insertions(+), 11 deletions(-)
fs/fuse/cuse.c +1 -0
index de792dc..e5cefa6 100644
@@ -181,6 +181,7 @@ static const struct file_operations cuse_frontend_fops = {
 	.unlocked_ioctl		= cuse_file_ioctl,
 	.compat_ioctl		= cuse_file_compat_ioctl,
 	.poll			= fuse_file_poll,
+	.mmap			= fuse_do_dmmap,
 };
 
 
fs/fuse/dev.c +294 -0

306 lines not on this page · raw message

fs/fuse/file.c +198 -11

227 lines not on this page · raw message

fs/fuse/fuse_i.h +19 -0
index 01cc462..32f01dd 100644
@@ -21,6 +21,7 @@
 #include <linux/rwsem.h>
 #include <linux/rbtree.h>
 #include <linux/poll.h>
+#include <linux/radix-tree.h>
 
 /** Max number of pages that can be used in a single read request */
 #define FUSE_MAX_PAGES_PER_REQ 32
@@ -270,6 +271,7 @@ struct fuse_req {
 			struct fuse_write_out out;
 		} write;
 		struct fuse_lk_in lk_in;
+		struct fuse_munmap_in munmap_in;
 	} misc;
 
 	/** page vector */
@@ -447,6 +449,12 @@ struct fuse_conn {
 	/** Is poll not implemented by fs? */
 	unsigned no_poll:1;
 
+	/** Is direct mmap not implemente by fs? */
+	unsigned no_dmmap:1;
+
+	/** Already warned unaligned direct mmap */
+	unsigned dmmap_misalign_warned:1;
+
 	/** Do multi-page cached writes */
 	unsigned big_writes:1;
 
@@ -494,6 +502,12 @@ struct fuse_conn {
 
 	/** Read/write semaphore to hold when accessing sb. */
 	struct rw_semaphore killsb;
+
+	/** dmmap unique id */
+	u64 dmmapctr;
+
+	/** Direct mmap regions */
+	struct rb_root dmmap_regions;
 };
 
 static inline struct fuse_conn *get_fuse_conn_super(struct super_block *sb)
@@ -746,4 +760,9 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg,
 unsigned fuse_file_poll(struct file *file, poll_table *wait);
 int fuse_dev_release(struct inode *inode, struct file *file);
 
+int fuse_dmmap_find_get_page(struct fuse_conn *fc, pgoff_t pgoff,
+			     struct page **pagep);
+
+int fuse_do_dmmap(struct file *file, struct vm_area_struct *vma);
+
 #endif /* _FS_FUSE_I_H */
fs/fuse/inode.c +1 -0
index 1a822ce..8557b59 100644
@@ -541,6 +541,7 @@ void fuse_conn_init(struct fuse_conn *fc)
 	fc->blocked = 1;
 	fc->attr_version = 1;
 	get_random_bytes(&fc->scramble_key, sizeof(fc->scramble_key));
+	fc->dmmap_regions = RB_ROOT;
 }
 EXPORT_SYMBOL_GPL(fuse_conn_init);
 
include/linux/fuse.h +32 -0
index 3e2925a..44bf759 100644
@@ -209,6 +209,13 @@ struct fuse_file_lock {
  */
 #define FUSE_POLL_SCHEDULE_NOTIFY (1 << 0)
 
+/**
+ * Mmap flags
+ *
+ * FUSE_MMAP_DONT_COPY: don't copy the region on fork
+ */
+#define FUSE_MMAP_DONT_COPY	(1 << 0)
+
 enum fuse_opcode {
 	FUSE_LOOKUP	   = 1,
 	FUSE_FORGET	   = 2,  /* no reply */
@@ -248,6 +255,8 @@ enum fuse_opcode {
 	FUSE_DESTROY       = 38,
 	FUSE_IOCTL         = 39,
 	FUSE_POLL          = 40,
+	FUSE_MMAP          = 41,
+	FUSE_MUNMAP        = 42,
 
 	/* CUSE specific operations */
 	CUSE_INIT          = 4096,
@@ -523,6 +532,29 @@ struct fuse_notify_poll_wakeup_out {
 	__u64	kh;
 };
 
+struct fuse_mmap_in {
+	__u64	fh;
+	__u64	mh;
+	__u64	addr;
+	__u64	len;
+	__u32	prot;
+	__u32	flags;
+	__u64	offset;
+};
+
+struct fuse_mmap_out {
+	__u64	offset;
+	__u32	flags;
+	__u32	padding;
+};
+
+struct fuse_munmap_in {
+	__u64	fh;
+	__u64	mh;
+	__u64	len;
+	__u64	offset;
+};
+
 struct fuse_in_header {
 	__u32	len;
 	__u32	opcode;
Signature · 1 line
1.6.4.2
maintainer
#1 lore raw
Quoted 8 lines · Tejun Heo

Implement FUSE direct mmap support. The server can redirect client mmap requests to any SHMLBA aligned offset in the custom address space attached to the fuse channel. The address space is managed by the server using mmap/munmap(2). The SHMLBA alignment requirement is necessary to avoid cache aliasing issues on archs with virtually indexed caches as FUSE direct mmaps are basically shared memory between clients and the server.

Quoted 5 lines

The direct mmap address space is backed by pinned kernel pages which are allocated on the first fault either from a client or the server. If used carelessly, this can easily waste and drain memory. Currently, a server must have CAP_SYS_ADMIN to manage dmmap regions by mmapping and munmapping the channel fd.

Okay, I'm a bit confused about these offsets.

Client asks to map a file at an offset. Server receives offset, may change it (but only by multiple of SHMLBA) then returns it to the kernel. The returned offset globally identifies not only the mapped region but the page within the region. Sounds neat.

But then fuse_do_mmap() goes and changes vma->vm_pgoff, which will show up in /proc/PID/maps for example, which is really not nice.

Can't this page ID rather be put in vma->vm_private_data?

Also can we take this page ID abstraction a step further, and say that the ID has nothing to do with the original offset, the only requirement is that it'd globally identify all direct mapped pages.

And the coherency requirements would be satisfied by the fuse_dev_mmap() code. Haven't looked into what that would take, but it sounds doable.

That would take the load off userspace having to search for a suitable offset using some magic architecture dependent constant.

Otherwise I like the interface, it can be extended with fault and reclaim requests and server side requests to load and save the map contents as well.

Thanks,
Miklos

#33 lore raw
In reply to Tejun Heo
Quoted 8 lines · Tejun Heo

Implement FUSE direct mmap support. The server can redirect client mmap requests to any SHMLBA aligned offset in the custom address space attached to the fuse channel. The address space is managed by the server using mmap/munmap(2). The SHMLBA alignment requirement is necessary to avoid cache aliasing issues on archs with virtually indexed caches as FUSE direct mmaps are basically shared memory between clients and the server.

Quoted 5 lines

The direct mmap address space is backed by pinned kernel pages which are allocated on the first fault either from a client or the server. If used carelessly, this can easily waste and drain memory. Currently, a server must have CAP_SYS_ADMIN to manage dmmap regions by mmapping and munmapping the channel fd.

Does that mean that for example in unionfs-fuse when a user wants to mmap a file I can just mmap the actual underlying file from the real filesystem and any read/write access would then shortcut fuse and go directly to the real file?

MfG
Goswin

maintainer
#34 lore raw
Quoted 16 lines · Goswin von Brederlow
Quoted 8 lines · Tejun Heo

Implement FUSE direct mmap support. The server can redirect client mmap requests to any SHMLBA aligned offset in the custom address space attached to the fuse channel. The address space is managed by the server using mmap/munmap(2). The SHMLBA alignment requirement is necessary to avoid cache aliasing issues on archs with virtually indexed caches as FUSE direct mmaps are basically shared memory between clients and the server.

Quoted 5 lines

The direct mmap address space is backed by pinned kernel pages which are allocated on the first fault either from a client or the server. If used carelessly, this can easily waste and drain memory. Currently, a server must have CAP_SYS_ADMIN to manage dmmap regions by mmapping and munmapping the channel fd.

Quoted 4 lines

Does that mean that for example in unionfs-fuse when a user wants to mmap a file I can just mmap the actual underlying file from the real filesystem and any read/write access would then shortcut fuse and go directly to the real file?

No. It is the /dev/fuse file descriptor that is being mapped to give access to the "direct mmaped" memory.

Thanks,
Miklos

#35 lore raw
In reply to Goswin von Brederlow

Hi.

I am just sharing my thoughts on the matter.

Maybe one can implement a separate "fuse sync" ioctl which can then be called by a user space program say "fusersync" to sync all the fuse filesystems you are allowed to.

I implemented sync to my filesystem by means of an extended attribute. I now do "setfattr -nsync /mountpoint" to get the syncing done.

This has the drawback that you can not easily put it in a script because you need to know the mount point. I know one can write smart scripts to figure out the mountpoint, but it would be nice if you can just say "fusersync" and all the fuse filesystems that you are allowed to will just sync.

One can then maybe rename the system wide sync to say "sync.system" and put a script in it's place which calls "fusersync;sync.system"

Hopefully this idea might be useful.

Regards
Paul

Quoted 35 lines · Goswin von Brederlow
Quoted 8 lines · Tejun Heo

Implement FUSE direct mmap support. The server can redirect client mmap requests to any SHMLBA aligned offset in the custom address space attached to the fuse channel. The address space is managed by the server using mmap/munmap(2). The SHMLBA alignment requirement is necessary to avoid cache aliasing issues on archs with virtually indexed caches as FUSE direct mmaps are basically shared memory between clients and the server.

Quoted 6 lines

The direct mmap address space is backed by pinned kernel pages which are allocated on the first fault either from a client or the server. If used carelessly, this can easily waste and drain memory. Currently, a server must have CAP_SYS_ADMIN to manage dmmap regions by mmapping and munmapping the channel fd.

Does that mean that for example in unionfs-fuse when a user wants to mmap a file I can just mmap the actual underlying file from the real filesystem and any read/write access would then shortcut fuse and go directly to the real file?

MfG
Goswin

------------------------------------------------------------------------------
SOLARIS 10 is the OS for Data Centers - provides features such as DTrace, Predictive Self Healing and Award Winning ZFS. Get Solaris 10 NOW
http://p.sf.net/sfu/solaris-dev2dev
_______________________________________________
fuse-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/fuse-devel

#36 lore raw

Sorry wrong thread !

Quoted 67 lines · Paul Schutte

Hi.

I am just sharing my thoughts on the matter.

Maybe one can implement a separate "fuse sync" ioctl which can then be called by a user space program say "fusersync" to sync all the fuse filesystems you are allowed to.

I implemented sync to my filesystem by means of an extended attribute. I now do "setfattr -nsync /mountpoint" to get the syncing done.

This has the drawback that you can not easily put it in a script because you need to know the mount point. I know one can write smart scripts to figure out the mountpoint, but it would be nice if you can just say "fusersync" and all the fuse filesystems that you are allowed to will just sync.

One can then maybe rename the system wide sync to say "sync.system" and put a script in it's place which calls "fusersync;sync.system"

Hopefully this idea might be useful.

Regards
Paul

Quoted 27 lines · Goswin von Brederlow
Quoted 8 lines · Tejun Heo

Implement FUSE direct mmap support. The server can redirect client mmap requests to any SHMLBA aligned offset in the custom address space attached to the fuse channel. The address space is managed by the server using mmap/munmap(2). The SHMLBA alignment requirement is necessary to avoid cache aliasing issues on archs with virtually indexed caches as FUSE direct mmaps are basically shared memory between clients and the server.

Quoted 7 lines

The direct mmap address space is backed by pinned kernel pages which are allocated on the first fault either from a client or the server. If used carelessly, this can easily waste and drain memory. Currently, a server must have CAP_SYS_ADMIN to manage dmmap regions by mmapping and munmapping the channel fd.

Does that mean that for example in unionfs-fuse when a user wants to mmap a file I can just mmap the actual underlying file from the real filesystem and any read/write access would then shortcut fuse and go directly to the real file?

MfG
Goswin

Quoted 10 lines

------------------------------------------------------------------------------
SOLARIS 10 is the OS for Data Centers - provides features such as DTrace, Predictive Self Healing and Award Winning ZFS. Get Solaris 10 NOW
http://p.sf.net/sfu/solaris-dev2dev
_______________________________________________
fuse-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/fuse-devel

------------------------------------------------------------------------------
SOLARIS 10 is the OS for Data Centers - provides features such as DTrace, Predictive Self Healing and Award Winning ZFS. Get Solaris 10 NOW
http://p.sf.net/sfu/solaris-dev2dev
_______________________________________________
fuse-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/fuse-devel

author
#2 lore raw
In reply to Miklos Szeredi

Hello, Miklos.

Quoted 8 lines · Miklos Szeredi

On Tue, 09 Feb 2010, Tejun Heo wrote:
Okay, I'm a bit confused about these offsets.

Client asks to map a file at an offset. Server receives offset, may change it (but only by multiple of SHMLBA) then returns it to the kernel. The returned offset globally identifies not only the mapped region but the page within the region. Sounds neat.

Quoted 4 lines

But then fuse_do_mmap() goes and changes vma->vm_pgoff, which will show up in /proc/PID/maps for example, which is really not nice.

Can't this page ID rather be put in vma->vm_private_data?

Yeah, sure, it can be put anywhere. I just never thought about it being visible under /proc. Will move it inside fuse_dmmap_vm.

Quoted 7 lines

Also can we take this page ID abstraction a step further, and say that the ID has nothing to do with the original offset, the only requirement is that it'd globally identify all direct mapped pages.

And the coherency requirements would be satisfied by the fuse_dev_mmap() code. Haven't looked into what that would take, but it sounds doable.

The coherency requirement is not really between the address and offset but virtual addresses different maps which may end up sharing the same physical page. On an architecture with virtually indexed but physically tagged caches, virtual addresses which map to the same page must end up hitting the same cache group; otherwise, the processor wouldn't be able to determine whether the two addresses point to the same page (as only the tags inside the same cache group are matched against the physical address).

This is achieved by enforcing address to offset alignment. IOW, all maps are forced to be SHMLBA aligned to offset so that all maps are SHMLBA aligned to each other. And without knowing which maps are gonna end up sharing which pages beforehand, enforcing SHMLBA alignment against something is pretty much the only way to achieve this and that's the reason why something as low level as processor cache orgnization is exported to userland as SHMLBA for shared memory in the first place.

FUSE dmmap has exactly the same problem as it basically is a shared memory implementation with slightly different interface. Unless we can know which maps are gonna be shared beforehand, and we can't know this either for between the client and server or between clients, the only way to make those shared mappings fall into the same cache slot is aligning them to address space offsets.

So, I don't think it's feasible to do the address matching from inside the kernel without a lot of convolution. The only way to do it would be adjusting the mapping addresses but this has at least two problems
- 1. the VM layer isn't made that way and virtual address is determined by vm generic and arch code before ->mmap() is invoked and can't be changed after that. 2. the SHMLBA alignment is good in the sense that it gives the userland something to rely on when determining mapping address for address hint or fixed mapping. I don't think it would be wise to break such assumptions which hold for *ALL* existing memory mappings.

Thanks.

Signature · 1 line
tejun
maintainer
#3 lore raw
Quoted 17 lines · Tejun Heo
Quoted 7 lines

Also can we take this page ID abstraction a step further, and say that the ID has nothing to do with the original offset, the only requirement is that it'd globally identify all direct mapped pages.

And the coherency requirements would be satisfied by the fuse_dev_mmap() code. Haven't looked into what that would take, but it sounds doable.

The coherency requirement is not really between the address and offset but virtual addresses different maps which may end up sharing the same physical page. On an architecture with virtually indexed but physically tagged caches, virtual addresses which map to the same page must end up hitting the same cache group; otherwise, the processor wouldn't be able to determine whether the two addresses point to the same page (as only the tags inside the same cache group are matched against the physical address).

Quoted 3 lines

This is achieved by enforcing address to offset alignment. IOW, all maps are forced to be SHMLBA aligned to offset so that all maps are SHMLBA aligned to each other.

Okay, lets be a little clearer. There are client side maps and server side maps. Client side maps are naturally aligned (same offset -> same page).

So that leaves server side maps needing to be aligned to client side maps. Since we use the offset into the mmap as the ID, we might as well just cheat and calculate a maching offset for the server side map and use that. I'm not worried about changing vma->vm_pgoff there, if that's the only way to get proper alignment, since those are not "proper" mmaps anyway.

Quoted 6 lines

FUSE dmmap has exactly the same problem as it basically is a shared memory implementation with slightly different interface. Unless we can know which maps are gonna be shared beforehand, and we can't know this either for between the client and server or between clients, the only way to make those shared mappings fall into the same cache slot is aligning them to address space offsets.

Currently you are aligning client/client maps by changing the requested offset. That's like saying: you wanted the file mapped from offset X, but we mapped it from offset Y because that's better aligned. It doesn't make much sense, if the filesystem is doing something nasty like that, then to hell with cache alignment.

Thanks,
Miklos

author
#32 lore raw
In reply to Tejun Heo
Quoted 2 lines · Tejun Heo

So, I don't think it's feasible to do the address matching from inside the kernel without a lot of convolution.

To clarify a bit. The alignment is still a must. What's theoretically feasible with convolution is hiding the alignment from the userland server by adjusting virtual addresses of maps, but this will visibly break alignment as seen from the clients (e.g. client may not be able to unmap part of existing mmap and mmap SHMLBA aligned disjoint part there) even if we ignore the fact that implementation would be so invasive to the vm layer that it has no possibility of getting accepted.

SHMLBA is something that a user of shm should know anyway. I don't think it's too much to ask for from FUSE servers which would implement direct mmap and it's easy to detect and enforce too. The proposed libfuse patch abort()s if the condition is not met.

Thanks.

Signature · 1 line
tejun
author
#4 lore raw
In reply to Miklos Szeredi

Hello,

Quoted 7 lines · Miklos Szeredi
Quoted 3 lines

This is achieved by enforcing address to offset alignment. IOW, all maps are forced to be SHMLBA aligned to offset so that all maps are SHMLBA aligned to each other.

Okay, lets be a little clearer. There are client side maps and server side maps. Client side maps are naturally aligned (same offset -> same page).

Same offset -> same page doesn't hold. Clients mappings sharing the same offset can end up with different physical pages. ie. Process A opening /dev/dsp and mmapping at 0 and process B doing the same thing clearly need to be served with different pages and that's one of the reasons why the server is given the ability to adjust the offset.

Quoted 6 lines

So that leaves server side maps needing to be aligned to client side maps. Since we use the offset into the mmap as the ID, we might as well just cheat and calculate a maching offset for the server side map and use that. I'm not worried about changing vma->vm_pgoff there, if that's the only way to get proper alignment, since those are not "proper" mmaps anyway.

...

Quoted 5 lines

Currently you are aligning client/client maps by changing the requested offset. That's like saying: you wanted the file mapped from offset X, but we mapped it from offset Y because that's better aligned. It doesn't make much sense, if the filesystem is doing something nasty like that, then to hell with cache alignment.

I think we're misunderstanding each other pretty good here. I really can't understand the above paragraph. The offset adjustment is to allow the server to choose which clients share which mappings. It's not to align anything. The alignment is done by the generic VM layer against the offset. The SHMLBA requirement for the FUSE server is there so that the FUSE server doesn't break the alignment while changing the offset.

Can you please elaborate how you think the thing can work without referencing the proposed implementation? Let's find out where the misundertanding is.

Thanks.

Signature · 1 line
tejun
maintainer
#5 lore raw
Quoted 5 lines · Tejun Heo
Quoted 3 lines

Okay, lets be a little clearer. There are client side maps and server side maps. Client side maps are naturally aligned (same offset -> same page).

Same offset -> same page doesn't hold.

Right, I really meant same page -> same offset. If the same offset is mapped to multiple pages: no problem. If the same page is mapped to multiple offsets, then obviously it's not going to work properly.

Quoted 3 lines

Can you please elaborate how you think the thing can work without referencing the proposed implementation? Let's find out where the misundertanding is.

Thinking about it I'm not really sure...

Maybe the problem is that the propsed solution allows too much freedom. Normally there's a 1:1 relationship between pages and offsets. But we want to break that for CUSE, because two different mappings of a char dev might point to completely different pages, right?

When does that happen? Can it happen that two mappings of the same file descriptor will have different backing pages?

Thanks,
Miklos

author
#6 lore raw

Hello, Miklos.

Quoted 5 lines · Miklos Szeredi
Quoted 1 line

Same offset -> same page doesn't hold.

Right, I really meant same page -> same offset. If the same offset is mapped to multiple pages: no problem. If the same page is mapped to multiple offsets, then obviously it's not going to work properly.

Yeap.

Quoted 11 lines
Quoted 3 lines

Can you please elaborate how you think the thing can work without referencing the proposed implementation? Let's find out where the misundertanding is.

Thinking about it I'm not really sure...

Maybe the problem is that the propsed solution allows too much freedom. Normally there's a 1:1 relationship between pages and offsets. But we want to break that for CUSE, because two different mappings of a char dev might point to completely different pages, right?

Yeap. It basically behaves like each mmap() instance is a shm instance and the offset into dmmap_regions is the shmkey.

Quoted 2 lines

When does that happen? Can it happen that two mappings of the same file descriptor will have different backing pages?

Yeah, sure. FUSE server is free to give them separate regions. The only restriction is the SHMLBA alignment which is pretty easy to adhere to.

Thanks.

Signature · 1 line
tejun
maintainer
#7 lore raw
Quoted 6 lines · Tejun Heo
Quoted 2 lines

When does that happen? Can it happen that two mappings of the same file descriptor will have different backing pages?

Yeah, sure. FUSE server is free to give them separate regions. The only restriction is the SHMLBA alignment which is pretty easy to adhere to.

What I really meant was: does it ever happen that mapping a *single open file* of a chardev twice will result in two different maps? Or will the maps be only different if it was a different open of the device?

If there's a pattern there, we might make the sharing/non-sharing automated, and greatly simplify the interface.

Thanks,
Miklos

author
#8 lore raw

Hello,

Quoted 8 lines · Miklos Szeredi
Quoted 3 lines

Yeah, sure. FUSE server is free to give them separate regions. The only restriction is the SHMLBA alignment which is pretty easy to adhere to.

What I really meant was: does it ever happen that mapping a *single open file* of a chardev twice will result in two different maps? Or will the maps be only different if it was a different open of the device?

I don't know if there is one but ->mmap() implementation is definitely allowed to do that.

Quoted 2 lines

If there's a pattern there, we might make the sharing/non-sharing automated, and greatly simplify the interface.

Isn't the interface pretty simple as it is? If we want to make it easier for API users by imposing limits, I think the correct layer to do that would be at the library level.

Thanks.

Signature · 1 line
tejun
maintainer
#9 lore raw
Quoted 4 lines · Tejun Heo
Quoted 2 lines

If there's a pattern there, we might make the sharing/non-sharing automated, and greatly simplify the interface.

Isn't the interface pretty simple as it is?

On one hand it's simple, on the other it has pretty weird limitations, considering that server side mmap shouldn't be mandatory. But of course if server side mmap isn't used, then the SHMLBA limitation is not necessary anymore so the implementation could choose an arbitrary "offset".

My biggest gripe with the kernel API is that we shouldn't be calling that thing in fuse_mmap_out an offset at all, because it's not and it's confusing (like making you set vma->vm_pgoff to that value, which is bogus). Adding a separate "page_id" or whatever would make me happy.

And if the server wants to mmap /dev/fuse then it can do that and send the result in "dev_offset", to make it clear that it's a different offset from the one the client used on the mmap. And it can even use that value as page_id, if it wants to or it can use a different page_id.

Does that sound reasonable?

Quoted 3 lines

If we want to make it
easier for API users by imposing limits, I think the correct layer to do that would be at the library level.

Hmm, might do that. High level library users definitely shouldn't be having to mess around with SHMLBA.

Thanks,
Miklos

maintainer
#10 lore raw
Quoted 11 lines · Miklos Szeredi

On one hand it's simple, on the other it has pretty weird limitations, considering that server side mmap shouldn't be mandatory. But of course if server side mmap isn't used, then the SHMLBA limitation is not necessary anymore so the implementation could choose an arbitrary "offset".

My biggest gripe with the kernel API is that we shouldn't be calling that thing in fuse_mmap_out an offset at all, because it's not and it's confusing (like making you set vma->vm_pgoff to that value, which is bogus). Adding a separate "page_id" or whatever would make me happy.

And even a page_id is superfluous, since all we want is differentiate between possibly separate maps for the same file and don't care about individual pages.

And even differentiating between maps on the same file is only ever possible for char devs, normal files will only ever have a single map.

So instead of "page_id" it could just be a "map_id", which will be zero for normal maps, and whatever for /dev/fuse maps.

Quoted 5 lines

And if the server wants to mmap /dev/fuse then it can do that and send the result in "dev_offset", to make it clear that it's a different offset from the one the client used on the mmap. And it can even use that value as page_id, if it wants to or it can use a different page_id.

And there's a weakness in the current server side mmap interface, for example:

1) user mmaps region 3-5 (in page index)
server maps this region at 3-5

2) user mmaps region 1-7
server can't tell kernel that it wants to reuse region 3-5 but wants to create two other regions

What happens in that case?

On the other hand if the map_id is separate from the dev_offset, then the server can map the second region as one map and the kernel can connect it up with the right pages based on the map_id returned in the MMAP reply.

Thanks,
Miklos

author
#18 lore raw
In reply to Miklos Szeredi

Hello,

Quoted 9 lines · Miklos Szeredi
Quoted 4 lines · Tejun Heo
Quoted 2 lines

If there's a pattern there, we might make the sharing/non-sharing automated, and greatly simplify the interface.

Isn't the interface pretty simple as it is?

On one hand it's simple, on the other it has pretty weird limitations, considering that server side mmap shouldn't be mandatory.

It's not like it's something completely foreign. It's a limitation which can also be found in shared memory and the server side mmap doesn't really have much to do with it. It's also necessary to avoid aliasing issues among clients.

Quoted 3 lines

But of course if server side mmap isn't used, then the SHMLBA limitation is not necessary anymore so the implementation could choose an arbitrary "offset".

It of course is necessary. How else can aliasing among clients be avoided? The alignment is not only for the server. If you're gonna share memories and want to adjust the mapping in any way, you need to follow the SHMLBA alignment.

Quoted 5 lines

My biggest gripe with the kernel API is that we shouldn't be calling that thing in fuse_mmap_out an offset at all, because it's not and it's confusing (like making you set vma->vm_pgoff to that value, which is bogus). Adding a separate "page_id" or whatever would make me happy.

It _is_ an offset into the dmmap address space. mmap() requests to map certain length at certain offset of an address space. The FUSE server can redirect the offset to make it share, not share or combine or whatever. It's very simple conceptually.

Quoted 7 lines

And if the server wants to mmap /dev/fuse then it can do that and send the result in "dev_offset", to make it clear that it's a different offset from the one the client used on the mmap. And it can even use that value as page_id, if it wants to or it can use a different page_id.

Does that sound reasonable?

I can't really follow what you're suggesting. Either you're misunderstanding why SHMLBA is necessary or I'm being plain dumb. Can you please describe what you have on mind without referring to the current implementation? How would it work?

Quoted 5 lines
Quoted 2 lines

If we want to make it easier for API users by imposing limits, I think the correct layer to do that would be at the library level.

Hmm, might do that. High level library users definitely shouldn't be having to mess around with SHMLBA.

The main thing would be to let the library manage mmap regions and ask the server only two things - region size and share ID both of which are optional. If both are not set (ie. -1 and -1), the mmap request won't be shareable and sized as requested by the client. If the server returns a positive ID, all mmap requests with the same ID will share the region.

Also, I said it before but SHMLBA is something which is already known to the userland for shared memories. I really don't think this is too much to ask for if the server is playing with direct mmap.

Thanks.

Signature · 1 line
tejun
author
#11 lore raw
In reply to Miklos Szeredi

Hello,

Quoted 9 lines · Miklos Szeredi

And even a page_id is superfluous, since all we want is differentiate between possibly separate maps for the same file and don't care about individual pages.

And even differentiating between maps on the same file is only ever possible for char devs, normal files will only ever have a single map.

So instead of "page_id" it could just be a "map_id", which will be zero for normal maps, and whatever for /dev/fuse maps.

Well, not exactly. There are device mmap() implementations which simply ignore @offset because offsetting doesn't make any sense at all. ossp mmap is actually done that way. I really don't think it would be wise to impose such restriction at the kernel API level. For highlevel interface, dealing with separate regions could be fine tho.

Quoted 9 lines
Quoted 5 lines

And if the server wants to mmap /dev/fuse then it can do that and send the result in "dev_offset", to make it clear that it's a different offset from the one the client used on the mmap. And it can even use that value as page_id, if it wants to or it can use a different page_id.

And there's a weakness in the current server side mmap interface, for example:

Quoted 8 lines

1) user mmaps region 3-5 (in page index)
server maps this region at 3-5

2) user mmaps region 1-7
server can't tell kernel that it wants to reuse region 3-5 but wants to create two other regions

What happens in that case?

If the server wants the two regions to be separate, it can map it to say 5-11 and returnt he offset of 5. If it wants them to be shared, it will have to mmap 1-2 and 6-7 and return offset of 1. But, the server should really know better than growing the area this way. If this type of expansion ever becomes problem, we can implement expendable mmap on the server side (ie. don't require VM_DONT_EXPAND).

Quoted 4 lines

On the other hand if the map_id is separate from the dev_offset, then the server can map the second region as one map and the kernel can connect it up with the right pages based on the map_id returned in the MMAP reply.

How is this not possible in the current implementation?

Thanks.

Signature · 1 line
tejun
maintainer
#12 lore raw
Quoted 10 lines · Tejun Heo
Quoted 10 lines

And there's a weakness in the current server side mmap interface, for example:

1) user mmaps region 3-5 (in page index)
server maps this region at 3-5

2) user mmaps region 1-7
server can't tell kernel that it wants to reuse region 3-5 but wants to create two other regions

Quoted 5 lines
Quoted 1 line

What happens in that case?

If the server wants the two regions to be separate, it can map it to say 5-11 and returnt he offset of 5. If it wants them to be shared, it will have to mmap 1-2 and 6-7 and return offset of 1.

What if region 6-7 is already occupied (e.g. because a separate region was put there)?

Thanks,
Miklos

maintainer
#19 lore raw
In reply to Tejun Heo
Quoted 4 lines · Tejun Heo

It's not like it's something completely foreign. It's a limitation which can also be found in shared memory and the server side mmap doesn't really have much to do with it. It's also necessary to avoid aliasing issues among clients.

What aliasing issues among clients? That's a job for the arch dependent part of mmap. And as you said, there's not a lot the driver can do (except play with ->vm_pgoff) to influence the address selection.

And playing with ->vm_pgoff is *not* a valid thing to do for the client, at least not on "normal" mmaps.

Thanks,
Miklos

author
#13 lore raw
In reply to Miklos Szeredi

Hello, Miklos.

Quoted 6 lines · Miklos Szeredi
Quoted 3 lines

If the server wants the two regions to be separate, it can map it to say 5-11 and returnt he offset of 5. If it wants them to be shared, it will have to mmap 1-2 and 6-7 and return offset of 1.

What if region 6-7 is already occupied (e.g. because a separate region was put there)?

Allocating and managing the address space ranges are the server's responsibility. If it expects the region to grow, it shouldn't colocate those regions. The kernel is just giving the server an address space to manage and letting it redirect mmaps to arbitrary (sans the SHMLBA alignment restriction) part of it. The rest is upto the server.

Thanks.

Signature · 1 line
tejun
maintainer
#14 lore raw
Quoted 10 lines · Tejun Heo

Hello, Miklos.

Quoted 6 lines · Miklos Szeredi
Quoted 3 lines

If the server wants the two regions to be separate, it can map it to say 5-11 and returnt he offset of 5. If it wants them to be shared, it will have to mmap 1-2 and 6-7 and return offset of 1.

What if region 6-7 is already occupied (e.g. because a separate region was put there)?

Quoted 6 lines

Allocating and managing the address space ranges are the server's responsibility. If it expects the region to grow, it shouldn't colocate those regions. The kernel is just giving the server an address space to manage and letting it redirect mmaps to arbitrary (sans the SHMLBA alignment restriction) part of it. The rest is upto the server.

The problem with that is you simply can't determine in advance where the region will grow. Okay, you can leave space according to the size of the file, but the size of the file can grow too.

*This* is the complexity that I want to get rid of.

Thanks,
Miklos

author
#20 lore raw
In reply to Miklos Szeredi

Hello,

Quoted 6 lines · Miklos Szeredi
Quoted 4 lines · Tejun Heo

It's not like it's something completely foreign. It's a limitation which can also be found in shared memory and the server side mmap doesn't really have much to do with it. It's also necessary to avoid aliasing issues among clients.

Quoted 7 lines

What aliasing issues among clients? That's a job for the arch dependent part of mmap. And as you said, there's not a lot the driver can do (except play with ->vm_pgoff) to influence the address selection.

And playing with ->vm_pgoff is *not* a valid thing to do for the client, at least not on "normal" mmaps.

That's the requirement coming from allowing the server to determine how these mmaps are served, not from the fact that the management is done via server side mmaps or the server maps those regions into its process address space. No matter how you do it, if you want to mix and match client mmap requests, the SHMLBA alignment will be visible. I suppose you're talking about not allowing offsets to be adjusted at all but I don't think that's a restriction we would want to have at the kernel API level because offset might encode different things for device mmaps.

Thanks.

Signature · 1 line
tejun
author
#15 lore raw
In reply to Miklos Szeredi

Hello,

Quoted 5 lines · Miklos Szeredi

The problem with that is you simply can't determine in advance where the region will grow. Okay, you can leave space according to the size of the file, but the size of the file can grow too.

*This* is the complexity that I want to get rid of.

Alright, then let's talk about that, not SHMLBA which doesn't really have much to do with this. So, you're basically saying that you want multiple address spaces. In that case, the only logical abstraction for that are files. ie. We need to be passing file descriptors to the server and asking the server which descriptor it would want to use, which was the previous implementation, which you objected mentioning that this type of direct mmap would probably be useful only for device mmap implementations.

The address space provided for dmmap is huge. It's any range which pgoff_t can address. I really can't imagine managing allocation from that space causing problems. If you want to use it for multiple huge mappings, we can't use the simple implementation w/ pinned pages anyway. We'll have to borrow anonymous file implementation or implement private swap backing.

Thanks.

Signature · 1 line
tejun
maintainer
#21 lore raw
In reply to Tejun Heo
Quoted 8 lines · Tejun Heo
Quoted 2 lines

And playing with ->vm_pgoff is *not* a valid thing to do for the client, at least not on "normal" mmaps.

That's the requirement coming from allowing the server to determine how these mmaps are served, not from the fact that the management is done via server side mmaps or the server maps those regions into its process address space. No matter how you do it, if you want to mix and match client mmap requests, the SHMLBA alignment will be visible.

Can you give an example?

Quoted 4 lines

I suppose you're talking about not allowing offsets to be adjusted at all but I don't think that's a restriction we would want to have at the kernel API level because offset might encode different things for device mmaps.

Possibly. But there's some confusion about offsets again. This offset is not the same as the one currently returned in fuse_mmap_out.

And neither is the SHMLBA alignment requirement so clear:

You say, that ossp ignores the client mmap offset. So basically it resets the offset to zero, whatever it was, no? That sounds fine, but then you are adjusting the offset by something not necessarily a multiple of SHMLBA.

So there are different offsets:

a) vma->vm_pgoff (which may mean anything, but usually means b)
b) the offset at which the pages of the mapping are located
c) the offset at which the server side mmap is located

Thanks,
Miklos

maintainer
#16 lore raw
In reply to Tejun Heo
Quoted 9 lines · Tejun Heo

Hello,

Quoted 5 lines · Miklos Szeredi

The problem with that is you simply can't determine in advance where the region will grow. Okay, you can leave space according to the size of the file, but the size of the file can grow too.

*This* is the complexity that I want to get rid of.

Quoted 8 lines

Alright, then let's talk about that, not SHMLBA which doesn't really have much to do with this. So, you're basically saying that you want multiple address spaces. In that case, the only logical abstraction for that are files. ie. We need to be passing file descriptors to the server and asking the server which descriptor it would want to use, which was the previous implementation, which you objected mentioning that this type of direct mmap would probably be useful only for device mmap implementations.

No, I don't want to pass file descriptors (early fuse did use that for readdir and it was a mistake).

Global address space for server side maps are OK. What I mind is that in the *absence* of server side maps the filesystem still has to deal with the global address space (with all its quirks). This is totally unnecessary.

Thanks,
Miklos

author
#22 lore raw
In reply to Miklos Szeredi

Hello,

Quoted 7 lines · Miklos Szeredi
Quoted 5 lines

That's the requirement coming from allowing the server to determine how these mmaps are served, not from the fact that the management is done via server side mmaps or the server maps those regions into its process address space. No matter how you do it, if you want to mix and match client mmap requests, the SHMLBA alignment will be visible.

Can you give an example?

Hmmm... I don't have any specific example. osspd is the only real thing which uses it and it serves each mmap request with a separate region. Oh... the fmmap example program serves the same region to mmap requests coming from the same UID, so it does both sharing and putting maps apart.

Quoted 7 lines
Quoted 4 lines

I suppose you're talking about not allowing offsets to be adjusted at all but I don't think that's a restriction we would want to have at the kernel API level because offset might encode different things for device mmaps.

Possibly. But there's some confusion about offsets again. This offset is not the same as the one currently returned in fuse_mmap_out.

This offset is the offset client requests.

Quoted 6 lines

And neither is the SHMLBA alignment requirement so clear:

You say, that ossp ignores the client mmap offset. So basically it resets the offset to zero, whatever it was, no? That sounds fine, but then you are adjusting the offset by something not necessarily a multiple of SHMLBA.

Yeap and that would be a bug. It will probably have to do % SHMLBA on the offset. I don't think all OSS drivers would be caring about these stuff anyway. Most of them work on only x86 where SHMLBA == PAGE_SIZE.

Quoted 3 lines

So there are different offsets:

a) vma->vm_pgoff (which may mean anything, but usually means b)

Yeap, vma->vm_pgoff can be any value and doesn't really matter. The only visible difference would be the /proc listing, right? Setting this to the requested offset is trivial.

Quoted 2 lines

b) the offset at which the pages of the mapping are located
c) the offset at which the server side mmap is located

There are three offsets.

a) the offset a client requested

b) the offset into dmmap AS, a client mmap region is mapped to. This could be different from a) by multiple of SHMLBA / PAGE_SIZE.

c) the offset into dmmap AS, a server mmap region is mapped to, where collection of these mmaps define the dmmap AS.

The offsets used in b) and c) are the same offsets.

Thanks.

Signature · 1 line
tejun
author
#17 lore raw
In reply to Miklos Szeredi

Hello,

Quoted 2 lines · Miklos Szeredi

No, I don't want to pass file descriptors (early fuse did use that for readdir and it was a mistake).

Yeah, I agree that passing fds is pretty painful to implement and look at.

Quoted 1 line

Global address space for server side maps are OK.

Do you mean having single dmmap AS is okay?

Quoted 3 lines

What I mind is that in the *absence* of server side maps the filesystem still has to deal with the global address space (with all its quirks). This is totally unnecessary.

And then the above seems contradictory with the first sentence. Sorry to be dense but I really don't follow what you're suggesting. If you're okay with single dmmap AS, how can you not be okay with the quirks of dealing with global AS? The only exit from that problem is having multiple ASes.

Thanks.

Signature · 1 line
tejun
maintainer
#23 lore raw
In reply to Tejun Heo
Quoted 7 lines · Tejun Heo
Quoted 3 lines

So there are different offsets:

a) vma->vm_pgoff (which may mean anything, but usually means b)

Yeap, vma->vm_pgoff can be any value and doesn't really matter. The only visible difference would be the /proc listing, right? Setting this to the requested offset is trivial.

You mean leaving it at the requested offset? Yes, that's the most trivial thing to do. Very few drivers change vm_pgoff:

git grep "vm_pgoff *=[^=]"

Quoted 9 lines
Quoted 2 lines

b) the offset at which the pages of the mapping are located
c) the offset at which the server side mmap is located

There are three offsets.

a) the offset a client requested

b) the offset into dmmap AS, a client mmap region is mapped to. This could be different from a) by multiple of SHMLBA / PAGE_SIZE.

No, it could be different from a) by an arbitrary value.

Quoted 5 lines

c) the offset into dmmap AS, a server mmap region is mapped to, where collection of these mmaps define the dmmap AS.

The offsets used in b) and c) are the same offsets.

Why are they the same?

Thanks,
Miklos

author
#24 lore raw

Hello,

Quoted 5 lines · Miklos Szeredi
Quoted 4 lines · Tejun Heo
Quoted 3 lines

So there are different offsets:

a) vma->vm_pgoff (which may mean anything, but usually means b)

Quoted 8 lines
Quoted 3 lines

Yeap, vma->vm_pgoff can be any value and doesn't really matter. The only visible difference would be the /proc listing, right? Setting this to the requested offset is trivial.

You mean leaving it at the requested offset? Yes, that's the most trivial thing to do. Very few drivers change vm_pgoff:

git grep "vm_pgoff *=[^=]"

Yeap, sure. I just didn't think it was visible outside.

Quoted 11 lines
Quoted 9 lines
Quoted 2 lines

b) the offset at which the pages of the mapping are located
c) the offset at which the server side mmap is located

There are three offsets.

a) the offset a client requested

b) the offset into dmmap AS, a client mmap region is mapped to. This could be different from a) by multiple of SHMLBA / PAGE_SIZE.

No, it could be different from a) by an arbitrary value.

Then, sharing those pages would cause aliasing issues.

Quoted 6 lines
Quoted 4 lines

c) the offset into dmmap AS, a server mmap region is mapped to, where collection of these mmaps define the dmmap AS.

The offsets used in b) and c) are the same offsets.

Why are they the same?

I meant they point into the same space. If they're the same value, they point to the same page.

Thanks.

Signature · 1 line
tejun
maintainer
#25 lore raw
Quoted 10 lines · Tejun Heo
Quoted 8 lines
Quoted 6 lines

There are three offsets.

a) the offset a client requested

b) the offset into dmmap AS, a client mmap region is mapped to. This could be different from a) by multiple of SHMLBA / PAGE_SIZE.

No, it could be different from a) by an arbitrary value.

Then, sharing those pages would cause aliasing issues.

You said a few mails up:

"There are device mmap() implementations which simply ignore @offset
because offsetting doesn't make any sense at all"

Which means a) doesn't necessarily matter, so it's not something that determines aliasing issues.

Quoted 9 lines
Quoted 6 lines
Quoted 4 lines

c) the offset into dmmap AS, a server mmap region is mapped to, where collection of these mmaps define the dmmap AS.

The offsets used in b) and c) are the same offsets.

Why are they the same?

I meant they point into the same space. If they're the same value, they point to the same page.

I'm beginning to undestand what you mean by "dmmap AS".

The thing is, I'm still not sure if or how this kind of mmap makes sense outside of the CUSE context. Which makes designing the API difficult.

So, for now maybe it's best to go with your implementation, fix issues with the offsets and make it CUSE only for the moment.

The alternative is for me to start implementing a coherent distributed filesystem, so I can see what the actual requirements for a direct mmap would be. That would be fun, but it would

a) delay direct mmap for CUSE by an unknown amount of time
b) delay everything else that I have in the pipeline ;)

Thanks,
Miklos

author
#26 lore raw

Hello,

Quoted 9 lines · Miklos Szeredi
Quoted 1 line

Then, sharing those pages would cause aliasing issues.

You said a few mails up:

"There are device mmap() implementations which simply ignore @offset
because offsetting doesn't make any sense at all"

Which means a) doesn't necessarily matter, so it's not something that determines aliasing issues.

Mmap regions of devices aren't always used as shared memories. IO regions often don't have cache backing at all. Also, certain arch is often assumed.

And yes it is something which determines aliasing issues. There is no way around it.

Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
Quoted 1 line

The offsets used in b) and c) are the same offsets.

Why are they the same?

I meant they point into the same space. If they're the same value, they point to the same page.

I'm beginning to undestand what you mean by "dmmap AS".

The thing described by rbtree of struct dmmap_regions.

Quoted 3 lines

The thing is, I'm still not sure if or how this kind of mmap makes sense outside of the CUSE context. Which makes designing the API difficult.

For this to be useful for normal FS, it has to be backed by multiple swap backed files and I can almost guarantee you would need to be passing fds around.

Quoted 2 lines

So, for now maybe it's best to go with your implementation, fix issues with the offsets and make it CUSE only for the moment.

The offset problem can't be fixed. If you allow offsets to be adjusted in any way, SHMLBA requirement is gonna be there and for CUSE I think having the ability to adjust offset would be useful even if multiple files are used. It can of course be hidden behind a highlevel library API tho.

Quoted 6 lines

The alternative is for me to start implementing a coherent distributed filesystem, so I can see what the actual requirements for a direct mmap would be. That would be fun, but it would

a) delay direct mmap for CUSE by an unknown amount of time
b) delay everything else that I have in the pipeline ;)

Well, for CUSE, it has been delayed for a long time already so I don't think there would be much harm in waiting a bit more. Any estimates on how long it would take?

Thanks.

Signature · 1 line
tejun
author
#27 lore raw

Hello,

Quoted 7 lines · Tejun Heo
Quoted 3 lines

The thing is, I'm still not sure if or how this kind of mmap makes sense outside of the CUSE context. Which makes designing the API difficult.

For this to be useful for normal FS, it has to be backed by multiple swap backed files and I can almost guarantee you would need to be passing fds around.

BTW, if you're looking this way, one way to do it would be to ask the server to return a fd to use for the mmap backing and offset adjustment (still needs to be SHMLBA aligned) rather than creating anonymous file from the kernel. In this approach, the problem would be wrapping the vma ops so that FUSE can determine when the mmap segment is released. I did it in pretty ugly way in the previous mmap implementation by simply overwriting the ops as anonymous files didn't have their own ->open or ->close but to do this for any file you'll need a way to nest or inherit vma ops in generic way. The problem is that the assumptions regarding vma are pretty deeply buried all over the VM layer. FWIW, I couldn't think of a way to do that in generic manner.

Thanks.

Signature · 1 line
tejun
maintainer
#28 lore raw
In reply to Tejun Heo
Quoted 5 lines · Tejun Heo

The offset problem can't be fixed. If you allow offsets to be adjusted in any way, SHMLBA requirement is gonna be there and for CUSE I think having the ability to adjust offset would be useful even if multiple files are used. It can of course be hidden behind a highlevel library API tho.

Yes, offset issues can be fixed:

1) don't change vma->vm_pgoff

2) in fuse_mmap_out, call it dev_offset, dev_mmap_offset or whatever

3) on the low level API don't make it an offset pointer that is adjusted. It's not an offset to be either left alone or changed (that would be the case if we wanted to allow adjustment to vma->vm_pgoff itself). It's about calclulating a completely new offset for the server side mmap.

Quoted 10 lines
Quoted 6 lines

The alternative is for me to start implementing a coherent distributed filesystem, so I can see what the actual requirements for a direct mmap would be. That would be fun, but it would

a) delay direct mmap for CUSE by an unknown amount of time
b) delay everything else that I have in the pipeline ;)

Well, for CUSE, it has been delayed for a long time already so I don't think there would be much harm in waiting a bit more. Any estimates on how long it would take?

Well if there were no higher priority things then I think I could do that in a month.

I don't think I want a swap backed solution. There is already a backing for these maps and that is the userspace filesystem. In fact most of what is required is already there in the form of the page cache. What I think would be interesting to be able to load/save contents of page cache from the server side, and not necessarily using server side mmaps (server side mmap is also a possibility, but not an easy one if it has to cooperate with the page cache).

Basically all we need to ensure, is that the mmap API doesn't conflict with the above usage. The problem is that the details for the above usage will only come out with a real implementation.

Thanks,
Miklos

author
#29 lore raw

Hello,

Quoted 10 lines · Miklos Szeredi
Quoted 5 lines · Tejun Heo

The offset problem can't be fixed. If you allow offsets to be adjusted in any way, SHMLBA requirement is gonna be there and for CUSE I think having the ability to adjust offset would be useful even if multiple files are used. It can of course be hidden behind a highlevel library API tho.

Yes, offset issues can be fixed:

1) don't change vma->vm_pgoff

vma->vm_pgoff is peripheral unless you mean not adjusting offset at all.

Quoted 1 line

2) in fuse_mmap_out, call it dev_offset, dev_mmap_offset or whatever

Okay.

Quoted 5 lines

3) on the low level API don't make it an offset pointer that is adjusted. It's not an offset to be either left alone or changed (that would be the case if we wanted to allow adjustment to vma->vm_pgoff itself). It's about calclulating a completely new offset for the server side mmap.

And now I'm completely lost. So, we'll assign new offset (no matter how it is called) but it doesn't have to be aligned? It seems like we've been having this disconnection from the beginning. Can you please describe how this can avoid aliasing issues between clients sharing the same page? So, in 2), whatever it is called, the server specifies a value, how is that value used?

Quoted 6 lines
Quoted 3 lines

Well, for CUSE, it has been delayed for a long time already so I don't think there would be much harm in waiting a bit more. Any estimates on how long it would take?

Well if there were no higher priority things then I think I could do that in a month.

Alright, we'll be missing the upcoming merge window anyway, so no biggie.

Quoted 2 lines

I don't think I want a swap backed solution. There is already a backing for these maps and that is the userspace filesystem.

Yeap, sure. This was what I was talking about in the other reply. Named files would probably be much easier to work with for the server implementations.

Quoted 6 lines

In fact most of what is required is already there in the form of the page cache. What I think would be interesting to be able to load/save contents of page cache from the server side, and not necessarily using server side mmaps (server side mmap is also a possibility, but not an easy one if it has to cooperate with the page cache).

Device mmap use cases might not work very well if the server can't mmap directly.

Quoted 3 lines

Basically all we need to ensure, is that the mmap API doesn't conflict with the above usage. The problem is that the details for the above usage will only come out with a real implementation.

Alright, I'll ping you in a while.

Thanks.

Signature · 1 line
tejun
maintainer
#30 lore raw
Quoted 6 lines · Tejun Heo
Quoted 5 lines

3) on the low level API don't make it an offset pointer that is adjusted. It's not an offset to be either left alone or changed (that would be the case if we wanted to allow adjustment to vma->vm_pgoff itself). It's about calclulating a completely new offset for the server side mmap.

Quoted 6 lines

And now I'm completely lost. So, we'll assign new offset (no matter how it is called) but it doesn't have to be aligned? It seems like we've been having this disconnection from the beginning. Can you please describe how this can avoid aliasing issues between clients sharing the same page? So, in 2), whatever it is called, the server specifies a value, how is that value used?

That dev_offset value is used as an offset into the server side mmap address space. And yes, vma->vm_pgoff and dev_offset should be SHMLBA multiples apart.

But don't call that _adjustment_. That's totally confusing, these are *two* *different* *offsets*. There's an alignment requirement but that's all. If they are the same that is pure coincidence.

And dev_offset (which points into the dmmap address space) is only required if the filesystem/CUSE driver needs server side mmap.

Quoted 9 lines
Quoted 6 lines

In fact most of what is required is already there in the form of the page cache. What I think would be interesting to be able to load/save contents of page cache from the server side, and not necessarily using server side mmaps (server side mmap is also a possibility, but not an easy one if it has to cooperate with the page cache).

Device mmap use cases might not work very well if the server can't mmap directly.

I understand that, and that's where the interesting part comes in:
make the mmap API in a way that it works with and without server side mmap.

Thanks,
Miklos

author
#31 lore raw

Hello,

Quoted 3 lines · Miklos Szeredi

That dev_offset value is used as an offset into the server side mmap address space. And yes, vma->vm_pgoff and dev_offset should be SHMLBA multiples apart.

Alright.

Quoted 2 lines

But don't call that _adjustment_. That's totally confusing, these are *two* *different* *offsets*.

At this point, we're completely in the realm of the babel tower. Unless we define all the terms explicitly, I don't think using or not using a single word would make much difference. Working with FUSE gets a bit confusing regarding which term means what but with multiple process address spaces and the dmmap address space and all the offsets, it's getting pretty ridiculous.

Quoted 5 lines

There's an alignment requirement but that's all. If they are the same that is pure coincidence.

And dev_offset (which points into the dmmap address space) is only required if the filesystem/CUSE driver needs server side mmap.

Hmmm... w/o the dev_offset how would the server designate which area to use for the client?

Thanks.

Signature · 1 line
tejun