unregistered changes to the user<->kernel API

There are a number of changes in kernel API visisble to userspace that are unregistered in 2.4 mainline. I recommend to merge them ASAP to avoid generating collisions across different versions of the kernel.

I'll attach here a number of patches that should make us to return in sync. They must be applied incrementally. (really the very last one is mostly here for comments, not intendeted for merging in mainline)

here the first that defines O_DIRECT (NOTE: the O_DIRECT value for alpha is not definitive yet, O_DIRECTIO of tru64 is our O_NOFOLLOW so we're just screwed as we just need a wrapper anyways to make complex programs like dbms to run correctly without having to natively port them to linux, 02000000 in tru64 is O_DSYNC, maybe I should move it to 010000000 instead which maybe unused in tru64, but still we would have no guarantee that it won't be used in the future, I was waiting Richard's comment about it).

The sparc64 values are approved by Dave.

diff -urN 2.4.6pre3/include/asm-alpha/fcntl.h o_direct/include/asm-alpha/fcntl.h
@@ -17,10 +17,10 @@
 #define O_NDELAY	O_NONBLOCK
 #define O_SYNC		040000
 #define FASYNC		020000	/* fcntl, for BSD compatibility */
-#define O_DIRECT	040000	/* direct disk access - should check with OSF/1 */
 #define O_DIRECTORY	0100000	/* must be a directory */
 #define O_NOFOLLOW	0200000 /* don't follow links */
 #define O_LARGEFILE	0400000 /* will be set by the kernel on every open */
+#define O_DIRECT	02000000 /* direct disk access - should check with OSF/1 */
 
 #define F_DUPFD		0	/* dup */
 #define F_GETFD		1	/* get close_on_exec */

diff -urN 2.4.6pre3/include/asm-i386/fcntl.h o_direct/include/asm-i386/fcntl.h
@@ -16,7 +16,7 @@
 #define O_NDELAY	O_NONBLOCK
 #define O_SYNC		 010000
 #define FASYNC		 020000	/* fcntl, for BSD compatibility */
-#define O_DIRECT	 040000	/* direct disk access hint - currently ignored */
+#define O_DIRECT	 040000	/* direct disk access hint */
 #define O_LARGEFILE	0100000
 #define O_DIRECTORY	0200000	/* must be a directory */
 #define O_NOFOLLOW	0400000 /* don't follow links */

diff -urN 2.4.6pre3/include/asm-sparc/fcntl.h o_direct/include/asm-sparc/fcntl.h
@@ -20,6 +20,7 @@
 #define O_DIRECTORY	0x10000	/* must be a directory */
 #define O_NOFOLLOW	0x20000	/* don't follow links */
 #define O_LARGEFILE	0x40000
+#define O_DIRECT        0x100000 /* direct disk access hint */
 
 #define F_DUPFD		0	/* dup */
 #define F_GETFD		1	/* get close_on_exec */

diff -urN 2.4.6pre3/include/asm-sparc64/fcntl.h o_direct/include/asm-sparc64/fcntl.h
@@ -20,6 +20,8 @@
 #define O_DIRECTORY	0x10000	/* must be a directory */
 #define O_NOFOLLOW	0x20000	/* don't follow links */
 #define O_LARGEFILE	0x40000
+#define O_DIRECT        0x100000 /* direct disk access hint */
+
 
 #define F_DUPFD		0	/* dup */
 #define F_GETFD		1	/* get close_on_exec */

Here the second patch that defines the PF_ATOMICALLOC (strictly speaking this is not visible from userspace but it also cleanups a bit the definitions).

@@ -403,18 +403,15 @@
 /*
  * Per process flags
  */
-#define PF_ALIGNWARN	0x00000001	/* Print alignment warning msgs */
-					/* Not implemented yet, only for 486*/
-#define PF_STARTING	0x00000002	/* being created */
-#define PF_EXITING	0x00000004	/* getting shut down */
-#define PF_FORKNOEXEC	0x00000040	/* forked but didn't exec */
-#define PF_SUPERPRIV	0x00000100	/* used super-user privileges */
-#define PF_DUMPCORE	0x00000200	/* dumped core */
-#define PF_SIGNALED	0x00000400	/* killed by a signal */
-#define PF_MEMALLOC	0x00000800	/* Allocating memory */
-#define PF_VFORK	0x00001000	/* Wake up parent in mm_release */
-
-#define PF_USEDFPU	0x00100000	/* task used FPU this quantum (SMP) */
+#define PF_EXITING	(1UL<<0)	/* getting shut down */
+#define PF_FORKNOEXEC	(1UL<<1)	/* forked but didn't exec */
+#define PF_SUPERPRIV	(1UL<<2)	/* used super-user privileges */
+#define PF_DUMPCORE	(1UL<<3)	/* dumped core */
+#define PF_SIGNALED	(1UL<<4)	/* killed by a signal */
+#define PF_MEMALLOC	(1UL<<5)	/* Allocating memory */
+#define PF_VFORK	(1UL<<6)	/* Wake up parent in mm_release */
+#define PF_USEDFPU	(1UL<<7)	/* task used FPU this quantum (SMP) */
+#define PF_ATOMICALLOC	(1UL<<8)	/* do not block during memalloc */
 
 /*
  * Ptrace flags

Here the third, it registers the tux syscall at for the alpha so other people won't use such same syscall for something else (I didn't remove the #ifdefs since they don't hurt as they're undefined in mainline).

diff -urN ref/arch/alpha/kernel/entry.S tuxsys/arch/alpha/kernel/entry.S
@@ -1004,7 +1004,15 @@
 	.quad alpha_ni_syscall
 	.quad alpha_ni_syscall			/* 220 */
 	.quad alpha_ni_syscall
+#ifdef CONFIG_TUX
+	.quad __sys_tux
+#else
+# ifdef CONFIG_TUX_MODULE
+	.quad sys_tux
+# else
 	.quad alpha_ni_syscall
+# endif
+#endif
 	.quad alpha_ni_syscall
 	.quad alpha_ni_syscall
 	.quad alpha_ni_syscall			/* 225 */

diff -urN ref/arch/i386/kernel/entry.S tuxsys/arch/i386/kernel/entry.S
@@ -645,7 +645,15 @@
 	.long SYMBOL_NAME(sys_madvise)
 	.long SYMBOL_NAME(sys_getdents64)	/* 220 */
 	.long SYMBOL_NAME(sys_fcntl64)
+#ifdef CONFIG_TUX
+	.long SYMBOL_NAME(__sys_tux)
+#else
+# ifdef CONFIG_TUX_MODULE
+	.long SYMBOL_NAME(sys_tux)
+# else
 	.long SYMBOL_NAME(sys_ni_syscall)	/* reserved for TUX */
+# endif
+#endif
 
 	.rept NR_syscalls-(.-sys_call_table)/4
 		.long SYMBOL_NAME(sys_ni_syscall)

Here the forth, this defines the O_ATOMICLOOKUP and EWOULDBLOCKIO for the non blocking dcache and pagecache lookups (the LOOKUP_ATOMIC isn't visible from userspace but I defined it since I was there, if you want you can drop the include/linux/fs.h part of the patch):

diff -urN ref/include/asm-alpha/fcntl.h atomiclookup/include/asm-alpha/fcntl.h
@@ -20,6 +20,7 @@
 #define O_DIRECTORY	0100000	/* must be a directory */
 #define O_NOFOLLOW	0200000 /* don't follow links */
 #define O_LARGEFILE	0400000 /* will be set by the kernel on every open */
+#define O_ATOMICLOOKUP  01000000 /* do atomic file lookup */
 #define O_DIRECT	02000000 /* direct disk access - should check with OSF/1 */
 
 #define F_DUPFD		0	/* dup */

diff -urN ref/include/asm-i386/fcntl.h atomiclookup/include/asm-i386/fcntl.h
@@ -20,6 +20,7 @@
 #define O_LARGEFILE	0100000
 #define O_DIRECTORY	0200000	/* must be a directory */
 #define O_NOFOLLOW	0400000 /* don't follow links */
+#define O_ATOMICLOOKUP	01000000 /* do atomic file lookup */
 
 #define F_DUPFD		0	/* dup */
 #define F_GETFD		1	/* get close_on_exec */

diff -urN ref/include/asm-ia64/fcntl.h atomiclookup/include/asm-ia64/fcntl.h
@@ -28,6 +28,7 @@
 #define O_LARGEFILE	0100000
 #define O_DIRECTORY	0200000	/* must be a directory */
 #define O_NOFOLLOW	0400000 /* don't follow links */
+#define O_ATOMICLOOKUP  01000000 /* do atomic file lookup */
 
 #define F_DUPFD		0	/* dup */
 #define F_GETFD		1	/* get close_on_exec */

diff -urN ref/include/asm-sparc/fcntl.h atomiclookup/include/asm-sparc/fcntl.h
@@ -20,6 +20,7 @@
 #define O_DIRECTORY	0x10000	/* must be a directory */
 #define O_NOFOLLOW	0x20000	/* don't follow links */
 #define O_LARGEFILE	0x40000
+#define O_ATOMICLOOKUP  0x80000 /* do atomic file lookup */
 #define O_DIRECT        0x100000 /* direct disk access hint */
 
 #define F_DUPFD		0	/* dup */

diff -urN ref/include/asm-sparc64/fcntl.h atomiclookup/include/asm-sparc64/fcntl.h
@@ -20,6 +20,7 @@
 #define O_DIRECTORY	0x10000	/* must be a directory */
 #define O_NOFOLLOW	0x20000	/* don't follow links */
 #define O_LARGEFILE	0x40000
+#define O_ATOMICLOOKUP  0x80000 /* do atomic file lookup */
 #define O_DIRECT        0x100000 /* direct disk access hint */
 
 

diff -urN ref/include/linux/errno.h atomiclookup/include/linux/errno.h
@@ -21,6 +21,9 @@
 #define EBADTYPE	527	/* Type not supported by server */
 #define EJUKEBOX	528	/* Request initiated, but will not complete before timeout */
 
+/* Defined for TUX async IO */
+#define EWOULDBLOCKIO	530	/* Would block due to block-IO */
+
 #endif
 
 #endif

diff -urN ref/include/linux/fs.h atomiclookup/include/linux/fs.h
@@ -1227,6 +1227,7 @@
 #define LOOKUP_POSITIVE		(8)
 #define LOOKUP_PARENT		(16)
 #define LOOKUP_NOALT		(32)
+#define LOOKUP_ATOMIC		(64)
 /*
  * Type of the last component on LOOKUP_PARENT
  */

Here the fifth, this defines the tux sysctl numbers (OTOH the sysctl by number gets broken all the time and nobody should use sysctl by number with new sysctls anyways).

This last one gets visible in /proc/stat and I definitely hate it, it should be really put somewhere else, it doesn't belong to /proc/stat, so I'd vote to change tux to put it in a directory specific to tux that is just present of course (but for now I'll keep it in my tree to avoid generating userspace incompatibilities).

Andrea

I just got the email from Richard that he prefers to break O_NOFOLLOW than to define O_DIRECT to something else than 0200000. So probably there will be an incrmeental patch for the alpha later to apply on top of the previous ones.

Also please folks remind to never choose random numbers for the alpha userspace visible kernel API.

Andrea

Richard are you sure we can break O_NOFOLLOW and still expect the machine to boot?

./elf/cache.c: fd = open (temp_name, O_CREAT|O_WRONLY|O_TRUNC|O_NOFOLLOW, ./elf/dl-profile.c:#ifdef O_NOFOLLOW
./elf/dl-profile.c:# define EXTRA_FLAGS | O_NOFOLLOW ./elf/rtld.c:#ifdef O_NOFOLLOW
./elf/rtld.c: const int flags = O_WRONLY | O_APPEND | O_CREAT | O_NOFOLLOW;
./include/asm/fcntl.h:#define O_NOFOLLOW 0400000 /* don't follow links */
./sysdeps/generic/check_fds.c: the O_NOFOLLOW flag for open() but only on some system. */
./sysdeps/generic/check_fds.c:#ifndef O_NOFOLLOW
./sysdeps/generic/check_fds.c:# define O_NOFOLLOW 0

./sysdeps/generic/check_fds.c:  check_one_fd (STDIN_FILENO, O_RDONLY | O_NOFOLLOW);
./sysdeps/generic/check_fds.c:  check_one_fd (STDOUT_FILENO, O_RDWR | O_NOFOLLOW);
./sysdeps/generic/check_fds.c:  check_one_fd (STDERR_FILENO, O_RDWR | O_NOFOLLOW);

./sysdeps/unix/sysv/linux/alpha/bits/fcntl.h:# define O_NOFOLLOW 0200000 /* Do not follow links. */
./sysdeps/unix/sysv/linux/shm_open.c: fd = open (fname, oflag | O_NOFOLLOW, mode);

Andrea

[uses in glibc]

Yes, I saw those. What is the effect of O_NOFOLLOW? To not follow symbolic links when opening the file. If you open a regular file, in effect nothing happens. Moreover, if these opens were not finding files now, the system wouldn't work.

So: the effect, I suppose, is (1) disabling some security within glibc, and (2) making these accesses slower since they will be considered O_DIRECT after the change.

Which doesn't seem that life-threatening to me.

r~

O_NOFOLLOW is used to deal with symlink attacks. Breaking it means that for quite a few binaries you are opening security holes. And since it's a flagday change, you'll get the situation when no version will work for all kernels. Bad idea, IMO.

If tux will ever be merged into mainline eventually I don't think there's a value in defer such bit. Of course if tux will never get merged then I totally agree with you.

same as above.

Then you may prefer to wait tux to be merged before merging the rest as well, in the meantime 90% of the kernels running out there will show such stuff out of /proc/stats (hopefully "the same stuff" which is why I'm posting those patches in first place).

Andrea

and then read/write will return -EINVAL which is life-threatening. O_DIRECT like rawio via /dev/raw imposes special buffer size and alignment (size multiple of softblocksize of the fs and softblocksize alignment, at max I can turn it down to hardblocksize without intensive changes and guaranteeing zerocopy [modulo bounce buffers on x86 of course]).

So in short at least glibc would need to be replaced...

Andrea

You're missing the point -- it's a bad precedent.

How many kernel forks and patches exist out there on the net?

Many of these patches will get merged eventually. But it is a bad idea to include bits of such into the Linus tree, when they are not used in the Linus tree.

-Exceptions- to this policy should be carefully considered... reserving syscall and sysctl numbers certainly makes sense. Bloating kernel_stat with tons of unused numbers, some specific to web servers AFAICS, does not make sense.

Tangent: Why is this webserver-specific crap in kernel_stat anyway? It looks like there should be a separate per-cpu structure for webserver statistics.

Jeff Garzik      | Andre the Giant has a posse.
Building 1024    |
MandrakeSoft     |

How many of them are applied to 90% of kernels running out there? How many of them will get merged eventually? How many of them makes modifications to the kernel that are visible to userspace in any possibly configuration of the kernel?

Indeed, I also said this in my first email :)

Andrea

I dont see why Tux should be merged. If we have people achieving the same performance in user space with the core facilities tux added to the kernel like the better irq/sendfile stuff why bother merging tux ?

1. We have khttpd, which should be replaced by something faster.
2. Tux makes a nice example.
3. Tux can be the fastest. If it isn't, it needs more work.

Toward the end of the X15 discussion, Ingo Molnar mentioned something he'd not implemented yet. I don't recall exactly, but for sure Tux hasn't run out of optimizations to do.

Also the kernel-CGI feature has not been used in benchmarks. Tux has been running user code.

IMHO the Tux server could be renamed "khttpd" and dropped in with whatever is needed to be compatible for existing setups.

I never had doubt that you could do the same in userspace using the zorocopy functionality (see the old threads with Jeff V. Merkey and netware when I was saying you don't need it in kernel) so in theory I totally agree since the first place...

... but in practice x15 is nor open source nor free software and I'd prefer to have an open choice. I wouldn't even think to merge tux if zope, apache, thttpd would run as fast as tux of course.

Andrea

in the meantime we solve this issue I released a new o_direct patch and a 2.4.6pre3aa1 with O_DIRECT set like in the patches I sent to Linus at the start of the thread. As soon as we take a definitive decision I will update them. (in the meantime alpha users will at least be allowed again to use O_SYNC with o_direct support applied ;)

Andrea

It would? I thought it would be ignored at minimum.

Damnit, I guess we'll have to move it after all. How completely irritating.

r~