Skip to contentSkip to conversation

ftrace: add an fsync tracer

PATCH In review Arjan van de Ven 9 participants 26 messages Nov 5, 2008 – Nov 7, 2008

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 ftrace: add an fsync tracer

Quoted 1 line

From 63c1b869d94eb31a98015af09fb24e22151f2f00 Mon Sep 17 00:00:00 2001

From: Arjan van de Ven <[email protected]>
Date: Tue, 4 Nov 2008 21:08:11 -0800
Subject: [PATCH] ftrace: add an fsync tracer

fsync() (and its cousin, fdatasync()) are important chokepoints in the kernel as they imply very expensive operations, both in terms of filesystem operations (ext3 writes back its entire journal) as well as the block layer (fsync() implies sending a cache flushing barrier to the SATA/SCSI disk).

This tracer makes a log of which application calls fsync() on which file, so that developers and others interested in finding these choke points can locate them and fix them in the apps that call this function.

---
 fs/sync.c                  |    4 +
 include/trace/fs.h         |   11 +++
 kernel/trace/Kconfig       |    9 +++
 kernel/trace/Makefile      |    1 +
 kernel/trace/trace.h       |    1 +
 kernel/trace/trace_fsync.c |  165 ++++++++++++++++++++++++++++++++++++++++++++
 6 files changed, 191 insertions(+), 0 deletions(-)
 create mode 100644 include/trace/fs.h
 create mode 100644 kernel/trace/trace_fsync.c
fs/sync.c +4 -0
index 2967562..dd5641b 100644
@@ -14,6 +14,8 @@
 #include <linux/quotaops.h>
 #include <linux/buffer_head.h>
 
+#include <trace/fs.h>
+
 #define VALID_FLAGS (SYNC_FILE_RANGE_WAIT_BEFORE|SYNC_FILE_RANGE_WRITE| \
 			SYNC_FILE_RANGE_WAIT_AFTER)
 
@@ -81,6 +83,8 @@ long do_fsync(struct file *file, int datasync)
 	int err;
 	struct address_space *mapping = file->f_mapping;
 
+	trace_do_fsync(file, datasync);
+
 	if (!file->f_op || !file->f_op->fsync) {
 		/* Why?  We can still call filemap_fdatawrite */
 		ret = -EINVAL;
include/trace/fs.h new file +11 -0
new file mode 100644
index 0000000..9d964f1
@@ -0,0 +1,11 @@
+#ifndef _TRACE_FS_H
+#define _TRACE_FS_H
+
+#include <linux/fs.h>
+#include <linux/tracepoint.h>
+
+DEFINE_TRACE(do_fsync,
+	TPPROTO(struct file *file, int datasync),
+		TPARGS(file, datasync));
+
+#endif
kernel/trace/Kconfig +9 -0
index 33dbefd..42d60c5 100644
@@ -119,6 +119,15 @@ config CONTEXT_SWITCH_TRACER
 	  This tracer gets called from the context switch and records
 	  all switching of tasks.
 
+config FSYNC_TRACER
+	bool "Trace fsync() calls"
+	depends on DEBUG_KERNEL
+	select TRACING
+	select MARKERS
+	help
+	  This tracer records fsync() and fdatasync() calls. These calls
+	  are key reasons for IO related system latency.
+
 config BOOT_TRACER
 	bool "Trace boot initcalls"
 	depends on DEBUG_KERNEL
kernel/trace/Makefile +1 -0
index c8228b1..292f39d 100644
@@ -15,6 +15,7 @@ obj-$(CONFIG_RING_BUFFER) += ring_buffer.o
 
 obj-$(CONFIG_TRACING) += trace.o
 obj-$(CONFIG_CONTEXT_SWITCH_TRACER) += trace_sched_switch.o
+obj-$(CONFIG_FSYNC_TRACER) += trace_fsync.o
 obj-$(CONFIG_SYSPROF_TRACER) += trace_sysprof.o
 obj-$(CONFIG_FUNCTION_TRACER) += trace_functions.o
 obj-$(CONFIG_IRQSOFF_TRACER) += trace_irqsoff.o
kernel/trace/trace.h +1 -0
index 8465ad0..dd0fef4 100644
@@ -22,6 +22,7 @@ enum trace_type {
 	TRACE_MMIO_RW,
 	TRACE_MMIO_MAP,
 	TRACE_BOOT,
+	TRACE_FSYNC,
 
 	__TRACE_LAST_TYPE
 };
kernel/trace/trace_fsync.c new file +165 -0

165 lines not on this page · raw message

Signature · 8 lines
1.6.0.3



-- 
Arjan van de Ven 	Intel Open Source Technology Centre
For development, discussion and tips for power savings, 
visit http://www.lesswatts.org
#1 lore raw
Quoted 32 lines · Arjan van de Ven
kernel/trace/trace_fsync.c new file +29 -0
@@ -0,0 +1,165 @@
+/*
+ * trace fsync calls
+ * Copyright (C) 2008 Intel Corporation
+ *
+ * Based extensively on trace_sched_switch.c
+ * Copyright (C) 2007 Steven Rostedt <[email protected]>
+ *
+ */
+#include <linux/module.h>
+#include <linux/fs.h>
+#include <linux/debugfs.h>
+#include <linux/kallsyms.h>
+#include <linux/uaccess.h>
+#include <linux/ftrace.h>
+#include <trace/fs.h>
+
+#include "trace.h"
+
+static struct trace_array	*ctx_trace;
+static int __read_mostly	tracer_enabled;
+static atomic_t			fsync_ref;
+
+static void
+probe_do_fsync(struct file *file, int datasync)
+{
+	char *buffer;
+	char *err;
+
+	if (!atomic_read(&fsync_ref))

kernel/trace/trace_fsync.c line 39

Quoted 10 lines
kernel/trace/trace_fsync.c new file +10 -0
+		return;
+
+	if (!tracer_enabled)
+		return;
+
+	buffer = kmalloc(4096, GFP_KERNEL);
+	if (!buffer)
+		return;
+
+	memset(buffer, 0, 4096);

kzalloc?

kernel/trace/trace_fsync.c line 46

Quoted 7 lines
kernel/trace/trace_fsync.c new file +7 -0
+	err = d_path(&file->f_path, buffer, 4096);
+
+	if (IS_ERR(err))
+		goto out;
+
+	ftrace_printk("Process %s is calling fsync on %s\n",
+			current->comm, err);

err? not buffer?

Quoted 6 lines
kernel/trace/trace_fsync.c new file +5 -0
+
+out:
+	kfree(buffer);
+}
+
(...)
#2 lore raw
Quoted 37 lines · On Wed, 5 Nov 2008 20:43:31 +0100 Marcin Slusarz <[email protected]>…
Quoted 36 lines · Arjan van de Ven
kernel/trace/trace_fsync.c new file +33 -0
@@ -0,0 +1,165 @@
+/*
+ * trace fsync calls
+ * Copyright (C) 2008 Intel Corporation
+ *
+ * Based extensively on trace_sched_switch.c
+ * Copyright (C) 2007 Steven Rostedt <[email protected]>
+ *
+ */
+#include <linux/module.h>
+#include <linux/fs.h>
+#include <linux/debugfs.h>
+#include <linux/kallsyms.h>
+#include <linux/uaccess.h>
+#include <linux/ftrace.h>
+#include <trace/fs.h>
+
+#include "trace.h"
+
+static struct trace_array	*ctx_trace;
+static int __read_mostly	tracer_enabled;
+static atomic_t			fsync_ref;
+
+static void
+probe_do_fsync(struct file *file, int datasync)
+{
+	char *buffer;
+	char *err;
+
+	if (!atomic_read(&fsync_ref))
+		return;
+
+	if (!tracer_enabled)
+		return;
Quoted 8 lines

kernel/trace/trace_fsync.c line 39

Quoted 6 lines
kernel/trace/trace_fsync.c new file +6 -0
+
+	buffer = kmalloc(4096, GFP_KERNEL);
+	if (!buffer)
+		return;
+
+	memset(buffer, 0, 4096);

kzalloc?

could do I suppose

Quoted 10 lines

kernel/trace/trace_fsync.c line 46

Quoted 8 lines
kernel/trace/trace_fsync.c new file +7 -0
+	err = d_path(&file->f_path, buffer, 4096);
+
+	if (IS_ERR(err))
+		goto out;
+
+	ftrace_printk("Process %s is calling fsync on %s\n",
+			current->comm, err);

err? not buffer?

correct, not buffer

it won't work if you use buffer.

Signature · 3 lines
Arjan van de Ven 	Intel Open Source Technology Centre
For development, discussion and tips for power savings, 
visit http://www.lesswatts.org
#3 lore raw
Quoted 3 lines · Arjan van de Ven
Quoted 3 lines

kernel/trace/trace_fsync.c line 40

Quoted 3 lines
kernel/trace/trace_fsync.c new file +2 -0
+	buffer = kmalloc(4096, GFP_KERNEL);
 
+	err = d_path(&file->f_path, buffer, 4096);

hm, magic constants.

Quoted 11 lines
Quoted 7 lines

kernel/trace/trace_fsync.c line 46

Quoted 5 lines
kernel/trace/trace_fsync.c new file +5 -0
+	if (IS_ERR(err))
+		goto out;
+
+	ftrace_printk("Process %s is calling fsync on %s\n",
+			current->comm, err);

err? not buffer?

correct, not buffer

it won't work if you use buffer.

please use some other name than 'err' - since "error" is just a property of the path string pointer you get back from do_path().

Ingo

#4 lore raw
In reply to Arjan van de Ven
Quoted 5 lines · Arjan van de Ven

From 63c1b869d94eb31a98015af09fb24e22151f2f00 Mon Sep 17 00:00:00 2001
From: Arjan van de Ven <[email protected]>
Date: Tue, 4 Nov 2008 21:08:11 -0800
Subject: [PATCH] ftrace: add an fsync tracer

Quoted 8 lines

fsync() (and its cousin, fdatasync()) are important chokepoints in the kernel as they imply very expensive operations, both in terms of filesystem operations (ext3 writes back its entire journal) as well as the block layer (fsync() implies sending a cache flushing barrier to the SATA/SCSI disk).

This tracer makes a log of which application calls fsync() on which file, so that developers and others interested in finding these choke points can locate them and fix them in the apps that call this function.

Sorry, but I have to object to such single purpose tracers..

If we go this way we'll end up with a gazillion little tracers, non of which are really useful.

Please work on getting something like a syscall tracer, or lttng like event tracer.

Both of those should be able to find such badness, but are also able to diagnose many more issues.

Also, sure fsync() is expensive, but you make it sound like its a bad thing. Lets not teach people to ignore data integrity just because ext3 is such a massive piece of shit.

#5 lore raw

2008/11/6 Peter Zijlstra <[email protected]>:

Quoted 5 lines

Please work on getting something like a syscall tracer, or lttng like event tracer.

Both of those should be able to find such badness, but are also able to diagnose many more issues.

BTW. Why not adding a kind of tracer_ctrl callback on the struct_tracer? This way we could add some custom
options on the iter_ctrl file that could be transmitted to the current_tracer.

For example considering a syscall tracer.
By default if we don't write anything on iter_ctrl, it traces all syscalls. But if we do:
echo fsync > iter_ctrl
Only the fsync call will be traced (unused tracepoint would be disabled).

A tracer just have to implement this new callback and transmit the options it support.

#6 lore raw
In reply to Peter Zijlstra
Quoted 17 lines · On Thu, 06 Nov 2008 13:55:38 +0100 Peter Zijlstra <[email protected]> wrote:
Quoted 11 lines · Arjan van de Ven

From 63c1b869d94eb31a98015af09fb24e22151f2f00 Mon Sep 17 00:00:00 2001 From: Arjan van de Ven <[email protected]>
Date: Tue, 4 Nov 2008 21:08:11 -0800
Subject: [PATCH] ftrace: add an fsync tracer

fsync() (and its cousin, fdatasync()) are important chokepoints in the kernel as they imply very expensive operations, both in terms of filesystem operations (ext3 writes back its entire journal) as well as the block layer (fsync() implies sending a cache flushing barrier to the SATA/SCSI disk).

Quoted 4 lines

This tracer makes a log of which application calls fsync() on which file, so that developers and others interested in finding these choke points can locate them and fix them in the apps that call this function.

Quoted 4 lines

Sorry, but I have to object to such single purpose tracers..

If we go this way we'll end up with a gazillion little tracers, non of which are really useful.

If we go this way we'll end up with a bunch of little tracers, all of which will be useful in their area, and people can also make "super tracers" out of the useful trace points.

Quoted 2 lines

Please work on getting something like a syscall tracer,

a syscall tracer will exactly not tell you which file(name) was being fsync()'d which was the whole point.
LatencyTOP already KNOWS that fsync is the problem. What it doesn't know is which file is being fsync()d.

fsync is a problem when used incorrectly, not just for ext3 but also due to barriers. That's why it's important to be able to find who calls it when it impacts interactive performance.

Signature · 3 lines
Arjan van de Ven 	Intel Open Source Technology Centre
For development, discussion and tips for power savings, 
visit http://www.lesswatts.org
#7 lore raw
Quoted 20 lines · Arjan van de Ven
Quoted 17 lines · On Thu, 06 Nov 2008 13:55:38 +0100 Peter Zijlstra <[email protected]> wrote:
Quoted 15 lines · Arjan van de Ven

From 63c1b869d94eb31a98015af09fb24e22151f2f00 Mon Sep 17 00:00:00 2001 From: Arjan van de Ven <[email protected]>
Date: Tue, 4 Nov 2008 21:08:11 -0800
Subject: [PATCH] ftrace: add an fsync tracer

fsync() (and its cousin, fdatasync()) are important chokepoints in the kernel as they imply very expensive operations, both in terms of filesystem operations (ext3 writes back its entire journal) as well as the block layer (fsync() implies sending a cache flushing barrier to the SATA/SCSI disk).

This tracer makes a log of which application calls fsync() on which file, so that developers and others interested in finding these choke points can locate them and fix them in the apps that call this function.

Quoted 8 lines
Quoted 4 lines

Sorry, but I have to object to such single purpose tracers..

If we go this way we'll end up with a gazillion little tracers, non of which are really useful.

If we go this way we'll end up with a bunch of little tracers, all of which will be useful in their area, and people can also make "super tracers" out of the useful trace points.

I don't think:

# cat available_tracers | wc -l
500

will do much good for people.

Also, I don't think do_fsync() is the right place to catch what you're trying to catch.

Quoted 5 lines
Quoted 2 lines

Please work on getting something like a syscall tracer,

a syscall tracer will exactly not tell you which file(name) was being fsync()'d which was the whole point.

It will tell you the process and the fd, and when you have those two its a simple step to find the actual file.

Quoted 6 lines

LatencyTOP already KNOWS that fsync is the problem. What it doesn't know is which file is being fsync()d.

fsync is a problem when used incorrectly, not just for ext3 but also due to barriers. That's why it's important to be able to find who calls it when it impacts interactive performance.

Which suggests you want a tracer that gives more information about who generates barriers, not specifically fsync().

#8 lore raw
Quoted 5 lines · On Thu, 06 Nov 2008 15:19:01 +0100 Peter Zijlstra <[email protected]> wrote:
Quoted 2 lines

a syscall tracer will exactly not tell you which file(name) was being fsync()'d which was the whole point.

It will tell you the process and the fd, and when you have those two its a simple step to find the actual file.

actually process+fd is absolutely useless; the typical useage is

fd = open(file)
write(fd, <> )
fsync(fd);
close(fd);

by the time userland gets the data the fd is closed. And heck, even the program may have exited.
Really, the fd number is only useful for the program itself, not for any outside part, and especially, later in time.

Quoted 10 lines
Quoted 7 lines

LatencyTOP already KNOWS that fsync is the problem. What it doesn't know is which file is being fsync()d.

fsync is a problem when used incorrectly, not just for ext3 but also due to barriers. That's why it's important to be able to find who calls it when it impacts interactive performance.

Which suggests you want a tracer that gives more information about who generates barriers, not specifically fsync().

that would be a fine second tracer. because the filesystem part of it is also expensive, and you can diss ext3 all you want, it is reality for 99% of the people...

(and I suspect that at the barrier level it'll be really hard to get to a filename)

Signature · 3 lines
Arjan van de Ven 	Intel Open Source Technology Centre
For development, discussion and tips for power savings, 
visit http://www.lesswatts.org
#9 lore raw
Quoted 16 lines · Arjan van de Ven
Quoted 5 lines · On Thu, 06 Nov 2008 15:19:01 +0100 Peter Zijlstra <[email protected]> wrote:
Quoted 2 lines

a syscall tracer will exactly not tell you which file(name) was being fsync()'d which was the whole point.

It will tell you the process and the fd, and when you have those two its a simple step to find the actual file.

actually process+fd is absolutely useless; the typical useage is

fd = open(file)
write(fd, <> )
fsync(fd);
close(fd);

Quoted 4 lines

by the time userland gets the data the fd is closed. And heck, even the program may have exited.
Really, the fd number is only useful for the program itself, not for any outside part, and especially, later in time.

The syscall tracer will also have told you about that open.

Anyway, do_fsync() doesn't catch all sync actions (although I suspect it catches most). We still have the mythical sync_file_range() that Andrew still wants a real program to use.

And then there are things like sync and umount that do syncs too. But I suspect you might not be interested in those.

Quoted 8 lines
Quoted 8 lines
Quoted 7 lines

LatencyTOP already KNOWS that fsync is the problem. What it doesn't know is which file is being fsync()d.

fsync is a problem when used incorrectly, not just for ext3 but also due to barriers. That's why it's important to be able to find who calls it when it impacts interactive performance.

Quoted 6 lines
Quoted 2 lines

Which suggests you want a tracer that gives more information about who generates barriers, not specifically fsync().

that would be a fine second tracer. because the filesystem part of it is also expensive, and you can diss ext3 all you want, it is reality for 99% of the people...

Lets hope btrfs will fix that quickly. fsync() causing latencies for anyone else besides tasks interested in that file is utterly unacceptable.

Quoted 2 lines

(and I suspect that at the barrier level it'll be really hard to get to a filename)

I suspect you might be right, but that's not a reason not to try ;-)

#10 lore raw
Quoted 17 lines · On Thu, 06 Nov 2008 15:50:52 +0100 Peter Zijlstra <[email protected]> wrote:
Quoted 16 lines · Arjan van de Ven
Quoted 5 lines · On Thu, 06 Nov 2008 15:19:01 +0100 Peter Zijlstra <[email protected]> wrote:
Quoted 2 lines

a syscall tracer will exactly not tell you which file(name) was being fsync()'d which was the whole point.

It will tell you the process and the fd, and when you have those two its a simple step to find the actual file.

actually process+fd is absolutely useless; the typical useage is

fd = open(file)
write(fd, <> )
fsync(fd);
close(fd);

Quoted 6 lines
Quoted 4 lines

by the time userland gets the data the fd is closed. And heck, even the program may have exited.
Really, the fd number is only useful for the program itself, not for any outside part, and especially, later in time.

The syscall tracer will also have told you about that open.

ok so now you're turning a simple problem into "also track all open, openat, dup, dup2, close, unix domain fs passing etc". That doesn't sound like an improvement to me.
And even the "open/fd" family isn't enough, because the open will be relative pathnames, so I now also need to second guess the PWD, and then follow a chain of "chdir" calls as well and then resolve the ".."'s in the path. And follow symlinks that may no longer be the same symlink it used to be. Etc. Etc.

No sorry, this really is not a viable approach.

And that goes beyond the fsync tracer. Anybody who thinks that syscall tracing is the right approach needs to seriously think about it; it really looks like the same problem as with audit... it started out with a bad hack that did it at the syscall level but it was likewise shown to just not be the right level. I am rather certain that the same will be shown for tracing.

Quoted 4 lines

Anyway, do_fsync() doesn't catch all sync actions (although I suspect it catches most). We still have the mythical sync_file_range() that Andrew still wants a real program to use.

a normal sync() I don't need any help for.. it has no filename and I already know who's calling it with what latency.

Quoted 3 lines

And then there are things like sync and umount that do syncs too. But I suspect you might not be interested in those.

I already know all I need about those.... again it's the fsync() that is specific, and I need to know the specifics to help the user figure out what is going on.
(and in case you worry that there's not going to be a user for this tracer, I already wrote the latencytop side of the code for this one, so I know I can use the current format)

Signature · 3 lines
Arjan van de Ven 	Intel Open Source Technology Centre
For development, discussion and tips for power savings, 
visit http://www.lesswatts.org
maintainer
#11 lore raw
Quoted 12 lines · Arjan van de Ven
Quoted 2 lines

The syscall tracer will also have told you about that open.

ok so now you're turning a simple problem into "also track all open, openat, dup, dup2, close, unix domain fs passing etc". That doesn't sound like an improvement to me.
And even the "open/fd" family isn't enough, because the open will be relative pathnames, so I now also need to second guess the PWD, and then follow a chain of "chdir" calls as well and then resolve the ".."'s in the path. And follow symlinks that may no longer be the same symlink it used to be. Etc. Etc.

Quoted 8 lines

No sorry, this really is not a viable approach.

And that goes beyond the fsync tracer. Anybody who thinks that syscall tracing is the right approach needs to seriously think about it; it really looks like the same problem as with audit... it started out with a bad hack that did it at the syscall level but it was likewise shown to just not be the right level. I am rather certain that the same will be shown for tracing.

What I would like is a single "event tracer" that does all of this. But add the ability to start and stop different events. We could start with this tracer, and then add more events to this tracer. Have a separate file to show which events are available.

This will keep the 'available_tracers' file small, but it would be fine if the 'events' file grew large.

-- Steve

#12 lore raw
Quoted 8 lines · On Thu, 6 Nov 2008 10:34:50 -0500 (EST) Steven Rostedt <[email protected]>…

What I would like is a single "event tracer" that does all of this. But add the ability to start and stop different events. We could start with this tracer, and then add more events to this tracer. Have a separate file to show which events are available.

This will keep the 'available_tracers' file small, but it would be fine if the 'events' file grew large.

well 2 ounces versus one liter ;)

what is the real need is
1) Have a trace point in the source
2) Associate a "formatting function" with that point
(which basically transforms the trace parameters to, say, a string)
3) A way to turn the trace point on/off.

right now this is what the fsync tracer does, because it's the only way to do such a thing.
I don't care how to otherwise do it ....

Signature · 3 lines
Arjan van de Ven 	Intel Open Source Technology Centre
For development, discussion and tips for power savings, 
visit http://www.lesswatts.org
#13 lore raw
Quoted 6 lines · Arjan van de Ven

[...]
what is the real need is
1) Have a trace point in the source
2) Associate a "formatting function" with that point
(which basically transforms the trace parameters to, say, a string)
3) A way to turn the trace point on/off.

For 1 and 2, it may be worth considering a plain trace_mark() in do_sync(). The complication that makes this uglier than a one-liner is d_path()'s buffer and error handling.

    {
    char *buffer = kzalloc (4096, GFP_KERNEL);
    trace_mark(fsync, "Process %s is calling fsync on %s\n",
                       current->comm,
                       ({char *err = d_path (...);
                         IS_ERR(err) ? "?" : err;}));
    kfree (buffer);
    }

With a bit of extension on the marker front, the allocation could be made conditional on the marker being enabled.

For 3, the kernel could merge a backend that connects arbitrary markers to an ftrace (or whatever) buffer. Several compact prototypes for the latter exist.

- FChE

#14 lore raw
Quoted 22 lines · Frank Ch. Eigler
Quoted 6 lines · Arjan van de Ven

[...]
what is the real need is
1) Have a trace point in the source
2) Associate a "formatting function" with that point
(which basically transforms the trace parameters to, say, a string)
3) A way to turn the trace point on/off.

For 1 and 2, it may be worth considering a plain trace_mark() in do_sync(). The complication that makes this uglier than a one-liner is d_path()'s buffer and error handling.

    {
    char *buffer = kzalloc (4096, GFP_KERNEL);
    trace_mark(fsync, "Process %s is calling fsync on %s\n",
                       current->comm,
                       ({char *err = d_path (...);
                         IS_ERR(err) ? "?" : err;}));
    kfree (buffer);
    }
Quoted 7 lines

With a bit of extension on the marker front, the allocation could be made conditional on the marker being enabled.

For 3, the kernel could merge a backend that connects arbitrary markers to an ftrace (or whatever) buffer. Several compact prototypes for the latter exist.

I prefer we keep using trace points but do what jason has been proposing for a while, which is add a format and arg list to the trace point definition.

Something like

DEFINE_TRACE_FMT(sched_switch,
TPPROTO(struct rq *rq, struct task_struct *prev,
struct task_struct *next),
TPARGS(rq, prev, next),
TPFMT("%d to %d\n", prev->pid, next->pid));

Which would be similar to attaching a trace_mark() to the trace point and can in these cases save a lot of lines of code.

Both lttng and the ftrace event tracer can use these default text strings.

#15 lore raw

Hi -

Quoted 7 lines · Peter Zijlstra

[...]
I prefer we keep using trace points but do what jason has been proposing for a while, which is add a format and arg list to the trace point definition.

Something like

Quoted 8 lines

DEFINE_TRACE_FMT(sched_switch,
TPPROTO(struct rq *rq, struct task_struct *prev,
struct task_struct *next),
TPARGS(rq, prev, next),
TPFMT("%d to %d\n", prev->pid, next->pid));

Which would be similar to attaching a trace_mark() to the trace point and can in these cases save a lot of lines of code.

Can you explain how this would save any lines of code at all compared to the trace_mark() example? Both cases still need a bit of ~identical additional code to couple the markers (specified whichever way) to a trace buffer. Your version has the tracepoint machinery too, which is strictly additional. Where's the savings?

- FChE

#16 lore raw
In reply to Peter Zijlstra
Quoted 44 lines · Peter Zijlstra
Quoted 22 lines · Frank Ch. Eigler
Quoted 6 lines · Arjan van de Ven

[...]
what is the real need is
1) Have a trace point in the source
2) Associate a "formatting function" with that point
(which basically transforms the trace parameters to, say, a string)
3) A way to turn the trace point on/off.

For 1 and 2, it may be worth considering a plain trace_mark() in do_sync(). The complication that makes this uglier than a one-liner is d_path()'s buffer and error handling.

    {
    char *buffer = kzalloc (4096, GFP_KERNEL);
    trace_mark(fsync, "Process %s is calling fsync on %s\n",
                       current->comm,
                       ({char *err = d_path (...);
                         IS_ERR(err) ? "?" : err;}));
    kfree (buffer);
    }
Quoted 7 lines

With a bit of extension on the marker front, the allocation could be made conditional on the marker being enabled.

For 3, the kernel could merge a backend that connects arbitrary markers to an ftrace (or whatever) buffer. Several compact prototypes for the latter exist.

I prefer we keep using trace points but do what jason has been proposing for a while, which is add a format and arg list to the trace point definition.

Something like

DEFINE_TRACE_FMT(sched_switch,
TPPROTO(struct rq *rq, struct task_struct *prev,
struct task_struct *next),
TPARGS(rq, prev, next),
TPFMT("%d to %d\n", prev->pid, next->pid));

Quoted 6 lines

Which would be similar to attaching a trace_mark() to the trace point and can in these cases save a lot of lines of code.

Both lttng and the ftrace event tracer can use these default text strings.

indeed...done this way I believe marker registration could then be done directly with the tracepoint itself. That is, callers that want the tracepoint interface register directly with the tracepoint and get called as before. However, callers that want the marker interface register directly with the tracepoint and get passed the associated marker string and arguments. Then, we could consolidate kernel/marker.c into kernel/tracepoint.c, which would simlify things, and reduce code duplication.

thanks,

-Jason

#17 lore raw
In reply to Peter Zijlstra
Quoted 44 lines · * Peter Zijlstra ([email protected]) wrote:
Quoted 22 lines · Frank Ch. Eigler
Quoted 6 lines · Arjan van de Ven

[...]
what is the real need is
1) Have a trace point in the source
2) Associate a "formatting function" with that point
(which basically transforms the trace parameters to, say, a string)
3) A way to turn the trace point on/off.

For 1 and 2, it may be worth considering a plain trace_mark() in do_sync(). The complication that makes this uglier than a one-liner is d_path()'s buffer and error handling.

    {
    char *buffer = kzalloc (4096, GFP_KERNEL);
    trace_mark(fsync, "Process %s is calling fsync on %s\n",
                       current->comm,
                       ({char *err = d_path (...);
                         IS_ERR(err) ? "?" : err;}));
    kfree (buffer);
    }
Quoted 7 lines

With a bit of extension on the marker front, the allocation could be made conditional on the marker being enabled.

For 3, the kernel could merge a backend that connects arbitrary markers to an ftrace (or whatever) buffer. Several compact prototypes for the latter exist.

I prefer we keep using trace points but do what jason has been proposing for a while, which is add a format and arg list to the trace point definition.

Something like

DEFINE_TRACE_FMT(sched_switch,
TPPROTO(struct rq *rq, struct task_struct *prev,
struct task_struct *next),
TPARGS(rq, prev, next),
TPFMT("%d to %d\n", prev->pid, next->pid));

Quoted 7 lines

Which would be similar to attaching a trace_mark() to the trace point and can in these cases save a lot of lines of code.

Both lttng and the ftrace event tracer can use these default text strings.

Argh. No, please. Doing this would end up exposing the inner kernel API (the tracepoints) directly to userspace.

I've done an addition to the markers which does approximately the same. It's in the -lttng tree now, I should post it soon.

A new marker type, trace_mark_tp() also takes a tracepoint name as parameter. When the tracepoint is enabled, it also enables the associated tracepoint. And it makes sure tracepoints are not exposed to userspace.

Mathieu

Signature · 2 lines
Mathieu Desnoyers
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68
#18 lore raw

Hi -

Quoted 9 lines · Mathieu Desnoyers

[...]

Quoted 8 lines

Something like

DEFINE_TRACE_FMT(sched_switch,
TPPROTO(struct rq *rq, struct task_struct *prev,
struct task_struct *next),
TPARGS(rq, prev, next),
TPFMT("%d to %d\n", prev->pid, next->pid));

Quoted 8 lines
Quoted 5 lines

Which would be similar to attaching a trace_mark() to the trace point and can in these cases save a lot of lines of code.

Both lttng and the ftrace event tracer can use these default text strings.

Argh. No, please. Doing this would end up exposing the inner kernel API (the tracepoints) directly to userspace. [...]

Arjan asked for something to send formatted strings to tracing buffers that are consumed by user space. Whether internally that is done by markers, or by tracepoints converted to markers, or by tracepoints with custom ftrace_printf-y code, is indistinguishable to the user.

Can you be specific in what way any of that is exposing an "API directly to userspace"?

- FChE

#19 lore raw
Quoted 27 lines · * Frank Ch. Eigler ([email protected]) wrote:

Hi -

Quoted 9 lines · Mathieu Desnoyers

[...]

Quoted 8 lines

Something like

DEFINE_TRACE_FMT(sched_switch,
TPPROTO(struct rq *rq, struct task_struct *prev,
struct task_struct *next),
TPARGS(rq, prev, next),
TPFMT("%d to %d\n", prev->pid, next->pid));

Quoted 8 lines
Quoted 5 lines

Which would be similar to attaching a trace_mark() to the trace point and can in these cases save a lot of lines of code.

Both lttng and the ftrace event tracer can use these default text strings.

Argh. No, please. Doing this would end up exposing the inner kernel API (the tracepoints) directly to userspace. [...]

Arjan asked for something to send formatted strings to tracing buffers that are consumed by user space. Whether internally that is done by markers, or by tracepoints converted to markers, or by tracepoints with custom ftrace_printf-y code, is indistinguishable to the user.

Quoted 4 lines

Can you be specific in what way any of that is exposing an "API directly to userspace"?

- FChE

The tracepoint declaration is linked to the kernel code that is meant to be traced. Given this code will change over time, it will be more difficult to perform those changes if it is in any way coupled with format strings exported to userspace via traced.

This is where using the markers as an indirection layer plays its role.

I also have a patch with "redux" markers, where I separate the marker "declaration" from the marker "call". By doing this, I remove the need for cascading function calls, which is more efficient.

Mathieu

Signature · 2 lines
Mathieu Desnoyers
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68
#20 lore raw

Hi -

Quoted 6 lines · Mathieu Desnoyers

[...]

Quoted 5 lines

Arjan asked for something to send formatted strings to tracing buffers that are consumed by user space. Whether internally that is done by markers, or by tracepoints converted to markers, or by tracepoints with custom ftrace_printf-y code, is indistinguishable to the user.

Quoted 7 lines
Quoted 2 lines

Can you be specific in what way any of that is exposing an "API directly to userspace"?

The tracepoint declaration is linked to the kernel code that is meant to be traced. Given this code will change over time, it will be more difficult to perform those changes if it is in any way coupled with format strings exported to userspace via traced. [...]

The userspace interface to all the various tracing widgets consists of the textual formatted trace data stream. ** IF ** these streams are to represent a long-term interface, can you elaborate why you believe it would be harder to generate identical text in the future with markers rather than another technique with more internal layers? Can you work out an example?

(BTW, is "traced" a typo?)

And as to that ** IF **: is there actually consensus that the format of tracing data generated by e.g. ftrace engines are supposed to be preserved into the future versions, just as if they were a user-space interface like /proc files? (That's certainly not my impression.)

- FChE

#21 lore raw
Quoted 13 lines · On Thu, 6 Nov 2008 18:25:06 -0500 "Frank Ch. Eigler" <[email protected]> wrote:

Hi -

Quoted 10 lines · Mathieu Desnoyers

[...]

Quoted 8 lines

Arjan asked for something to send formatted strings to tracing buffers that are consumed by user space. Whether internally that is done by markers, or by tracepoints converted to markers, or by tracepoints with custom ftrace_printf-y code, is indistinguishable to the user.

Can you be specific in what way any of that is exposing an "API directly to userspace"?

Quoted 8 lines
Quoted 4 lines

The tracepoint declaration is linked to the kernel code that is meant to be traced. Given this code will change over time, it will be more difficult to perform those changes if it is in any way coupled with format strings exported to userspace via traced. [...]

The userspace interface to all the various tracing widgets consists of the textual formatted trace data stream. ** IF ** these streams are to represent a long-term interface, can you elaborate why you believe

tracepoints are not and cannot be a stable interface, and users of these points, preferably scripts that are part of the kernel source, need to learn to adapt to a changing environment.
At least if we make the formatting function a *function*, we can even do very minor things to deal with some changes, but even then.

And this is ok. You're talking about deep internals, and for tracepoints to be useful they HAVE to be about these internals.

Signature · 3 lines
Arjan van de Ven 	Intel Open Source Technology Centre
For development, discussion and tips for power savings, 
visit http://www.lesswatts.org
#22 lore raw
In reply to Frank Ch. Eigler

Hi Frank,

Quoted 12 lines · * Frank Ch. Eigler ([email protected]) wrote:

Hi -

Quoted 9 lines · Mathieu Desnoyers

[...]

Quoted 7 lines

Arjan asked for something to send formatted strings to tracing buffers that are consumed by user space. Whether internally that is done by markers, or by tracepoints converted to markers, or by tracepoints with custom ftrace_printf-y code, is indistinguishable to the user.

Can you be specific in what way any of that is exposing an "API directly to userspace"?

Quoted 7 lines
Quoted 4 lines

The tracepoint declaration is linked to the kernel code that is meant to be traced. Given this code will change over time, it will be more difficult to perform those changes if it is in any way coupled with format strings exported to userspace via traced. [...]

The userspace interface to all the various tracing widgets consists of the textual formatted trace data stream.

Oh ? Really ? I thought a binary data stream wasn't out of the question ? Actually, having both ascii and binary output seems like a good way to do it.

Quoted 6 lines

** IF ** these streams are
to represent a long-term interface, can you elaborate why you believe it would be harder to generate identical text in the future with markers rather than another technique with more internal layers? Can you work out an example?

Hum.. I never said it would be harder to generate the trace data. I said it would be harder for kernel developers to change kernel code because they would have to change (and understand the usefulness down to the analysis tools) all the subtile tracepoint fields exported to userspace tools. I am not willing to push tracepoints in the kernel as a supplementary maintainance burden for kernel developers. With the format string identifying subfield, this burden will fall on the kernel developers. If we put the field selection in a callback sitting in a different module, well, it won't compile for as long as nobody think it's important enough to be fixed according to new in-kernel APIs, but it won't break the standard kernel build, and this is a _must_.

Quoted 2 lines

(BTW, is "traced" a typo?)

Yep, I should have written "via a tracer".

Quoted 5 lines

And as to that ** IF **: is there actually consensus that the format of tracing data generated by e.g. ftrace engines are supposed to be preserved into the future versions, just as if they were a user-space interface like /proc files? (That's certainly not my impression.)

We aim at having a standardized buffer format at some point, and expect the consumer tools to be shipped with the kernel tree for most part. We also plan to put a version number on the trace buffer. I am merely relating the discussions that led to Steven Rostedt unified trace buffer mechanism. (I do not say it actually does everything that was agreed on, but we have to start somewhere)

If I can get my head out of the sand and start posting the implementations I have, we'll have more concrete stuff to discuss about :) Only two things left on my 30-ish things long todo list.. almost there ;)

Mathieu

Signature · 2 lines
Mathieu Desnoyers
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68
#23 lore raw
In reply to Frank Ch. Eigler
Quoted 11 lines · On Thu, 06 Nov 2008 15:19:48 -0500 [email protected] (Frank Ch. Eigler) wrote:
Quoted 6 lines · Arjan van de Ven

[...]
what is the real need is
1) Have a trace point in the source
2) Associate a "formatting function" with that point (which basically transforms the trace parameters to, say, a
string) 3) A way to turn the trace point on/off.

For 1 and 2, it may be worth considering a plain trace_mark() in do_sync(). The complication that makes this uglier than a one-liner

no why is that?

what you really need is to be able to provide a callback function pointer that will do the formatting,
or as Peter wants it, a format string.

(and we can easily make a format string that knows how to print a struct file, no big deal, I suspect that is a common thing actually)

Doing this like you propose is just too complex and too specialistic; the reality is that merely formatting the arguments of a trace point is the common case, and I suspect for 99.9% of the cases we can get away with a standard default formatting.

We should make THAT easy. Not complex or ugly. But easy.

And then if I or some tool wants to see a tracepoint, we have some standard way to enumerate them and turn individual ones on with their standard formatting ... and that's it.
no iffs or buts

Signature · 3 lines
Arjan van de Ven 	Intel Open Source Technology Centre
For development, discussion and tips for power savings, 
visit http://www.lesswatts.org
#24 lore raw

Hi -

Quoted 17 lines · Arjan van de Ven

[...]

Quoted 11 lines · [email protected] (Frank Ch. Eigler) wrote:
Quoted 6 lines · Arjan van de Ven

[...]
what is the real need is
1) Have a trace point in the source
2) Associate a "formatting function" with that point (which basically transforms the trace parameters to, say, a
string) 3) A way to turn the trace point on/off.

For 1 and 2, it may be worth considering a plain trace_mark() in do_sync(). The complication that makes this uglier than a one-liner

no why is that?

Quoted 7 lines

what you really need is to be able to provide a callback function pointer that will do the formatting, or as Peter wants it, a format string. [...]
Doing this like you propose is just too complex and too specialistic; the reality is that merely formatting the arguments of a trace point is the common case, and I suspect for 99.9% of the cases we can get away with a standard default formatting.

I don't understand what you're arguing against. If formatting strings learn to deal with struct files, then the marker approach would be even simpler, and the fsync "tracer" would consist of this single line:

   trace_mark (fsync, "process %s fsyncd file %pF\n", comm->execname, file);

Via some simple additional generic code (the generic marker->ftrace backend), that would turn into exactly the "merely formatting the arguments ... 99.9% of the cases" you're talking about.

- FChE

#25 lore raw
In reply to Peter Zijlstra
Quoted 17 lines · On Thu, 06 Nov 2008 13:55:38 +0100 Peter Zijlstra <[email protected]> wrote:
Quoted 11 lines · Arjan van de Ven

From 63c1b869d94eb31a98015af09fb24e22151f2f00 Mon Sep 17 00:00:00 2001 From: Arjan van de Ven <[email protected]>
Date: Tue, 4 Nov 2008 21:08:11 -0800
Subject: [PATCH] ftrace: add an fsync tracer

fsync() (and its cousin, fdatasync()) are important chokepoints in the kernel as they imply very expensive operations, both in terms of filesystem operations (ext3 writes back its entire journal) as well as the block layer (fsync() implies sending a cache flushing barrier to the SATA/SCSI disk).

Quoted 4 lines

This tracer makes a log of which application calls fsync() on which file, so that developers and others interested in finding these choke points can locate them and fix them in the apps that call this function.

Quoted 8 lines

Sorry, but I have to object to such single purpose tracers..

If we go this way we'll end up with a gazillion little tracers, non of which are really useful.

Please work on getting something like a syscall tracer, or lttng like event tracer.

btw a syscall tracer is not long term right, just like system call level auditing was the wrong thing: you don't have the real information of what's being worked on. having the trace points on the do_FOO() level is the right thing, and that's exactly what my patch does.