binary stream format for /proc/stat
Subsystems (2)
RFC PATCH 1/2 binary stream format for /proc/stat
Provides another format of /proc/stat for showing data in binary and compress sequence of Zeros to some extent.
Any other idea of formats ?
==
Quoted 1 line
From
dea9521e84701631e718884d40b5c96cdcd62d7dMon Sep 17 00:00:00 2001
From: KAMEZAWA Hiroyuki <[email protected]>
Date: Tue, 27 Mar 2012 13:40:54 +0900
Subject: [PATCH 1/2] proc: seq_file add binary stream support
/proc/stat shows all data in ascii format and it's easily readable. But considering a program which reads /proc/stat periodically, translations of ascii <-> binary happens a few times.
1. binrary -> ascii at showing seq_file
2. ascii -> binary for handling in the program.
This patch adds binstream format support in seq_file. In this format, all data will be shown after a header
uint32_t type # type of data, ZERO, STRING, ULLONG, LLONG.
uint32_t len # length of string or repeat of type.
If type = ULLONG and len=8, it means an array of 8 uulong values. If type = ZERO and len=4, it means 4 zeros are outputted.
For example, a sequence
cpu 123 456 789 0 0 0 0 111
cpu1
will be shows as
offset header values
0x00 STRING,3, "cpu" # aligned to 8bytes.
0x10 UULONG,3, 123, 456, 789 # array of uulong.
0x30 ZERO ,4 # zero has no data.
0x38 UULONG,1 111
0x48 STRING,4 "cpu0"(ZERO is for representing tons of 0s in /proc/stat...)
The file provider just needs to call seq_set_binstream(). Once seq_set_binstream() is called, following seq_put_decimal_(u)ll, seq_puts, seq_put_nl will generate appropriate outputs.
--- fs/seq_file.c | 143 ++++++++++++++++++++++++++++++++++++++++++++++ include/linux/seq_file.h | 36 ++++++++++++ 2 files changed, 179 insertions(+), 0 deletions(-)
fs/seq_file.c +143 -0
@@ -349,6 +349,114 @@ int seq_release(struct inode *inode, struct file *file) }EXPORT_SYMBOL (seq_release); + +static bool seq_binstream_continued (struct seq_file *m,int type) +{ +return m->binstream_head && m->binstream_head->type == type; +} + + +static int seq_binstream_align (struct seq_file *m) +{ +size_t pos; + + pos = m->count; + m->count =ALIGN (m->count,SEQ_BINSTREAM_ALIGNMENT ); +if (unlikely (m->count >= m->size)) { +seq_set_overflow (m); +return -1 ; + } +while (pos != m->count) { + *(m->buf + pos) =0 ; + ++pos; + } +return 0 ; +} + +static int seq_put_binstream_head (struct seq_file *m,int type) +{ +struct seq_binstream_head *head; +int payload =0 ; + + m->binstream_head =NULL ; + +if (seq_binstream_align (m)) +return -1 ; + +if (type ==SEQ_BINSTREAM_ULLONG || type ==SEQ_BINSTREAM_LLONG ) + payload =8 ; + +if (m->count +sizeof (*head) + payload >= m->size) { +seq_set_overflow (m); +return -1 ; + } + head = (struct seq_binstream_head *)(m->buf + m->count); + head->type = type; + head->len =1 ; + m->binstream_head = head; + m->count +=sizeof (*head); +return 0 ; +} + +static int seq_binstream_str_prepare (struct seq_file *m) +{ +if (!m->binstream) +return 0 ; +return seq_put_binstream_head (m,SEQ_BINSTREAM_STRING ); +} + +static void seq_binstream_str_end (struct seq_file *m,int len) +{ +if (m->binstream_head) + m->binstream_head->len = len; +} + +static int seq_put_binstream_zero (struct seq_file *m) +{ +if (seq_binstream_continued (m,SEQ_BINSTREAM_ZERO )) { + m->binstream_head->len +=1 ; +return 0 ; + } +return seq_put_binstream_head (m,SEQ_BINSTREAM_ZERO ); +} + +static int seq_put_binstream_num (struct seq_file *m,int type, +unsigned long long val) +{ +if (!val) +return seq_put_binstream_zero (m); + +if (seq_binstream_continued (m, type)) { +if (m->count +sizeof (val) >= m->size) + goto overflow; + m->binstream_head->len +=1 ; + *(unsigned long long *)(m->buf + m->count) = val; + m->count +=sizeof (unsigned long long ); +return 0 ; + } + +if (seq_put_binstream_head (m, type)) + goto overflow; + + *(unsigned long long *)(m->buf + m->count) = val; + m->count +=sizeof (val); +return 0 ; +overflow: + m->binstream_head =NULL ; +seq_set_overflow (m); +return -1 ; +} + +static int seq_put_binstream_ull (struct seq_file *m,unsigned long long val) +{ +return seq_put_binstream_num (m,SEQ_BINSTREAM_ULLONG , val); +} + +static int seq_put_binstream_ll (struct seq_file *m,unsigned long long val) +{ +return seq_put_binstream_num (m,SEQ_BINSTREAM_LLONG , val); +} +/** * seq_escape - print string into buffer, escaping some characters * @m: target buffer
@@ -390,12 +498,16 @@ int seq_printf(struct seq_file *m, const char *f, ...)va_list args;int len; +if (seq_binstream_str_prepare (m)) +return -1 ; +if (m->count < m->size) {va_start (args, f); len =vsnprintf (m->buf + m->count, m->size - m->count, f, args);va_end (args);if (m->count + len < m->size) { m->count += len; +seq_binstream_str_end (m, len);return 0 ; } }
@@ -639,8 +751,12 @@ EXPORT_SYMBOL(seq_open_private);int seq_putc (struct seq_file *m,char c) { +if (seq_binstream_str_prepare (m)) +return -1 ; +if (m->count < m->size) { m->buf[m->count++] = c; +seq_binstream_str_end (m,1 );return 0 ; }return -1 ;
@@ -650,9 +766,14 @@ EXPORT_SYMBOL(seq_putc);int seq_puts (struct seq_file *m,const char *s) {int len =strlen (s); + +if (seq_binstream_str_prepare (m)) +return -1 ; +if (m->count + len < m->size) {memcpy (m->buf + m->count, s, len); m->count += len; +seq_binstream_str_end (m, len);return 0 ; }seq_set_overflow (m);
@@ -672,6 +793,9 @@ int seq_put_decimal_ull(struct seq_file *m, char delimiter, {int len; +if (m->binstream) +return seq_put_binstream_ull (m, num); +if (m->count +2 >= m->size)/* we'll write 2 bytes at least */ goto overflow;
@@ -697,6 +821,9 @@ EXPORT_SYMBOL(seq_put_decimal_ull);int seq_put_decimal_ll (struct seq_file *m,char delimiter,long long num) { +if (m->binstream) +return seq_put_binstream_ll (m, num); +if (num <0 ) {if (m->count +3 >= m->size) {seq_set_overflow (m);
@@ -732,6 +859,22 @@ int seq_write(struct seq_file *seq, const void *data, size_t len) }EXPORT_SYMBOL (seq_write); +void seq_set_binstream (struct seq_file *m) +{ + m->binstream = true; +return ; +} +EXPORT_SYMBOL (seq_set_binstream); + +int seq_put_nl (struct seq_file *m) +{ +if (!m->binstream) +return seq_putc (m,'\n' ); +return 0 ; +} +EXPORT_SYMBOL (seq_put_nl); + +struct list_head *seq_list_start (struct list_head *head,loff_t pos) {struct list_head *lh;
include/linux/seq_file.h +36 -0
@@ -8,6 +8,36 @@#include <linux/cpumask.h> #include <linux/nodemask.h> +/* +* In binstream format, all data will follow after seq_binstream_head. +* The 1st 4bytes of head shows the type of information, the next 4bytes shows +* the length of data including the header. +* All seq_binstream_heads will be aligned to 8bytes. So, if the string data +* is not aligned to 8bytes, padding will be added, they'll be filled with 0. +*/ +enum { +SEQ_BINSTREAM_ZERO =1 ,/* represents the number of 0 */ +SEQ_BINSTREAM_STRING =2 , +SEQ_BINSTREAM_LLONG =3 , +SEQ_BINSTREAM_ULLONG =4 , +}; +#define SEQ_BINSTREAM_ALIGNMENT (8) + +/* +* if type==string, len means the length of string. If not, len represents +* the number of values represented by this head. +* +* type=string,len=10 .... a string will follow this. its length is 10bytes. +* type=ZERO,len=10 ...... means 0 repeated 10times. +* type=LLONG, len=10, .... means LLONG repeated 10 times. +* 10 values will follow this header. +*/ +struct seq_binstream_head { +uint32_t type; +uint32_t len;/* length of string or the number of repeats of type*/ +}; + +#ifdef __KERNEL__struct seq_operations ;struct file ;struct path ;
@@ -25,6 +55,8 @@ struct seq_file {struct mutex lock;const struct seq_operations *op;int poll_event; +bool binstream; +struct seq_binstream_head *binstream_head;void *private; };
@@ -127,6 +159,9 @@ int seq_put_decimal_ull(struct seq_file *m, char delimiter,int seq_put_decimal_ll (struct seq_file *m,char delimiter,long long num); +void seq_set_binstream (struct seq_file *m); +int seq_put_nl (struct seq_file *m); +#define SEQ_START_TOKEN ((void *)1)/* * Helpers for iteration over list_head-s in seq_files
@@ -157,4 +192,5 @@ extern struct hlist_node *seq_hlist_start_head_rcu(struct hlist_head *head,extern struct hlist_node *seq_hlist_next_rcu (void *v,struct hlist_head *head,loff_t *ppos); +#endif /* __KERNEL__ */ #endif
Signature · 1 line
1.7.4.1
RFC PATCH 2/2 add /proc/stat.bin
Quoted 1 line
From
5551c12359e0069561bb507bac7454cdfd4edeafMon Sep 17 00:00:00 2001
From: KAMEZAWA Hiroyuki <[email protected]>
Date: Tue, 27 Mar 2012 14:17:55 +0900
Subject: [PATCH 2/2] proc: add /proc/stat.bin
Add a binary format of /proc/stat as /proc/stat.bin. The format is seq file's binstream. The information is shown as folllowing..
[kamezawa@bluextal linux]$ od -t x8 /proc/stat.bin
0000000 0000000400000002 0000000020757063 # string "cpu" 0000020 0000000100000004 000000000000039a # uulong 0x39a 0000040 0000000100000001 0000000500000004 # zero, 5 ents of uulong array starts.. 0000060 0000000000000551 00000000000cb2e6 # 0x551, 0xcb2e6 0000100 0000000000000bf0 0000000000000082 # 0xbf0, 0x082, 0000120 000000000000007d 0000000300000001 # 0x7d, 3 zeros 0000140 0000000400000002 0000000030757063 # string "cpu0" ... 0001540 0000000400000002 0000000072746e69 # string 'intr' 0001560 0000000300000004 000000000002d28c # 3 ents of uulong array of values 0001600 00000000000000b6 0000000000000008 # ... 0002000 00000000000016aa 0000002f00000001 # ...47 zeros. 0002040 0000000100000001 0000000100000004 # zero, 1 ents of uulong 0002060 0000000000000004 0000043a00000001 # 0x4, 1082 of zeros..
...
In general, string and uulong values will have overheads of headers but zeros are compressed...
[kamezawa@bluextal linux]$ wc -c /proc/stat
2817 /proc/stat
[kamezawa@bluextal linux]$ wc -c /proc/stat.bin
1392 /proc/stat.bin
--- fs/proc/stat.c | 79 ++++++++++++++++++++++++++++++++++++++++++++----------- 1 files changed, 63 insertions(+), 16 deletions(-)
fs/proc/stat.c +63 -16
@@ -100,7 +100,7 @@ static int show_stat(struct seq_file *p, void *v)seq_put_decimal_ull (p,' ' ,cputime64_to_clock_t (steal));seq_put_decimal_ull (p,' ' ,cputime64_to_clock_t (guest));seq_put_decimal_ull (p,' ' ,cputime64_to_clock_t (guest_nice)); -seq_putc (p,'\n' ); +seq_put_nl (p);for_each_online_cpu (i) {/* Copy values here to work around gcc-2.95.3, gcc-2.96 */
@@ -125,31 +125,42 @@ static int show_stat(struct seq_file *p, void *v)seq_put_decimal_ull (p,' ' ,cputime64_to_clock_t (steal));seq_put_decimal_ull (p,' ' ,cputime64_to_clock_t (guest));seq_put_decimal_ull (p,' ' ,cputime64_to_clock_t (guest_nice)); -seq_putc (p,'\n' ); +seq_put_nl (p); } -seq_printf (p,"intr %llu" , (unsigned long long )sum); +seq_puts (p,"intr" ); +seq_put_decimal_ull (p,' ' , sum);/* sum again ? it could be updated? */ for_each_irq_nr (j)seq_put_decimal_ull (p,' ' ,kstat_irqs (j)); +seq_put_nl (p); -seq_printf (p, -"\nctxt %llu\n" -"btime %lu\n" -"processes %lu\n" -"procs_running %lu\n" -"procs_blocked %lu\n" , -nr_context_switches (), - (unsigned long )jif, - total_forks, -nr_running (), -nr_iowait ()); +seq_puts (p,"ctxt" ); +seq_put_decimal_ull (p,' ' ,nr_context_switches ()); +seq_put_nl (p); -seq_printf (p,"softirq %llu" , (unsigned long long )sum_softirq); +seq_puts (p,"btime" ); +seq_put_decimal_ull (p,' ' , (unsigned long )jif); +seq_put_nl (p); + +seq_puts (p,"processes" ); +seq_put_decimal_ull (p,' ' , total_forks); +seq_put_nl (p); + +seq_puts (p,"process_running" ); +seq_put_decimal_ull (p,' ' ,nr_running ()); +seq_put_nl (p); + +seq_puts (p,"process_blocked" ); +seq_put_decimal_ull (p,' ' ,nr_iowait ()); +seq_put_nl (p); + +seq_puts (p,"softirq" ); +seq_put_decimal_ull (p,' ' , sum_softirq);for (i =0 ; i <NR_SOFTIRQS ; i++)seq_put_decimal_ull (p,' ' , per_softirq_sums[i]); -seq_putc (p,'\n' ); +seq_put_nl (p);return 0 ; }
@@ -188,9 +199,45 @@ static const struct file_operations proc_stat_operations = { .release = single_release, }; +static int statbin_open (struct inode *inode,struct file *file) +{ +unsigned size =1024 +160 *num_possible_cpus (); +char *buf; +struct seq_file *m; +int res; + +/* each uulong entries requires: 16 bytes */ + size +=16 * nr_irqs; + +/* don't ask for more than the kmalloc() max size */ +if (size >KMALLOC_MAX_SIZE ) + size =KMALLOC_MAX_SIZE ; + buf =kmalloc (size,GFP_KERNEL ); +if (!buf) +return -ENOMEM ; + + res =single_open (file, show_stat,NULL ); +if (!res) { + m = file->private_data; + m->buf = buf; + m->size =ksize (buf); +seq_set_binstream (m); + }else +kfree (buf); +return res; +} + +static const struct file_operations proc_statbin_operations = { + .open = statbin_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; +static int __init proc_stat_init (void ) {proc_create ("stat" ,0 ,NULL , &proc_stat_operations); +proc_create ("stat.bin" ,0 ,NULL , &proc_statbin_operations);return 0 ; }module_init (proc_stat_init);
Signature · 1 line
1.7.4.1
2012/3/27 KAMEZAWA Hiroyuki <[email protected]>:
Quoted 4 lines
Provides another format of
/proc/statfor showing data in binary and compress sequence of Zeros to some extent.Any other idea of formats ?
It mixes everything into one pile.
Quoted 11 lines
From
dea9521e84701631e718884d40b5c96cdcd62d7dMon Sep 17 00:00:00 2001
From: KAMEZAWA Hiroyuki <[email protected]>
Date: Tue, 27 Mar 2012 13:40:54 +0900
Subject: [PATCH 1/2] proc:seq_fileadd binary stream support
/proc/statshows all data in ascii format and it's easily readable. But considering a program which reads/proc/statperiodically, translations of ascii <-> binary happens a few times.1. binrary -> ascii at showing
seq_file
2. ascii -> binary for handling in the program.
Quoted 4 lines · KAMEZAWA Hiroyuki
Provides another format of
/proc/statfor showing data in binary and compress sequence of Zeros to some extent.Any other idea of formats ?
Rather than adding more obscure formats and fields to /proc(/N)/stat, we should pursue descriptive lines like in /proc/N/status.
Or, if so desired, sysfs-style splitted attributes where the filename takes on the description.
2012年3月31日23:42 Jan Engelhardt <[email protected]>:
Quoted 11 lines · KAMEZAWA Hiroyuki
Quoted 5 lines
Provides another format of
/proc/statfor showing data in binary and compress sequence of Zeros to some extent.Any other idea of formats ?
Rather than adding more obscure formats and fields to /proc(/N)/stat, we should pursue descriptive lines like in
/proc/N/status.
Or, if so desired, sysfs-style splitted attributes where the filename takes on the description.
Yes, I like format like /proc/N/status , /proc/meminfo...which is readable to me.
But when I wrote this patch, I didn't want to break current format.
My concern on sysfs-style one data per a file is that it will add many open/close and making top/ps/sar slow.
Thanks,
-Kame
Quoted 6 lines · Hiroyuki Kamezawa
Quoted 5 lines
Rather than adding more obscure formats and fields to /proc(/N)/stat, we should pursue descriptive lines like in
/proc/N/status.
Or, if so desired, sysfs-style splitted attributes where the filename takes on the description.
Quoted 6 lines
Yes, I like format like
/proc/N/status,/proc/meminfo...whichis readable to me.
But when I wrote this patch, I didn't want to break current format.My concern on sysfs-style one data per a file is that it will add many open/close and making top/ps/sar slow.
Or you go netlink. That gives you a binary format.
That said however, binary does not always mean faster. (It's curious to see that generating and grepping through the text dump of the Windows registry is faster than letting regedit.exe search through the binary tree.)
2012年4月1日6:36 Jan Engelhardt <[email protected]>:
Quoted 15 lines · Hiroyuki Kamezawa
Quoted 6 lines
Quoted 5 lines
Rather than adding more obscure formats and fields to /proc(/N)/stat, we should pursue descriptive lines like in
/proc/N/status.
Or, if so desired, sysfs-style splitted attributes where the filename takes on the description.Quoted 6 lines
Yes, I like format like
/proc/N/status,/proc/meminfo...whichis readable to me.
But when I wrote this patch, I didn't want to break current format.My concern on sysfs-style one data per a file is that it will add many open/close and making top/ps/sar slow.
Quoted 6 lines
Or you go netlink. That gives you a binary format.
That said however, binary does not always mean faster. (It's curious to see that generating and grepping through the text dump of the Windows registry is faster than letting regedit.exe search through the binary tree.)
I agree. I need to consider more.
I'm now thinking of adding some data filtering feature of /proc contents may be helpful rather than adding complex format...then we can skip some heavy calls if allowd.
Thanks,
-Kame
Quoted 8 lines · Jan Engelhardt
Quoted 6 lines · Hiroyuki Kamezawa
Quoted 5 lines
Rather than adding more obscure formats and fields to /proc(/N)/stat, we should pursue descriptive lines like in
/proc/N/status.
Or, if so desired, sysfs-style splitted attributes where the filename takes on the description.
Quoted 8 lines
Quoted 6 lines
Yes, I like format like
/proc/N/status,/proc/meminfo...whichis readable to me.
But when I wrote this patch, I didn't want to break current format.My concern on sysfs-style one data per a file is that it will add many open/close and making top/ps/sar slow.
Or you go netlink. That gives you a binary format.
And one more system call to acquire necessary data.
2012/3/27 KAMEZAWA Hiroyuki <[email protected]>:
Quoted 4 lines
Provides another format of
/proc/statfor showing data in binary and compress sequence of Zeros to some extent.Any other idea of formats ?
Not for the format. /proc/stat will likely not be the only file to be treated like this. How abou an open() flag O_??? which indicates the binary format is wanted? Maybe re-purpose a flag which doesn't have a meaning on those files (like, for instance, O_DIRECT). Files which don't support the format can fail to open with the flag set. This way all relevant files can be converted over time.
2012年4月1日4:47 Ulrich Drepper <[email protected]>:
Quoted 4 lines
2012/3/27 KAMEZAWA Hiroyuki <[email protected]>:
Quoted 3 lines
Provides another format of
/proc/statfor showing data in binary and compress sequence of Zeros to some extent.
Quoted 8 lines
Quoted 1 line
Any other idea of formats ?
Not for the format.
/proc/statwill likely not be the only file to be treated like this. How abou anopen()flag O_??? which indicates the binary format is wanted? Maybe re-purpose a flag which doesn't have a meaning on those files (like, for instance,O_DIRECT). Files which don't support the format can fail to open with the flag set. This way all relevant files can be converted over time.
Sounds interesting. I'd like to think of that.
Thanks,
-Kame