Filesystem when perfomance streaming small writes to multiple files
I have a mchine running as a digital video recorder using the VDR software to record DVB broadcasts and noticed a few performance anomalies when accessing the recordings. The anomaly which really drew my attention to was when I found that the maximum read speed of one particular recording was only 1MB/s, on a drive which normally manages 30MB/s!
I spent a while scratching my head and then wrote a synthetic benchmark to simulate the kind of access pattern generated when a multimedia application streams mutiple files to disk. I have found that some filesystems perform really badly when subjected to lots of 4kB writes to multiple files. I know that such small write is an extreme test case but it matches fairly closely the pattern of streaming a broadcast stream to disk one frame at a time (typically 10 - 50kB for TV, ~1kB for radio).
The benchmark program is attached to this email. It isn't particularly polished, but seems to work OK. Here is a very quick summary of what it does:
- Writes to N files simultaneously, using a 4kB write to each file at a time.
- Reads each one of the N files on its own.
It tests a single file and then writing to two files and can be changed to do any number you like.
Here is the output of a test run, using ext3 on an old drive:
Writing 256Mb of data to 1 files in parallel Transferred 256Mb of data in 9.95 seconds (25.73Mb/s) Reading files in sequence
Transferred 256Mb of data in 8.77 seconds (29.21Mb/s)
Writing 128Mb of data to 2 files in parallel Transferred 256Mb of data in 530.84 seconds (0.48Mb/s) Reading files in sequence
Transferred 128Mb of data in 114.09 seconds (1.12Mb/s) Transferred 128Mb of data in 119.04 seconds (1.08Mb/s) Transferred 256Mb of data in 233.14 seconds (1.10Mb/s)
The results above show what a huge difference in performance can occur when ext3 used to record a stream of two files to disk simultaneously.
Below is a summary of some tests comparing the performance of two different hard drives and different filesystems. The tests were performed on a freshly formatted partition using kernels around 2.6.0 - 2.6.1, I'm sorry I can't be more exact - but the results were collected over a period of time.
Firstly a test of some different filesystems on a fairly old large drive (Maxtor 60GB IDE, 2MB Cache, model 96147H6) The numbers shown are the average speed in MB/s.
Num streams |1 1 |2 2 Filesystem |Write Read |Write Read --------------|----------------|-------------- Ext2 |27.7 29.17 | 5.89 14.43 ext3-ordered |25.73 29.21 | 0.48 1.1 Reiserfs |25.31 26.25 | 7.47 13.55 JFS |26.27 26.95 |26.92 28.5 XFS |27.51 26.00 |27.35 27.42
All filesystems give good performance for streaming a single file to and from the disk and shows that the disk has still fairly respectable performance.
As soon as the benchmark tries two streams ext3 performs really badly.
XFS and JFS are clearly the winners, giving exactly the same performance streaming two files as they do with one.
The ext3 results improve significantly on a higher performance hard drive. Below are the resuls of ext2/ext3 tests on a new Seagate 80Gb SATA, 8MB Cache, model ST380023AS.
Num streams |1 1 |2 2 |4 4 Filesystem |Write Read |Write Read |Write Read ------------------------------|--------------|-------------- Ext2 |40.17 43.07 |10.88 21.49 |10.13 11.41 ext3-journal |16.06 42.24 | 7.56 16.28 | 7.17 11.25 ext3-ordered |37.31 43.12 | 4.64 15.33 | 5.25 11.28 ext3-writeback |37.33 42.93 | 4.00 14.88 | 2.97 11.26
I am interested to know what you think about the results. I was surprised to see such a huge performance drop on ext3 with this access pattern. I think it explains why my ext3 formatted disk has been performing badly when I manipulate the TV and radio recordings.
Jon
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/param.h>
#include <string.h>
#include <sys/time.h>
#define BSIZE (4 * 1024)
#define BALIGN (4096)
#define MAX_NAME (256)
char base_name[MAX_NAME];
char *get_name(int n)
{
static char name[MAX_NAME + 5];
sprintf(name, "%s%d", base_name, n);
return name;
}
void display_rate(struct timeval start, struct timeval end, int len) {
int d_s, d_us;
float sec;
d_s = end.tv_sec - start.tv_sec; d_us = end.tv_usec - start.tv_usec; sec = d_s + d_us / 1000000.0;
printf("Transferred %dMb of data in %.2f seconds (%.2fMb/s)\n",
len, sec, len / sec);
fflush(NULL);
}
void create_files(int n, int sz)
{
int out[n], i;
char tmp[BSIZE+BALIGN];
char *buf = (char *)(((unsigned int)tmp + BALIGN - 1) & ~(BALIGN - 1)); int pos;
struct timeval start, end;
printf("Writing %dMb of data to %d files in parallel\n", sz, n); fflush(NULL);
for (i = 0; i < n; i++) {
out[i] = open(get_name(i), O_WRONLY | O_CREAT | O_TRUNC, 0666);
if (out[i] < 0) {
perror("Creating output file");
exit(1);
}
}
memset(buf, 0, BSIZE);
gettimeofday(&start, NULL); for (pos = 0; pos < (sz * 1024 * 1024); pos += BSIZE) {
for(i = 0; i < n; i++) {
if (write(out[i], buf, BSIZE) != BSIZE) {
fprintf(stderr, "Problem writing output file\n");
exit(2);
}
}}
for (i=0; i<n; i++) {
fdatasync(out[i]);
close(out[i]);
}
gettimeofday(&end, NULL); display_rate(start, end, sz * n);
}
void read_files(int n, int sz)
{
int fd[n], i;
char tmp[BSIZE+BALIGN];
char *buf = (char *)(((unsigned int)tmp + BALIGN - 1) & ~(BALIGN - 1)); int pos;
struct timeval o_start, o_end;
printf("Reading files in sequence\n");
fflush(NULL);
for (i = 0; i < n; i++) {
fd[i] = open(get_name(i), O_RDONLY);
if (fd[i] < 0) {
perror("Creating reading file");
exit(1);
}
}
gettimeofday(&o_start, NULL);
for(i = 0; i < n; i++) {
struct timeval start, end; gettimeofday(&start, NULL);
for (pos = 0; pos < (sz * 1024 * 1024); pos += BSIZE) {
if (read(fd[i], buf, BSIZE) != BSIZE) {
fprintf(stderr, "Problem reading file\n");
exit(2);
}
}
gettimeofday(&end, NULL);
display_rate(start, end, sz); }
for (i=0; i<n; i++) {
close(fd[i]);
}
gettimeofday(&o_end, NULL);if (n > 1)
display_rate(o_start, o_end, sz * n); }
void delete_files(int n)
{
int i;
for (i = 0; i < n; i++) {
unlink(get_name(i)); }
}
void usage(char **argv)
{
fprintf(stderr, "Usage: %s <name> <size>\n", argv[0]);
fprintf(stderr, " Creates files name0, name1 ... nameN in parallel all of given size (in Mb)\n"); exit(1);
}
void run_test(int n, int s)
{
delete_files(n);
create_files(n, s); read_files(n, s); delete_files(n);
printf("\n");
fflush(NULL);
}
int main(int argc, char *argv[])
{
unsigned int s = 512;
strcpy(base_name, "temp_");
if (argc > 1) {
int len = strlen(argv[1]);
if ((len == 0) || (len >= MAX_NAME) || (*argv[1] == '-'))
usage(argv);
strcpy(base_name, argv[1]);
}#if 0
if (argc > 2) {
n = atoi(argv[2]);
if ((n == 0) || (n > 10))
usage(argv); }
#endif
if (argc > 2) {
s = atoi(argv[2]);
if ((s == 0) || (s > 4000))
usage(argv);
}
if (argc > 3) {
usage(argv);
}
run_test(1, s);
run_test(2, s / 2);
//run_test(4, s / 4); return 0;
}