Skip to contentSkip to conversation

mmap over nfs leads to excessive system load

DISCUSSION Kenny Simpson 2 participants 4 messages Nov 8, 2005

lore
Share & tools
lore
#0 lore raw

Just another data point....
If I open the file with O_DIRECT.. not much changes:

samples  %        image name               app name                 symbol name
12585321 18.9373  vmlinux-2.6.14           vmlinux-2.6.14           find_get_pages_tag
8608887  12.9539  vmlinux-2.6.14           vmlinux-2.6.14           mpage_writepages
6870600  10.3383  vmlinux-2.6.14           vmlinux-2.6.14           unlock_page
6605417   9.9393  vmlinux-2.6.14           vmlinux-2.6.14           clear_page_dirty_for_io
6259207   9.4183  vmlinux-2.6.14           vmlinux-2.6.14           release_pages
3249493   4.8896  vmlinux-2.6.14           vmlinux-2.6.14           __lookup_tag
3248871   4.8886  vmlinux-2.6.14           vmlinux-2.6.14           pci_conf1_write
2677914   4.0295  vmlinux-2.6.14           vmlinux-2.6.14           page_waitqueue
982811    1.4789  vmlinux-2.6.14           vmlinux-2.6.14           _read_lock_irqsave
917165    1.3801  vmlinux-2.6.14           vmlinux-2.6.14           _read_unlock_irq
758960    1.1420  vmlinux-2.6.14           vmlinux-2.6.14           __wake_up_bit
706607    1.0632  vmlinux-2.6.14           vmlinux-2.6.14           _spin_lock_irqsave

-Kenny

__________________________________
Yahoo! Mail - PC Magazine Editors' Choice 2005
http://mail.yahoo.com

#1 lore raw
Quoted 2 lines · Kenny Simpson

Just another data point....
If I open the file with O_DIRECT.. not much changes:

Hmm... Are you mounting using the -osync or -onoac options? Doing synchronous writes will tend to slow down flushing considerably, and the VM appears to be very fragile w.r.t. slow filesystems.

Cheers,
Trond

Quoted 29 lines
samples  %        image name               app name                 symbol name
12585321 18.9373  vmlinux-2.6.14           vmlinux-2.6.14           find_get_pages_tag
8608887  12.9539  vmlinux-2.6.14           vmlinux-2.6.14           mpage_writepages
6870600  10.3383  vmlinux-2.6.14           vmlinux-2.6.14           unlock_page
6605417   9.9393  vmlinux-2.6.14           vmlinux-2.6.14           clear_page_dirty_for_io
6259207   9.4183  vmlinux-2.6.14           vmlinux-2.6.14           release_pages
3249493   4.8896  vmlinux-2.6.14           vmlinux-2.6.14           __lookup_tag
3248871   4.8886  vmlinux-2.6.14           vmlinux-2.6.14           pci_conf1_write
2677914   4.0295  vmlinux-2.6.14           vmlinux-2.6.14           page_waitqueue
982811    1.4789  vmlinux-2.6.14           vmlinux-2.6.14           _read_lock_irqsave
917165    1.3801  vmlinux-2.6.14           vmlinux-2.6.14           _read_unlock_irq
758960    1.1420  vmlinux-2.6.14           vmlinux-2.6.14           __wake_up_bit
706607    1.0632  vmlinux-2.6.14           vmlinux-2.6.14           _spin_lock_irqsave

-Kenny

__________________________________
Yahoo! Mail - PC Magazine Editors' Choice 2005
http://mail.yahoo.com
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

#2 lore raw

I knew I forgoet something important in the info:
type nfs (rw,tcp,rsize=32768,wsize=32768,hard,intr,vers=3,tcp,rsize=32768,wsize=32768,hard,intr

-Kenny

Quoted 10 lines · --- Trond Myklebust
Quoted 2 lines · Kenny Simpson

Just another data point....
If I open the file with O_DIRECT.. not much changes:

Hmm... Are you mounting using the -osync or -onoac options? Doing synchronous writes will tend to slow down flushing considerably, and the VM appears to be very fragile w.r.t. slow filesystems.

Cheers,
Trond

__________________________________
Start your day with Yahoo! - Make it your home page!
http://www.yahoo.com/r/hs

#3 lore raw
In reply to Trond Myklebust

I am attaching a sample piece of code to show the behavior.
This simply tries to grow a file as fast as it can using different methods. Use with caution as it does not stop, and will fill the disk if you let it run.

Run as "nfstest -m <filename>" where <filename> is on an nfs mount.

-Kenny

__________________________________
Start your day with Yahoo! - Make it your home page!
http://www.yahoo.com/r/hs

/* test the write throughput of a sliding mmap window vs simple FILE* */ #define _GNU_SOURCE

#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <signal.h>
#include <string.h>
#include <unistd.h>
#include <time.h>
#include <errno.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/time.h>

int total_bytes = 0;
int last_bytes = 0;

struct timeval last_time;

void timeout(int i)
{
struct timeval now;
gettimeofday(&now, 0);

  (void) i;

  {
    double time_diff = (now.tv_sec - last_time.tv_sec) + ((now.tv_usec - last_time.tv_usec) / 1000000.);
    int new_bytes = total_bytes;
    int diff = new_bytes - last_bytes;
    printf("wrote %dk %dM bytes in %f seconds -> %fM/sec\n", diff / 1024, diff / (1024 * 1024), time_diff, (diff / (time_diff * 1000000.)));
    last_bytes = new_bytes;
    last_time = now;

}
}

void do_mapwrite(int fd)
{
/* lets write as fast as we can.... */
int const window_size = 2 * 1024 * 1024; /* 16k */ int const window_pages = window_size / 4096;

  int file_page_offset = 0;
  long long file_size = window_size;

  ftruncate64(fd, file_size);

  {
    char* mapping_start = (char*)mmap64(0, window_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
    /* scribble into buffer and walk the window */
    for (;;) {
      memset(mapping_start, 0, window_size);

      /* grow file */
      file_size += window_size;
      ftruncate64(fd, file_size);
      file_page_offset += window_pages;
      /*msync(mapping_start, window_size, MS_SYNC);*/
      /*munmap(mapping_start, window_size);*/
      /*mapping_start = (char*)mmap64(mapping_start, window_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd, file_size - window_size);*/
      remap_file_pages(mapping_start, window_size, 0, file_page_offset, MAP_SHARED);

      total_bytes += window_size;
    }

}
}

void do_filewrite(int fd)
{
FILE* ptr = fdopen(fd, "w+");

  int line_len = 80;
  char* buf = (char*)malloc(line_len);
  memset(buf, 0, line_len);
  for (;;) {
    fwrite(buf, line_len, 1, ptr);
    total_bytes += line_len;

}
}

void do_syswrite(int fd)
{
int line_len = 64 * 1024;
char* buf = (char*)malloc(line_len);
memset(buf, 0, line_len);

  for (;;) {
    write(fd, buf, line_len);
    total_bytes += line_len;

}
}

int main(int argc, char* argv[])
{

  if (argc != 3) {
    printf("usage: %s -[mfw] <filename>\n", argv[0]);
    return 0;

}

  if ((argv[1][0] != '-') || 
      ((argv[1][1] != 'm') &&
       (argv[1][1] != 'w') &&
       (argv[1][1] != 'f'))) {
    printf("usage: %s -[mfw] <filename>\n", argv[0]);
    return 0;
  }

  int fd = open(argv[2], O_RDWR | O_CREAT | O_LARGEFILE | O_DIRECT, 0644);
  if (fd < 0) {
    perror("open");
    return 0;
  }

  /* start the clock...  */
  signal(SIGALRM, timeout);
  {
    struct itimerval itv;
    itv.it_interval.tv_sec = 1;
    itv.it_interval.tv_usec = 0;
    itv.it_value = itv.it_interval;
    setitimer(ITIMER_REAL, &itv, 0);

}

  switch (argv[1][1]) {
  case 'm': do_mapwrite(fd); break;
  case 'f': do_filewrite(fd); break;
  case 'w': do_syswrite(fd); break;
  }

return 0;
}