Skip to contentSkip to conversation

write file larger than avail in nfs mp hangs but it should return ENOSPC

Problem in 5.5.0-rc6+ Yongcheng Yang 1 participant 1 message Jan 21, 2020

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

Hello,

I have just hit a problem that a process writing a large file (bigger than available) in nfs mountpoint hangs forever (even after it's full).

It might loop in read()/write() when checked by ltrace.

Please does anyone hit this problem? And do we need to fix it?

Thanks,
Yongcheng

Steps:
~~~~~~~~~~~~
- NFS server
1. Prepare a partition with arbitrary size (e.g., 4G).
2. Format the partition as xfs or ext4.
3. Mount the partition and then exports the directory via nfs.

- NFS client
1. Mount the NFS server's directory.
2. Execute dd command to the directory's file which should hit the size limit and it should return "No space left on device".

   # dd if=/dev/zero of=/mnt/nfs/file bs=100K count=4G

For examples (I just test in single host):
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
[root ~]# truncate --size 4G /tmp/test.img
[root ~]# losetup -f
/dev/loop0
[root ~]# losetup /dev/loop0 /tmp/test.img
[root ~]# mkdir /export_test /mnt/mnt_test
[root ~]# mkfs.ext4 /dev/loop0
mke2fs 1.44.6 (5-Mar-2019)
Discarding device blocks: done
Creating filesystem with 1048576 4k blocks and 262144 inodes Filesystem UUID: c3bd7b89-b134-4930-82f3-7489baa76849 Superblock backups stored on blocks:

        32768, 98304, 163840, 229376, 294912, 819200, 884736

Allocating group tables: done
Writing inode tables: done
Creating journal (16384 blocks): done
Writing superblocks and filesystem accounting information: done

[root ~]# echo $?
0
[root ~]# mount -t ext4 /dev/loop0 /export_test/
[root ~]# systemctl start nfs-server
[root ~]# exportfs -o rw,no_root_squash *:/export_test/ [root ~]# mount -t nfs $HOSTNAME:/export_test /mnt/mnt_test [root ~]# dd if=/dev/random of=/mnt/mnt_test/testfile bs=100K count=4G
...

# hang
...

[root ~]# umount /mnt/mnt_test/
[root ~]#
[root ~]# systemctl stop nfs-server
[root ~]#
[root ~]# umount /export_test/
[root ~]# mount | grep loop
[root ~]# wipefs -a /dev/loop0
/dev/loop0: 2 bytes were erased at offset 0x00000438 (ext4): 53 ef [root ~]#
[root ~]#
[root ~]# mkfs.xfs /dev/loop0

meta-data=/dev/loop0             isize=512    agcount=4, agsize=262144 blks
         =                       sectsz=512   attr=2, projid32bit=1
         =                       crc=1        finobt=1, sparse=1, rmapbt=0
         =                       reflink=1 
data     =                       bsize=4096   blocks=1048576, imaxpct=25
         =                       sunit=0      swidth=0 blks
naming   =version 2              bsize=4096   ascii-ci=0, ftype=1
log      =internal log           bsize=4096   blocks=2560, version=2
         =                       sectsz=512   sunit=0 blks, lazy-count=1
realtime =none                   extsz=4096   blocks=0, rtextents=0

[root ~]#
[root ~]# mount -t xfs /dev/loop0 /export_test/ [root ~]# systemctl start nfs-server
[root ~]# exportfs -o rw,no_root_squash *:/export_test/ [root ~]# mount -t nfs $HOSTNAME:/export_test /mnt/mnt_test [root ~]# dd if=/dev/zero of=/mnt/mnt_test/testfile count=4G
...

# hang
...

#

# debug from another terminal
#

[root temp]# nfsstat -m
/mnt/mnt_test from nfsserver.redhat.com:/export_test
Flags: rw,relatime,vers=4.2,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=10.73.4.183,local_lock=none,addr=10.73.4.183

[root temp]# df -h /mnt/mnt_test/

Filesystem                         Size  Used Avail Use% Mounted on
nfsserver.redhat.com:/export_test  4.0G  4.0G  1.0M 100% /mnt/mnt_test
[root temp]# ps aux | grep -v grep | grep -w dd
root      5569 97.5  0.0   4792  1700 pts/0    R+   03:10 132:13 dd if=/dev/zero of=/mnt/mnt_test/testfile count=4G
[root temp]# ltrace -p 5569
memcpy(0x56179886d000, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 512)                      = 0x56179886d000
write(1, "", 512)                                                                                                       = 512
read(0, "", 512)                                                                                                        = 512
memcpy(0x56179886d000, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 512)                      = 0x56179886d000
write(1, "", 512)                                                                                                       = 512
read(0, "", 512)                                                                                                        = 512
memcpy(0x56179886d000, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 512)                      = 0x56179886d000
write(1, "", 512)                                                                                                       = 512
read(0, "", 512)                                                                                                        = 512

...
...
...
...