Skip to contentSkip to conversation

NFS related Oops in 2.4.[39]-xfs

DISCUSSION Peter Wächtler 3 participants 7 messages Oct 17, 2001 NFS server

lore
Share & tools
lore
#0 lore raw

noisy:/usr/src/linux # uname -a
Linux noisy 2.4.3-XFS #8 SMP Sam Mai 19 18:21:36 CEST 2001 i686 unknown

dual board with just one processor
/usr/local and /home on LogicalVolumes
tried also 2.4.9-XFS (but see below)

------------ oops -------------
Warning (compare_maps): ksyms_base symbol __rta_fill_R__ver___rta_fill not found in System.map. Ignoring ksyms_base entry
[other warnings stripped]

Oops: 0000
CPU:    0
EIP:    0010:<c016b3c4>

Using defaults from ksymoops -t elf32-i386 -a i386
EFLAGS: 00010246
eax: 00000000 ebx: 00000000 ecx: cff805b8 edx: 00000010
esi: ca018260 edi: ca0182e0 ebp: cb9e5800 esp: cb9edeec
ds: 0018 es: 0018 ss: 0018

Stack: 0100708b ca018360 c016b7d8 ca0182e0 00000002 cba07000 cb9efc00 cb9e5800
       ce078bc4 ca018360 00000000 ca018360 c016bb70 ce078a80 0102a048 00000005
Call Trace: <c01bb7d8> <c016bb70> <c016a0b5>c0169b73> <c02a60bf> <c016999d> <c01054c4>
Code: 8b 40 10 39 d0 74 21 8d 58 c8 39 f3 75 06 8b 5a 04 83 c3 c8
Quoted 1 line
Quoted 1 line

EIP; c016b3c4 <nfsd_findparent+78/e8> <=====

Trace; c01bb7d8 <xfs_buf_item_bits+38/58>
Trace; c016bb70 <fh_verify+26c/48c>
Trace; c016a0b5 <nfsd_proc_getattr+91/98>
Code; c016b3c4 <nfsd_findparent+78/e8>
00000000 <_EIP>:
Code; c016b3c4 <nfsd_findparent+78/e8> <=====
0: 8b 40 10 mov 0x10(%eax),%eax <=====
Code; c016b3c7 <nfsd_findparent+7b/e8>
3: 39 d0 cmp %edx,%eax
Code; c016b3c9 <nfsd_findparent+7d/e8>
5: 74 21 je 28 <_EIP+0x28> c016b3ec <nfsd_findparent+a0/e8>
Code; c016b3cb <nfsd_findparent+7f/e8>
7: 8d 58 c8 lea 0xffffffc8(%eax),%ebx
Code; c016b3ce <nfsd_findparent+82/e8>
a: 39 f3 cmp %esi,%ebx
Code; c016b3d0 <nfsd_findparent+84/e8>
c: 75 06 jne 14 <_EIP+0x14> c016b3d8 <nfsd_findparent+8c/e8>
Code; c016b3d2 <nfsd_findparent+86/e8>
e: 8b 5a 04 mov 0x4(%edx),%ebx
Code; c016b3d5 <nfsd_findparent+89/e8>
11: 83 c3 c8 add $0xffffffc8,%ebx

31 warnings issued. Results may not be reliable.
------------ oops -------------

Why do I get all those warnings about symbol mismatch?

The code causing the Oops:

struct dentry *nfsd_findparent(struct dentry *child)

{
        struct dentry *tdentry, *pdentry;
        tdentry = d_alloc(child, &(const struct qstr) {"..", 2, 0});
        if (!tdentry)
                return ERR_PTR(-ENOMEM);
 
        /* I'm going to assume that if the returned dentry is different, then
         * it is well connected.  But nobody returns different dentrys do they?
         */
        pdentry = child->d_inode->i_op->lookup(child->d_inode, tdentry);
        d_drop(tdentry); /* we never want ".." hashed */
        if (!pdentry) {
                /* I don't want to return a ".." dentry.
                 * I would prefer to return an unconnected "IS_ROOT" dentry,
                 * though a properly connected dentry is even better
                 */
                /* if first or last of alias list is not tdentry, use that
                 * else make a root dentry
                 */
                struct list_head *aliases = &tdentry->d_inode->i_dentry;
                spin_lock(&dcache_lock);
                if (aliases->next != aliases) {         <=========== CRASH
!!!!!!!!!!!!!!!!!!!!!!!
                        pdentry = list_entry(aliases->next, struct dentry, d_alias);
                        if (pdentry == tdentry)
                                pdentry = list_entry(aliases->prev, struct dentry,
d_alias);
                        if (pdentry == tdentry)
                                pdentry = NULL;
                        if (pdentry) dget_locked(pdentry);
                }
                spin_unlock(&dcache_lock);
                if (pdentry == NULL) {
                        pdentry = d_alloc_root(igrab(tdentry->d_inode));
                        if (pdentry) {
                                pdentry->d_flags |= DCACHE_NFSD_DISCONNECTED;
                                d_rehash(pdentry);
                        }
                }
                if (pdentry == NULL)
                        pdentry = ERR_PTR(-ENOMEM);
        }
        dput(tdentry); /* it is not hashed, it will be discarded */
        return pdentry;
}

(gdb) disass nfsd_findparent
Dump of assembler code for function nfsd_findparent:

0xc016b34c <nfsd_findparent>:   push   %esi
0xc016b34d <nfsd_findparent+1>: push   %ebx
0xc016b34e <nfsd_findparent+2>: mov    0xc(%esp,1),%ebx
0xc016b352 <nfsd_findparent+6>: push   $0xc02ce724
0xc016b357 <nfsd_findparent+11>:        push   %ebx
0xc016b358 <nfsd_findparent+12>:        call   0xc0144608 <d_alloc>
0xc016b35d <nfsd_findparent+17>:        mov    %eax,%esi
0xc016b35f <nfsd_findparent+19>:        add    $0x8,%esp
        tdentry = d_alloc(child, &(const struct qstr) {"..", 2, 0});

0xc016b362 <nfsd_findparent+22>: test %esi,%esi
if (!tdentry)

0xc016b364 <nfsd_findparent+24>:        jne    0xc016b370 <nfsd_findparent+36>
0xc016b366 <nfsd_findparent+26>:        mov    $0xfffffff4,%eax
0xc016b36b <nfsd_findparent+31>:        jmp    0xc016b431 <nfsd_findparent+229>
			return ERR_PTR(-ENOMEM);
0xc016b370 <nfsd_findparent+36>:        mov    0x8(%ebx),%eax
0xc016b373 <nfsd_findparent+39>:        mov    0x84(%eax),%edx
0xc016b379 <nfsd_findparent+45>:        push   %esi
0xc016b37a <nfsd_findparent+46>:        push   %eax
0xc016b37b <nfsd_findparent+47>:        mov    0x4(%edx),%eax
0xc016b37e <nfsd_findparent+50>:        call   *%eax
0xc016b380 <nfsd_findparent+52>:        mov    %eax,%ebx
0xc016b382 <nfsd_findparent+54>:        add    $0x8,%esp
        pdentry = child->d_inode->i_op->lookup(child->d_inode, tdentry);

static __inline__ void d_drop(struct dentry * dentry)

{
        spin_lock(&dcache_lock);
        list_del(&dentry->d_hash);
        INIT_LIST_HEAD(&dentry->d_hash);
        spin_unlock(&dcache_lock);
}		
0xc016b385 <nfsd_findparent+57>:        lock decb 0xc0329898
0xc016b38c <nfsd_findparent+64>:        js     0xc02aefa4 <stext_lock+12548>
0xc016b392 <nfsd_findparent+70>:        lea    0x18(%esi),%eax
0xc016b395 <nfsd_findparent+73>:        mov    0x4(%eax),%ecx
0xc016b398 <nfsd_findparent+76>:        mov    0x18(%esi),%edx
0xc016b39b <nfsd_findparent+79>:        mov    %ecx,0x4(%edx)
0xc016b39e <nfsd_findparent+82>:        mov    %edx,(%ecx)
0xc016b3a0 <nfsd_findparent+84>:        mov    %eax,0x18(%esi)
0xc016b3a3 <nfsd_findparent+87>:        mov    %eax,0x1c(%esi)
0xc016b3a6 <nfsd_findparent+90>:        movb   $0x1,0xc0329898

d_drop(tdentry); /* we never want ".." hashed */

0xc016b3ad <nfsd_findparent+97>:        test   %ebx,%ebx
0xc016b3af <nfsd_findparent+99>:        jne    0xc016b426 <nfsd_findparent+218>
        if (!pdentry) {
0xc016b3b1 <nfsd_findparent+101>:       mov    0x8(%esi),%eax
0xc016b3b4 <nfsd_findparent+104>:       lea    0x10(%eax),%edx   // edx holds aliases
                struct list_head *aliases = &tdentry->d_inode->i_dentry; 
				
		// is tdentry->d_inode->i_dentry not valid anymore?  aliases gets NULL

0xc016b3b7 <nfsd_findparent+107>: lock decb 0xc0329898
0xc016b3be <nfsd_findparent+114>: js 0xc02aefb4 <stext_lock+12564>

                spin_lock(&dcache_lock);

0xc016b3c4 <nfsd_findparent+120>: mov 0x10(%eax),%eax <=====================
CRASH !!!
0xc016b3c7 <nfsd_findparent+123>: cmp %edx,%eax // eax holds aliases->next

                if (aliases->next != aliases) { 
0xc016b3c9 <nfsd_findparent+125>:       je     0xc016b3ec <nfsd_findparent+160>
0xc016b3cb <nfsd_findparent+127>:       lea    0xffffffc8(%eax),%ebx
                        pdentry = list_entry(aliases->next, struct dentry, d_alias);
0xc016b3ce <nfsd_findparent+130>:       cmp    %esi,%ebx
0xc016b3d0 <nfsd_findparent+132>:       jne    0xc016b3d8 <nfsd_findparent+140>
0xc016b3d2 <nfsd_findparent+134>:       mov    0x4(%edx),%ebx
0xc016b3d5 <nfsd_findparent+137>:       add    $0xffffffc8,%ebx
0xc016b3d8 <nfsd_findparent+140>:       xor    %eax,%eax
0xc016b3da <nfsd_findparent+142>:       cmp    %esi,%ebx
0xc016b3dc <nfsd_findparent+144>:       cmove  %eax,%ebx
0xc016b3df <nfsd_findparent+147>:       test   %ebx,%ebx
0xc016b3e1 <nfsd_findparent+149>:       je     0xc016b3ec <nfsd_findparent+160>
0xc016b3e3 <nfsd_findparent+151>:       push   %ebx
0xc016b3e4 <nfsd_findparent+152>:       call   0xc0144080 <dget_locked>
0xc016b3e9 <nfsd_findparent+157>:       add    $0x4,%esp
                        if (pdentry) dget_locked(pdentry);  
0xc016b3ec <nfsd_findparent+160>:       movb   $0x1,0xc0329898
0xc016b3f3 <nfsd_findparent+167>:       test   %ebx,%ebx
0xc016b3f5 <nfsd_findparent+169>:       jne    0xc016b41c <nfsd_findparent+208>
                spin_unlock(&dcache_lock);
0xc016b3f7 <nfsd_findparent+171>:       mov    0x8(%esi),%eax
0xc016b3fa <nfsd_findparent+174>:       push   %eax
0xc016b3fb <nfsd_findparent+175>:       call   0xc0145ee8 <igrab>
0xc016b400 <nfsd_findparent+180>:       push   %eax
0xc016b401 <nfsd_findparent+181>:       call   0xc01447f4 <d_alloc_root>
0xc016b406 <nfsd_findparent+186>:       mov    %eax,%ebx
0xc016b408 <nfsd_findparent+188>:       add    $0x8,%esp
                        pdentry = d_alloc_root(igrab(tdentry->d_inode));
0xc016b40b <nfsd_findparent+191>:       test   %ebx,%ebx
0xc016b40d <nfsd_findparent+193>:       je     0xc016b41c <nfsd_findparent+208>
0xc016b40f <nfsd_findparent+195>:       orb    $0x4,0x4(%ebx)
0xc016b413 <nfsd_findparent+199>:       push   %ebx
0xc016b414 <nfsd_findparent+200>:       call   0xc0144ab4 <d_rehash>
0xc016b419 <nfsd_findparent+205>:       add    $0x4,%esp
0xc016b41c <nfsd_findparent+208>:       mov    $0xfffffff4,%eax
0xc016b421 <nfsd_findparent+213>:       test   %ebx,%ebx
0xc016b423 <nfsd_findparent+215>:       cmove  %eax,%ebx
0xc016b426 <nfsd_findparent+218>:       push   %esi
        }  /* if (!pdentry) */
0xc016b427 <nfsd_findparent+219>:       call   0xc0143e90 <dput>
0xc016b42c <nfsd_findparent+224>:       mov    %ebx,%eax
0xc016b42e <nfsd_findparent+226>:       add    $0x4,%esp
0xc016b431 <nfsd_findparent+229>:       pop    %ebx
0xc016b432 <nfsd_findparent+230>:       pop    %esi
0xc016b433 <nfsd_findparent+231>:       ret

End of assembler dump.
(gdb)

2.4.3-xfs is compiled with gcc 2.95.2
2.4.9-XFS is compiled with gcc 2.91.6

After the crash I tried to reboot, but it quickly failed again.

reboot   system boot  2.4.3-XFS        Mon Oct 15 18:20          (21:49)
reboot   system boot  2.4.3-XFS        Mon Oct 15 18:17          (21:52)
reboot   system boot  2.4.3-XFS        Mon Oct 15 18:13          (21:57)
reboot   system boot  2.4.3-XFS        Thu Oct 11 16:51         (4+23:18)
reboot   system boot  2.4.3-XFS        Thu Oct 11 16:32         (4+23:37)
reboot   system boot  2.4.3-XFS        Thu Oct 11 16:28         (4+23:41)
reboot   system boot  2.4.3-XFS        Thu Oct 11 16:23         (4+23:46)
reboot   system boot  2.4.3-XFS        Thu Oct 11 16:12         (4+23:57)
reboot   system boot  2.4.3-XFS        Thu Oct 11 15:05         (5+01:04)
reboot   system boot  2.4.9-xfs        Thu Oct  4 17:02         (11+23:08)
reboot   system boot  2.4.9-xfs        Thu Oct  4 16:54         (11+23:15)
reboot   system boot  2.4.3-XFS        Thu Oct  4 16:47         (11+23:23)
reboot   system boot  2.4.9-xfs        Thu Oct  4 16:39         (11+23:31)
reboot   system boot  2.4.9-xfs        Thu Oct  4 16:14         (11+23:55)

To recover I had to unplug/powerdown all connected clients. :-( Uhh. I think 2.4.9 crashed at the same instructions. But I can't reproduce it on demand (hey, in the meantime we can work!).
Seeing the NFS related changes in 2.4.10: should I upgrade? We use only NFSv2. Is NFSv3 more stable, if that matters here?

The machine with 2.4.3 was up for several months - with light load. Now the number of crashes (with 6 NFS clients using /home + cvs) went up.

We do mount our compile environment in a strange way:

NOTE: /server is a symlink to /usr/local/export
# See exports(5) for a description.
# This file contains a list of all directories exported to other computers. # It is used by rpc.nfsd and rpc.mountd.

/opt/xxx                        *.xxx(ro)
/home                           *.xxx(rw)
/tmp                            *.xxx(rw)
/usr/local/export               *.xxx(rw)

/etc/fstab
devserv:/server/compileenv/lib/lib \
/compenv/xxx/lib nfs ro,auto 0 0
devserv:/server/compileenv/usr_lib/usr/lib \
/compenv/xxx/usr_lib nfs ro,auto 0 0
devserv:/server/compileenv/gcc-lib \
/compenv/xxx/gcc-lib nfs ro,exec,auto 0 0
devserv:/server/compileenv/usr_include/usr/include \
/compenv/xxx/usr_include nfs ro,auto 0 0
devserv:/server/compileenv/usr_linux/usr/linux/include \
/compenv/xxx/usr_linux_include nfs ro,auto 0 0

Would it help to mount a single /usr/local/export and work with a symlink tree (as I actually do with autofs) ?
What can cause the "invalid" dentries? Did someone remove some files or were some dentries de'hashed because of dcache growth?

As workaround I will do:

                struct list_head *aliases = &tdentry->d_inode->i_dentry;

                if (aliases && (aliases->next != aliases) ) {
                        spin_lock(&dcache_lock);
                        pdentry = list_entry(aliases->next, struct dentry, d_alias);
                        if (pdentry == tdentry)
                                pdentry = list_entry(aliases->prev, struct dentry,
d_alias);
                        if (pdentry == tdentry)
                                pdentry = NULL;
                        if (pdentry) dget_locked(pdentry);
                        spin_unlock(&dcache_lock);
                } else
                     pdentry=NULL;
#1 lore raw

Where did you get your kernel (the 2.4.9 version that is) this problem sounds familiar, but I am pretty sure we fixed this case in XFS somewhere between 2.4.3 and 2.4.9.

Steve

Quoted 336 lines

noisy:/usr/src/linux # uname -a
Linux noisy 2.4.3-XFS #8 SMP Sam Mai 19 18:21:36 CEST 2001 i686 unknown

dual board with just one processor
/usr/local and /home on LogicalVolumes
tried also 2.4.9-XFS (but see below)

------------ oops -------------
Warning (compare_maps): ksyms_base symbol __rta_fill_R__ver___rta_fill not fo und in
System.map. Ignoring ksyms_base entry
[other warnings stripped]

Oops: 0000
CPU:    0
EIP:    0010:<c016b3c4>

Using defaults from ksymoops -t elf32-i386 -a i386
EFLAGS: 00010246
eax: 00000000 ebx: 00000000 ecx: cff805b8 edx: 00000010
esi: ca018260 edi: ca0182e0 ebp: cb9e5800 esp: cb9edeec
ds: 0018 es: 0018 ss: 0018

Stack: 0100708b ca018360 c016b7d8 ca0182e0 00000002 cba07000 cb9efc00 cb9e580
0
       ce078bc4 ca018360 00000000 ca018360 c016bb70 ce078a80 0102a048 0000000
5
Call Trace: <c01bb7d8> <c016bb70> <c016a0b5>c0169b73> <c02a60bf> <c016999d> <
c01054c4>
Code: 8b 40 10 39 d0 74 21 8d 58 c8 39 f3 75 06 8b 5a 04 83 c3 c8
Quoted 1 line
Quoted 1 line

EIP; c016b3c4 <nfsd_findparent+78/e8> <=====

Trace; c01bb7d8 <xfs_buf_item_bits+38/58>
Trace; c016bb70 <fh_verify+26c/48c>
Trace; c016a0b5 <nfsd_proc_getattr+91/98>
Code; c016b3c4 <nfsd_findparent+78/e8>
00000000 <_EIP>:
Code; c016b3c4 <nfsd_findparent+78/e8> <=====
0: 8b 40 10 mov 0x10(%eax),%eax <=====
Code; c016b3c7 <nfsd_findparent+7b/e8>
3: 39 d0 cmp %edx,%eax
Code; c016b3c9 <nfsd_findparent+7d/e8>
5: 74 21 je 28 <_EIP+0x28> c016b3ec <nfsd_findpa
rent+a0/e8>
Code; c016b3cb <nfsd_findparent+7f/e8>
7: 8d 58 c8 lea 0xffffffc8(%eax),%ebx
Code; c016b3ce <nfsd_findparent+82/e8>
a: 39 f3 cmp %esi,%ebx
Code; c016b3d0 <nfsd_findparent+84/e8>
c: 75 06 jne 14 <_EIP+0x14> c016b3d8 <nfsd_findpa
rent+8c/e8>
Code; c016b3d2 <nfsd_findparent+86/e8>
e: 8b 5a 04 mov 0x4(%edx),%ebx
Code; c016b3d5 <nfsd_findparent+89/e8>
11: 83 c3 c8 add $0xffffffc8,%ebx

31 warnings issued. Results may not be reliable.
------------ oops -------------

Why do I get all those warnings about symbol mismatch?

The code causing the Oops:

struct dentry *nfsd_findparent(struct dentry *child)

{
        struct dentry *tdentry, *pdentry;
        tdentry = d_alloc(child, &(const struct qstr) {"..", 2, 0});
        if (!tdentry)
                return ERR_PTR(-ENOMEM);
 
        /* I'm going to assume that if the returned dentry is different, then
         * it is well connected.  But nobody returns different dentrys do the
y?
         */
        pdentry = child->d_inode->i_op->lookup(child->d_inode, tdentry);
        d_drop(tdentry); /* we never want ".." hashed */
        if (!pdentry) {
                /* I don't want to return a ".." dentry.
                 * I would prefer to return an unconnected "IS_ROOT" dentry,
                 * though a properly connected dentry is even better
                 */
                /* if first or last of alias list is not tdentry, use that
                 * else make a root dentry
                 */
                struct list_head *aliases = &tdentry->d_inode->i_dentry;
                spin_lock(&dcache_lock);
                if (aliases->next != aliases) {         <=========== CRASH
!!!!!!!!!!!!!!!!!!!!!!!
                        pdentry = list_entry(aliases->next, struct dentry, d_
alias);
                        if (pdentry == tdentry)
                                pdentry = list_entry(aliases->prev, struct de
ntry,
d_alias);
                        if (pdentry == tdentry)
                                pdentry = NULL;
                        if (pdentry) dget_locked(pdentry);
                }
                spin_unlock(&dcache_lock);
                if (pdentry == NULL) {
                        pdentry = d_alloc_root(igrab(tdentry->d_inode));
                        if (pdentry) {
                                pdentry->d_flags |= DCACHE_NFSD_DISCONNECTED;
                                d_rehash(pdentry);
                        }
                }
                if (pdentry == NULL)
                        pdentry = ERR_PTR(-ENOMEM);
        }
        dput(tdentry); /* it is not hashed, it will be discarded */
        return pdentry;
}

(gdb) disass nfsd_findparent
Dump of assembler code for function nfsd_findparent:

0xc016b34c <nfsd_findparent>:   push   %esi
0xc016b34d <nfsd_findparent+1>: push   %ebx
0xc016b34e <nfsd_findparent+2>: mov    0xc(%esp,1),%ebx
0xc016b352 <nfsd_findparent+6>: push   $0xc02ce724
0xc016b357 <nfsd_findparent+11>:        push   %ebx
0xc016b358 <nfsd_findparent+12>:        call   0xc0144608 <d_alloc>
0xc016b35d <nfsd_findparent+17>:        mov    %eax,%esi
0xc016b35f <nfsd_findparent+19>:        add    $0x8,%esp
        tdentry = d_alloc(child, &(const struct qstr) {"..", 2, 0});

0xc016b362 <nfsd_findparent+22>: test %esi,%esi
if (!tdentry)
0xc016b364 <nfsd_findparent+24>: jne 0xc016b370 <nfsd_findparent+36

Quoted 1 line
0xc016b366 <nfsd_findparent+26>:        mov    $0xfffffff4,%eax
0xc016b36b <nfsd_findparent+31>:        jmp    0xc016b431 <nfsd_findparent+22
9>
			return ERR_PTR(-ENOMEM);
0xc016b370 <nfsd_findparent+36>:        mov    0x8(%ebx),%eax
0xc016b373 <nfsd_findparent+39>:        mov    0x84(%eax),%edx
0xc016b379 <nfsd_findparent+45>:        push   %esi
0xc016b37a <nfsd_findparent+46>:        push   %eax
0xc016b37b <nfsd_findparent+47>:        mov    0x4(%edx),%eax
0xc016b37e <nfsd_findparent+50>:        call   *%eax
0xc016b380 <nfsd_findparent+52>:        mov    %eax,%ebx
0xc016b382 <nfsd_findparent+54>:        add    $0x8,%esp
        pdentry = child->d_inode->i_op->lookup(child->d_inode, tdentry);

static __inline__ void d_drop(struct dentry * dentry)

{
        spin_lock(&dcache_lock);
        list_del(&dentry->d_hash);
        INIT_LIST_HEAD(&dentry->d_hash);
        spin_unlock(&dcache_lock);
}		
0xc016b385 <nfsd_findparent+57>:        lock decb 0xc0329898
0xc016b38c <nfsd_findparent+64>:        js     0xc02aefa4 <stext_lock+12548>
0xc016b392 <nfsd_findparent+70>:        lea    0x18(%esi),%eax
0xc016b395 <nfsd_findparent+73>:        mov    0x4(%eax),%ecx
0xc016b398 <nfsd_findparent+76>:        mov    0x18(%esi),%edx
0xc016b39b <nfsd_findparent+79>:        mov    %ecx,0x4(%edx)
0xc016b39e <nfsd_findparent+82>:        mov    %edx,(%ecx)
0xc016b3a0 <nfsd_findparent+84>:        mov    %eax,0x18(%esi)
0xc016b3a3 <nfsd_findparent+87>:        mov    %eax,0x1c(%esi)
0xc016b3a6 <nfsd_findparent+90>:        movb   $0x1,0xc0329898

d_drop(tdentry); /* we never want ".." hashed */

0xc016b3ad <nfsd_findparent+97>:        test   %ebx,%ebx
0xc016b3af <nfsd_findparent+99>:        jne    0xc016b426 <nfsd_findparent+21
8>
        if (!pdentry) {
0xc016b3b1 <nfsd_findparent+101>:       mov    0x8(%esi),%eax
0xc016b3b4 <nfsd_findparent+104>:       lea    0x10(%eax),%edx   // edx holds
 aliases
                struct list_head *aliases = &tdentry->d_inode->i_dentry; 
				
		// is tdentry->d_inode->i_dentry not valid anymore?  aliases ge

ts NULL

0xc016b3b7 <nfsd_findparent+107>: lock decb 0xc0329898
0xc016b3be <nfsd_findparent+114>: js 0xc02aefb4 <stext_lock+12564>

                spin_lock(&dcache_lock);

0xc016b3c4 <nfsd_findparent+120>: mov 0x10(%eax),%eax <==========
===========
CRASH !!!
0xc016b3c7 <nfsd_findparent+123>: cmp %edx,%eax // eax holds alias
es->next

                if (aliases->next != aliases) { 

0xc016b3c9 <nfsd_findparent+125>: je 0xc016b3ec <nfsd_findparent+16
0>
0xc016b3cb <nfsd_findparent+127>: lea 0xffffffc8(%eax),%ebx

                        pdentry = list_entry(aliases->next, struct dentry, d_

alias);

0xc016b3ce <nfsd_findparent+130>:       cmp    %esi,%ebx
0xc016b3d0 <nfsd_findparent+132>:       jne    0xc016b3d8 <nfsd_findparent+14
0>
0xc016b3d2 <nfsd_findparent+134>:       mov    0x4(%edx),%ebx
0xc016b3d5 <nfsd_findparent+137>:       add    $0xffffffc8,%ebx
0xc016b3d8 <nfsd_findparent+140>:       xor    %eax,%eax
0xc016b3da <nfsd_findparent+142>:       cmp    %esi,%ebx
0xc016b3dc <nfsd_findparent+144>:       cmove  %eax,%ebx
0xc016b3df <nfsd_findparent+147>:       test   %ebx,%ebx
0xc016b3e1 <nfsd_findparent+149>:       je     0xc016b3ec <nfsd_findparent+16
0>
0xc016b3e3 <nfsd_findparent+151>:       push   %ebx
0xc016b3e4 <nfsd_findparent+152>:       call   0xc0144080 <dget_locked>
0xc016b3e9 <nfsd_findparent+157>:       add    $0x4,%esp
                        if (pdentry) dget_locked(pdentry);  
0xc016b3ec <nfsd_findparent+160>:       movb   $0x1,0xc0329898
0xc016b3f3 <nfsd_findparent+167>:       test   %ebx,%ebx
0xc016b3f5 <nfsd_findparent+169>:       jne    0xc016b41c <nfsd_findparent+20
8>
                spin_unlock(&dcache_lock);
0xc016b3f7 <nfsd_findparent+171>:       mov    0x8(%esi),%eax
0xc016b3fa <nfsd_findparent+174>:       push   %eax
0xc016b3fb <nfsd_findparent+175>:       call   0xc0145ee8 <igrab>
0xc016b400 <nfsd_findparent+180>:       push   %eax
0xc016b401 <nfsd_findparent+181>:       call   0xc01447f4 <d_alloc_root>
0xc016b406 <nfsd_findparent+186>:       mov    %eax,%ebx
0xc016b408 <nfsd_findparent+188>:       add    $0x8,%esp
                        pdentry = d_alloc_root(igrab(tdentry->d_inode));
0xc016b40b <nfsd_findparent+191>:       test   %ebx,%ebx
0xc016b40d <nfsd_findparent+193>:       je     0xc016b41c <nfsd_findparent+20
8>
0xc016b40f <nfsd_findparent+195>:       orb    $0x4,0x4(%ebx)
0xc016b413 <nfsd_findparent+199>:       push   %ebx
0xc016b414 <nfsd_findparent+200>:       call   0xc0144ab4 <d_rehash>
0xc016b419 <nfsd_findparent+205>:       add    $0x4,%esp
0xc016b41c <nfsd_findparent+208>:       mov    $0xfffffff4,%eax
0xc016b421 <nfsd_findparent+213>:       test   %ebx,%ebx
0xc016b423 <nfsd_findparent+215>:       cmove  %eax,%ebx
0xc016b426 <nfsd_findparent+218>:       push   %esi
        }  /* if (!pdentry) */
0xc016b427 <nfsd_findparent+219>:       call   0xc0143e90 <dput>
0xc016b42c <nfsd_findparent+224>:       mov    %ebx,%eax
0xc016b42e <nfsd_findparent+226>:       add    $0x4,%esp
0xc016b431 <nfsd_findparent+229>:       pop    %ebx
0xc016b432 <nfsd_findparent+230>:       pop    %esi
0xc016b433 <nfsd_findparent+231>:       ret

End of assembler dump.
(gdb)

2.4.3-xfs is compiled with gcc 2.95.2
2.4.9-XFS is compiled with gcc 2.91.6

After the crash I tried to reboot, but it quickly failed again.

reboot   system boot  2.4.3-XFS        Mon Oct 15 18:20          (21:49)
reboot   system boot  2.4.3-XFS        Mon Oct 15 18:17          (21:52)
reboot   system boot  2.4.3-XFS        Mon Oct 15 18:13          (21:57)
reboot   system boot  2.4.3-XFS        Thu Oct 11 16:51         (4+23:18)
reboot   system boot  2.4.3-XFS        Thu Oct 11 16:32         (4+23:37)
reboot   system boot  2.4.3-XFS        Thu Oct 11 16:28         (4+23:41)
reboot   system boot  2.4.3-XFS        Thu Oct 11 16:23         (4+23:46)
reboot   system boot  2.4.3-XFS        Thu Oct 11 16:12         (4+23:57)
reboot   system boot  2.4.3-XFS        Thu Oct 11 15:05         (5+01:04)
reboot   system boot  2.4.9-xfs        Thu Oct  4 17:02         (11+23:08)
reboot   system boot  2.4.9-xfs        Thu Oct  4 16:54         (11+23:15)
reboot   system boot  2.4.3-XFS        Thu Oct  4 16:47         (11+23:23)
reboot   system boot  2.4.9-xfs        Thu Oct  4 16:39         (11+23:31)
reboot   system boot  2.4.9-xfs        Thu Oct  4 16:14         (11+23:55)

To recover I had to unplug/powerdown all connected clients. :-( Uhh. I think 2.4.9 crashed at the same instructions. But I can't reproduce it on demand (hey, in the meantime we can work!).
Seeing the NFS related changes in 2.4.10: should I upgrade? We use only NFSv2. Is NFSv3 more stable, if that matters here?

The machine with 2.4.3 was up for several months - with light load. Now the number of crashes (with 6 NFS clients using /home + cvs) went up.

We do mount our compile environment in a strange way:

NOTE: /server is a symlink to /usr/local/export
# See exports(5) for a description.
# This file contains a list of all directories exported to other computers. # It is used by rpc.nfsd and rpc.mountd.

/opt/xxx                        *.xxx(ro)
/home                           *.xxx(rw)
/tmp                            *.xxx(rw)
/usr/local/export               *.xxx(rw)

/etc/fstab
devserv:/server/compileenv/lib/lib \
/compenv/xxx/lib nfs ro,auto 0 0
devserv:/server/compileenv/usr_lib/usr/lib \
/compenv/xxx/usr_lib nfs ro,auto 0 0
devserv:/server/compileenv/gcc-lib \
/compenv/xxx/gcc-lib nfs ro,exec,auto 0 0
devserv:/server/compileenv/usr_include/usr/include \
/compenv/xxx/usr_include nfs ro,auto 0 0
devserv:/server/compileenv/usr_linux/usr/linux/include \
/compenv/xxx/usr_linux_include nfs ro,auto 0 0

Would it help to mount a single /usr/local/export and work with a symlink tree (as I actually do with autofs) ?
What can cause the "invalid" dentries? Did someone remove some files or were some dentries de'hashed because of dcache growth?

As workaround I will do:

                struct list_head *aliases = &tdentry->d_inode->i_dentry;

                if (aliases && (aliases->next != aliases) ) {
                        spin_lock(&dcache_lock);
                        pdentry = list_entry(aliases->next, struct dentry, d_
alias);
                        if (pdentry == tdentry)
                                pdentry = list_entry(aliases->prev, struct de
ntry,
d_alias);
                        if (pdentry == tdentry)
                                pdentry = NULL;
                        if (pdentry) dget_locked(pdentry);
                        spin_unlock(&dcache_lock);
                } else
                     pdentry=NULL;

-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

#2 lore raw
Quoted 5 lines · Steve Lord

Where did you get your kernel (the 2.4.9 version that is) this problem sounds familiar, but I am pretty sure we fixed this case in XFS somewhere between 2.4.3 and 2.4.9.

ftp://oss.sgi.com/projects/xfs/download/patches/linux-2.4.9-xfs-2001-08-26.patch.bz2

the 2.4.3 was there in May, 2001.
-rwxr-xr-x 1 peewee peewee 47713 Mai 17 16:15 linux-2.4.3-core-xfs-1.0.patch.gz

I cannot confirm the Oops in 2.4.9-xfs, but I _think_ it was in the same place. Do you think, it's xfs related? Some dentries vanashing beneath xfs?

What version do you want me to try, 2.4.12-xfs? I will apply my "safety" patch anyway (with a printk) ;-)

Quoted 58 lines
Quoted 58 lines

noisy:/usr/src/linux # uname -a
Linux noisy 2.4.3-XFS #8 SMP Sam Mai 19 18:21:36 CEST 2001 i686 unknown

dual board with just one processor
/usr/local and /home on LogicalVolumes
tried also 2.4.9-XFS (but see below)

------------ oops -------------
Warning (compare_maps): ksyms_base symbol __rta_fill_R__ver___rta_fill not fo und in
System.map. Ignoring ksyms_base entry
[other warnings stripped]

Oops: 0000
CPU:    0
EIP:    0010:<c016b3c4>

Using defaults from ksymoops -t elf32-i386 -a i386
EFLAGS: 00010246
eax: 00000000 ebx: 00000000 ecx: cff805b8 edx: 00000010
esi: ca018260 edi: ca0182e0 ebp: cb9e5800 esp: cb9edeec
ds: 0018 es: 0018 ss: 0018

Stack: 0100708b ca018360 c016b7d8 ca0182e0 00000002 cba07000 cb9efc00 cb9e580
0
       ce078bc4 ca018360 00000000 ca018360 c016bb70 ce078a80 0102a048 0000000
5
Call Trace: <c01bb7d8> <c016bb70> <c016a0b5>c0169b73> <c02a60bf> <c016999d> <
c01054c4>
Code: 8b 40 10 39 d0 74 21 8d 58 c8 39 f3 75 06 8b 5a 04 83 c3 c8
Quoted 1 line
Quoted 1 line

EIP; c016b3c4 <nfsd_findparent+78/e8> <=====

Trace; c01bb7d8 <xfs_buf_item_bits+38/58>
Trace; c016bb70 <fh_verify+26c/48c>
Trace; c016a0b5 <nfsd_proc_getattr+91/98>
Code; c016b3c4 <nfsd_findparent+78/e8>
00000000 <_EIP>:
Code; c016b3c4 <nfsd_findparent+78/e8> <=====
0: 8b 40 10 mov 0x10(%eax),%eax <=====
Code; c016b3c7 <nfsd_findparent+7b/e8>
3: 39 d0 cmp %edx,%eax
Code; c016b3c9 <nfsd_findparent+7d/e8>
5: 74 21 je 28 <_EIP+0x28> c016b3ec <nfsd_findpa
rent+a0/e8>
Code; c016b3cb <nfsd_findparent+7f/e8>
7: 8d 58 c8 lea 0xffffffc8(%eax),%ebx
Code; c016b3ce <nfsd_findparent+82/e8>
a: 39 f3 cmp %esi,%ebx
Code; c016b3d0 <nfsd_findparent+84/e8>
c: 75 06 jne 14 <_EIP+0x14> c016b3d8 <nfsd_findpa
rent+8c/e8>
Code; c016b3d2 <nfsd_findparent+86/e8>
e: 8b 5a 04 mov 0x4(%edx),%ebx
Code; c016b3d5 <nfsd_findparent+89/e8>
11: 83 c3 c8 add $0xffffffc8,%ebx

31 warnings issued. Results may not be reliable.
------------ oops -------------

[...]

#3 lore raw

NET_PROFILE results for 2.4.12 [was sentto is slow]

Hi,
In my previous posts I was thinking why my PII/375 system can't handle more than 25000 packet/s. I repaired kernel's NET_PROFILEr and got an answer.
So that for curious, first gprof output of program doing send/poll/recv of 200 byte packets loop on the loopback interface using PF_PACKET (total running time was 20 sec):

 49.67      9.93     9.93   549384     0.02     0.02  sendto
 17.71     13.47     3.54   549384     0.01     0.01  recv
 12.01     15.87     2.40   553784     0.00     0.00  poll

Now we compare them with kernel profile stats:
af_packet_send: 7876 ms
nested dev_queue_xmit: 3714 ms

net_rx_action:           1464 ms
af_packet_recv:          411  ms

For send gprof tells 9930 ms and kernel 7876 so 2 sec in difference is generic socket and system call handling overhead (and measure error). The recv gproffed is 3540 ms, kernel 1464+411 so there is again about 2 sec overhead.

Quoted 1 line

From this analysis I can finally say that there is not error in my

netflow simulator :) but the 2.4 on PII/375 can handle up to 30000 pps in userspace loop.
Without userspace overhead and not bouncing packets (like in my test) the router based on 2.4 could handle about 70000 pps without qos,routing and filtering overhead ..
Someone tested in real world ?

best regards, devik

#4 lore raw
In reply to Steve Lord
Quoted 5 lines · Steve Lord

Where did you get your kernel (the 2.4.9 version that is) this problem sounds familiar, but I am pretty sure we fixed this case in XFS somewhere between 2.4.3 and 2.4.9.

The following diff was made in 2.4.4.

fs/nfsd/nfsfh.c Sat May 19 17:47:55 2001 renamed from linux/fs/nfsd/nfsfh.c Fri Feb 9 11:29:44 2001 +5 -0
diff -u --recursive --new-file v2.4.4/linux/fs/nfsd/nfsfh.c linux/fs/nfsd/nfsfh.c
@@ -244,6 +245,11 @@
         */
        pdentry = child->d_inode->i_op->lookup(child->d_inode, tdentry);
        d_drop(tdentry); /* we never want ".." hashed */
+       if (!pdentry && tdentry->d_inode == NULL) {
+               /* File system cannot find ".." ... sad but possible */
+               dput(tdentry);
+               pdentry = ERR_PTR(-EINVAL);
+       }
 

But it would not prevent the code path 2.4.3-xfs hit.
pdentry is !=NULL and tdentry->d_inode is always NULL after d_alloc():611

Does xfs' child->d_inode->i_op->lookup(child->d_inode, tdentry); change the contents of tdentry if the lookup("..") succeeds?

Then I suggest:

! if (!pdentry || tdentry->d_inode == NULL) {

What do you think?
BTW, when does a lookup("..") fail? Even in "/", lookup("..") returns "."

Quoted 7 lines
Quoted 7 lines

struct dentry *nfsd_findparent(struct dentry *child)

{
        struct dentry *tdentry, *pdentry;
        tdentry = d_alloc(child, &(const struct qstr) {"..", 2, 0});
        if (!tdentry)
                return ERR_PTR(-ENOMEM);
Quoted 5 lines
Quoted 5 lines
        /* I'm going to assume that if the returned dentry is different, then
         * it is well connected.  But nobody returns different dentrys do they?
         */
        pdentry = child->d_inode->i_op->lookup(child->d_inode, tdentry);
        d_drop(tdentry); /* we never want ".." hashed */

+ if (!pdentry && tdentry->d_inode == NULL) {

+               /* File system cannot find ".." ... sad but possible */
+               dput(tdentry);
+               pdentry = ERR_PTR(-EINVAL);

+ }

Quoted 12 lines
Quoted 12 lines
        if (!pdentry) {
                /* I don't want to return a ".." dentry.
                 * I would prefer to return an unconnected "IS_ROOT" dentry,
                 * though a properly connected dentry is even better
                 */
                /* if first or last of alias list is not tdentry, use that
                 * else make a root dentry
                 */
                struct list_head *aliases = &tdentry->d_inode->i_dentry;
                spin_lock(&dcache_lock);
                if (aliases->next != aliases) {         <=========== CRASH
                        pdentry = list_entry(aliases->next, struct dentry, d_
#5 lore raw
In reply to Steve Lord
Quoted 23 lines · Peter Wächtler
Quoted 5 lines · Steve Lord

Where did you get your kernel (the 2.4.9 version that is) this problem sounds familiar, but I am pretty sure we fixed this case in XFS somewhere between 2.4.3 and 2.4.9.

The following diff was made in 2.4.4.

diff -u --recursive --new-file v2.4.4/linux/fs/nfsd/nfsfh.c linux/fs/nfsd/nfsfh.c

fs/nfsd/nfsfh.c Sat May 19 17:47:55 2001 renamed from linux/fs/nfsd/nfsfh.c Fri Feb 9 11:29:44 2001 +5 -0
@@ -244,6 +245,11 @@
         */
        pdentry = child->d_inode->i_op->lookup(child->d_inode, tdentry);
        d_drop(tdentry); /* we never want ".." hashed */
+       if (!pdentry && tdentry->d_inode == NULL) {
+               /* File system cannot find ".." ... sad but possible */
+               dput(tdentry);
+               pdentry = ERR_PTR(-EINVAL);
+       }
 
Quoted 3 lines

But it would not prevent the code path 2.4.3-xfs hit.
pdentry is !=NULL and tdentry->d_inode is always NULL after d_alloc():611

Damn. pdentry IS NULL.
Sorry, the patch would prevent the crash.

#6 lore raw
Quoted 25 lines
Quoted 11 lines · Peter Wdchtler
Quoted 5 lines · Steve Lord

Where did you get your kernel (the 2.4.9 version that is) this problem sounds familiar, but I am pretty sure we fixed this case in XFS somewhere between 2.4.3 and 2.4.9.

The following diff was made in 2.4.4.

diff -u --recursive --new-file v2.4.4/linux/fs/nfsd/nfsfh.c linux/fs/nfsd/n

fsfh.c

Quoted 12 lines
fs/nfsd/nfsfh.c Sat May 19 17:47:55 2001 renamed from linux/fs/nfsd/nfsfh.c Fri Feb 9 11:29:44 2001 +5 -0
@@ -244,6 +245,11 @@
         */
        pdentry = child->d_inode->i_op->lookup(child->d_inode, tdentry);
        d_drop(tdentry); /* we never want ".." hashed */
+       if (!pdentry && tdentry->d_inode == NULL) {
+               /* File system cannot find ".." ... sad but possible */
+               dput(tdentry);
+               pdentry = ERR_PTR(-EINVAL);
+       }
 
Quoted 6 lines
Quoted 3 lines

But it would not prevent the code path 2.4.3-xfs hit.
pdentry is !=NULL and tdentry->d_inode is always NULL after d_alloc():611

Damn. pdentry IS NULL.
Sorry, the patch would prevent the crash.

OK, I am back again....

I went and looked through xfs code changes, and the 2.4.9 patch should have some changes in it related to this type of issue. However, you would have to be using xfsdump to trigger them I think, or possibly the xfs_fsr program. The combination of xfsdump and an nfs server did have some problems.

So, does this tie in with your configuation? It does not however explain problems in 2.4.9. There was just a recent code cleanup of the code paths used by xfsdump (post 2.4.12 patch), but I am not aware of any other XFS changes in this area.

XFS does work OK with the new VM, so you can try later kernels if you want, I am not aware of anyone else having problems with XFS which appear dcache related.

Steve (slightly stumped and in need of coffee)

Quoted 6 lines

-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/