cifs: Fix cifsInodeInfo lock_sem deadlock when reconnect occurs
Subsystems (2)
PATCH 1/1 cifs: Fix cifsInodeInfo lock_sem deadlock when reconnect occurs
There's a deadlock that is possible and can easily be seen with a test where multiple readers open/read/close of the same file and a disruption occurs causing reconnect. The deadlock is due a reader thread inside cifs_strict_readv calling down_read and obtaining lock_sem, and then after reconnect inside cifs_reopen_file calling down_read a second time. If in between the two down_read calls, a down_write comes from another process, deadlock occurs.
CPU0 CPU1
---- ----
cifs_strict_readv()
down_read(&cifsi->lock_sem);
_cifsFileInfo_put
OR
cifs_new_fileinfo
down_write(&cifsi->lock_sem);cifs_reopen_file()
down_read(&cifsi->lock_sem);
Fix the above by changing all down_write(lock_sem) calls to down_write_trylock(lock_sem)/msleep() loop, which in turn makes the second down_read call benign since it will never block behind the writer while holding lock_sem.
--- fs/cifs/cifsglob.h | 5 +++++ fs/cifs/file.c | 24 ++++++++++++++++-------- fs/cifs/smb2file.c | 3 ++- 3 files changed, 23 insertions(+), 9 deletions(-)
fs/cifs/cifsglob.h +5 -0
@@ -1391,6 +1391,11 @@ struct cifs_writedata {struct cifsInodeInfo {bool can_cache_brlcks;struct list_head llist;/* locks helb by this inode */ +/* +* NOTE: Some code paths call down_read(lock_sem) twice, so +* we must always use use down_write_trylock()/msleep() loop +* to avoid deadlock. +*/ struct rw_semaphore lock_sem;/* protect the fields above */ /* BB add in lists for dirty pages i.e. write caching info for oplock */ struct list_head openFileList;
fs/cifs/file.c +16 -8
@@ -306,7 +306,8 @@ struct cifsFileInfo *INIT_LIST_HEAD (&fdlocks->locks); fdlocks->cfile = cfile; cfile->llist = fdlocks; -down_write (&cinode->lock_sem); +while (!down_write_trylock (&cinode->lock_sem)) +msleep (125 );list_add (&fdlocks->llist, &cinode->llist);up_write (&cinode->lock_sem);
@@ -464,7 +465,8 @@ void _cifsFileInfo_put(struct cifsFileInfo *cifs_file, bool wait_oplock_handler)* Delete any outstanding lock records. We'll lose them when the file * is closed anyway. */ -down_write (&cifsi->lock_sem); +while (!down_write_trylock (&cifsi->lock_sem)) +msleep (125 );list_for_each_entry_safe (li, tmp, &cifs_file->llist->locks, llist) {list_del (&li->llist);cifs_del_lock_waiters (li);
@@ -1027,7 +1029,8 @@ int cifs_closedir(struct inode *inode, struct file *file) cifs_lock_add(struct cifsFileInfo *cfile, struct cifsLockInfo *lock) { struct cifsInodeInfo *cinode = CIFS_I(d_inode(cfile->dentry)); -down_write (&cinode->lock_sem); +while (!down_write_trylock (&cinode->lock_sem)) +msleep (125 );list_add_tail (&lock->llist, &cfile->llist->locks);up_write (&cinode->lock_sem); }
@@ -1049,7 +1052,8 @@ int cifs_closedir(struct inode *inode, struct file *file) try_again: exist = false; -down_write (&cinode->lock_sem); +while (!down_write_trylock (&cinode->lock_sem)) +msleep (125 ); exist = cifs_find_lock_conflict(cfile, lock->offset, lock->length, lock->type, lock->flags, &conf_lock,
@@ -1072,7 +1076,8 @@ int cifs_closedir(struct inode *inode, struct file *file) (lock->blist.next == &lock->blist));if (!rc) goto try_again; -down_write (&cinode->lock_sem); +while (!down_write_trylock (&cinode->lock_sem)) +msleep (125 );list_del_init (&lock->blist); }
@@ -1125,7 +1130,8 @@ int cifs_closedir(struct inode *inode, struct file *file)return rc; try_again: -down_write (&cinode->lock_sem); +while (!down_write_trylock (&cinode->lock_sem)) +msleep (125 );if (!cinode->can_cache_brlcks) {up_write (&cinode->lock_sem);return rc;
@@ -1331,7 +1337,8 @@ struct lock_to_push {int rc =0 ;/* we are going to update can_cache_brlcks here - need a write access */ -down_write (&cinode->lock_sem); +while (!down_write_trylock (&cinode->lock_sem)) +msleep (125 );if (!cinode->can_cache_brlcks) {up_write (&cinode->lock_sem);return rc;
@@ -1522,7 +1529,8 @@ struct lock_to_push {if (!buf)return -ENOMEM ; -down_write (&cinode->lock_sem); +while (!down_write_trylock (&cinode->lock_sem)) +msleep (125 );for (i =0 ; i <2 ; i++) { cur = buf; num =0 ;
fs/cifs/smb2file.c +2 -1
@@ -145,7 +145,8 @@ cur = buf; -down_write (&cinode->lock_sem); +while (!down_write_trylock (&cinode->lock_sem)) +msleep (125 ); list_for_each_entry_safe(li, tmp, &cfile->llist->locks, llist) { if (flock->fl_start > li->offset || (flock->fl_start + length) <
Signature · 1 line
1.8.3.1
Reviewed-by me
----- Original Message -----
From: "Dave Wysochanski" <[email protected]>
To: [email protected]
Sent: Thursday, 24 October, 2019 12:08:22 PM
Subject: [PATCH] cifs: Fix cifsInodeInfo lock_sem deadlock when reconnect occurs
There's a deadlock that is possible and can easily be seen with a test where multiple readers open/read/close of the same file and a disruption occurs causing reconnect. The deadlock is due a reader thread inside cifs_strict_readv calling down_read and obtaining lock_sem, and then after reconnect inside cifs_reopen_file calling down_read a second time. If in between the two down_read calls, a down_write comes from another process, deadlock occurs.
CPU0 CPU1
---- ----
cifs_strict_readv()
down_read(&cifsi->lock_sem);
_cifsFileInfo_put
OR
cifs_new_fileinfo
down_write(&cifsi->lock_sem);cifs_reopen_file()
down_read(&cifsi->lock_sem);
Fix the above by changing all down_write(lock_sem) calls to down_write_trylock(lock_sem)/msleep() loop, which in turn makes the second down_read call benign since it will never block behind the writer while holding lock_sem.
--- fs/cifs/cifsglob.h | 5 +++++ fs/cifs/file.c | 24 ++++++++++++++++-------- fs/cifs/smb2file.c | 3 ++- 3 files changed, 23 insertions(+), 9 deletions(-)
fs/cifs/cifsglob.h +5 -0
@@ -1391,6 +1391,11 @@ struct cifs_writedata {struct cifsInodeInfo {bool can_cache_brlcks;struct list_head llist;/* locks helb by this inode */ +/* +* NOTE: Some code paths call down_read(lock_sem) twice, so +* we must always use use down_write_trylock()/msleep() loop +* to avoid deadlock. +*/ struct rw_semaphore lock_sem;/* protect the fields above */ /* BB add in lists for dirty pages i.e. write caching info for oplock */ struct list_head openFileList;
fs/cifs/file.c +16 -8
@@ -306,7 +306,8 @@ struct cifsFileInfo *INIT_LIST_HEAD (&fdlocks->locks); fdlocks->cfile = cfile; cfile->llist = fdlocks; -down_write (&cinode->lock_sem); +while (!down_write_trylock (&cinode->lock_sem)) +msleep (125 );list_add (&fdlocks->llist, &cinode->llist);up_write (&cinode->lock_sem);
@@ -464,7 +465,8 @@ void _cifsFileInfo_put(struct cifsFileInfo *cifs_file, bool wait_oplock_handler)* Delete any outstanding lock records. We'll lose them when the file * is closed anyway. */ -down_write (&cifsi->lock_sem); +while (!down_write_trylock (&cifsi->lock_sem)) +msleep (125 );list_for_each_entry_safe (li, tmp, &cifs_file->llist->locks, llist) {list_del (&li->llist);cifs_del_lock_waiters (li);
@@ -1027,7 +1029,8 @@ int cifs_closedir(struct inode *inode, struct file *file) cifs_lock_add(struct cifsFileInfo *cfile, struct cifsLockInfo *lock) { struct cifsInodeInfo *cinode = CIFS_I(d_inode(cfile->dentry)); -down_write (&cinode->lock_sem); +while (!down_write_trylock (&cinode->lock_sem)) +msleep (125 );list_add_tail (&lock->llist, &cfile->llist->locks);up_write (&cinode->lock_sem); }
@@ -1049,7 +1052,8 @@ int cifs_closedir(struct inode *inode, struct file *file) try_again: exist = false; -down_write (&cinode->lock_sem); +while (!down_write_trylock (&cinode->lock_sem)) +msleep (125 ); exist = cifs_find_lock_conflict(cfile, lock->offset, lock->length, lock->type, lock->flags, &conf_lock,
@@ -1072,7 +1076,8 @@ int cifs_closedir(struct inode *inode, struct file *file) (lock->blist.next == &lock->blist));if (!rc) goto try_again; -down_write (&cinode->lock_sem); +while (!down_write_trylock (&cinode->lock_sem)) +msleep (125 );list_del_init (&lock->blist); }
@@ -1125,7 +1130,8 @@ int cifs_closedir(struct inode *inode, struct file *file)return rc; try_again: -down_write (&cinode->lock_sem); +while (!down_write_trylock (&cinode->lock_sem)) +msleep (125 );if (!cinode->can_cache_brlcks) {up_write (&cinode->lock_sem);return rc;
@@ -1331,7 +1337,8 @@ struct lock_to_push {int rc =0 ;/* we are going to update can_cache_brlcks here - need a write access */ -down_write (&cinode->lock_sem); +while (!down_write_trylock (&cinode->lock_sem)) +msleep (125 );if (!cinode->can_cache_brlcks) {up_write (&cinode->lock_sem);return rc;
@@ -1522,7 +1529,8 @@ struct lock_to_push {if (!buf)return -ENOMEM ; -down_write (&cinode->lock_sem); +while (!down_write_trylock (&cinode->lock_sem)) +msleep (125 );for (i =0 ; i <2 ; i++) { cur = buf; num =0 ;
fs/cifs/smb2file.c +2 -1
@@ -145,7 +145,8 @@ cur = buf; -down_write (&cinode->lock_sem); +while (!down_write_trylock (&cinode->lock_sem)) +msleep (125 ); list_for_each_entry_safe(li, tmp, &cfile->llist->locks, llist) { if (flock->fl_start > li->offset || (flock->fl_start + length) <
Signature · 1 line
1.8.3.1
чт, 24 окт. 2019 г. в 04:04, Dave Wysochanski <[email protected]>:
Quoted 65 lines
There's a deadlock that is possible and can easily be seen with a test where multiple readers open/read/close of the same file and a disruption occurs causing reconnect. The deadlock is due a reader thread inside
cifs_strict_readvcallingdown_readand obtaininglock_sem, and then after reconnect insidecifs_reopen_filecallingdown_reada second time. If in between the twodown_readcalls, adown_writecomes from another process, deadlock occurs.CPU0 CPU1 ---- ---- cifs_strict_readv() down_read(&cifsi->lock_sem); _cifsFileInfo_put OR cifs_new_fileinfo down_write(&cifsi->lock_sem);
cifs_reopen_file()
down_read(&cifsi->lock_sem);Fix the above by changing all
down_write(lock_sem)calls to down_write_trylock(lock_sem)/msleep() loop, which in turn makes the seconddown_readcall benign since it will never block behind the writer while holdinglock_sem.Signed-off-by Dave Wysochanski <[email protected]>Suggested-by Ronnie Sahlberg <[email protected]>--- fs/cifs/cifsglob.h | 5 +++++ fs/cifs/file.c | 24 ++++++++++++++++-------- fs/cifs/smb2file.c | 3 ++- 3 files changed, 23 insertions(+), 9 deletions(-)fs/cifs/cifsglob.h +5
-0@@ -1391,6 +1391,11 @@ struct cifs_writedata {struct cifsInodeInfo {bool can_cache_brlcks;struct list_head llist;/* locks helb by this inode */ +/* +* NOTE: Some code paths call down_read(lock_sem) twice, so +* we must always use use down_write_trylock()/msleep() loop +* to avoid deadlock. +*/ struct rw_semaphore lock_sem;/* protect the fields above */ /* BB add in lists for dirty pages i.e. write caching info for oplock */ struct list_head openFileList;fs/cifs/file.c +2
-1@@ -306,7 +306,8 @@ struct cifsFileInfo *INIT_LIST_HEAD (&fdlocks->locks); fdlocks->cfile = cfile; cfile->llist = fdlocks; -down_write (&cinode->lock_sem); +while (!down_write_trylock (&cinode->lock_sem)) +msleep (125 );list_add (&fdlocks->llist, &cinode->llist);up_write (&cinode->lock_sem);
fs/cifs/file.c void _cifsFileInfo_put(struct cifsFileInfo *cifs_file, bool wait_oplock_handler) · line 469
Quoted 6 lines
fs/cifs/file.c +2
-1@@ -464,7 +465,8 @@ void _cifsFileInfo_put(struct cifsFileInfo *cifs_file, bool wait_oplock_handler)* Delete any outstanding lock records. We'll lose them when the file * is closed anyway. */ -down_write (&cifsi->lock_sem); +while (!down_write_trylock (&cifsi->lock_sem)) +msleep (125 );
Please wrap the above code into a helper function e.g. cifs_acquire_lock_sem(struct cifsInodeInfo *cinode) or any other name you like.
Other than that it looks good.
--
Best regards,
Pavel Shilovsky