Skip to contentSkip to conversation

xenstore: Fix deadlock in xs_read_watch

PATCH In review Daniel De Graaf 2 participants 5 messages Aug 26, 2010 – Aug 30, 2010

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 xenstore: Fix deadlock in xs_read_watch

When read_message returns -1 while a read is pending, an attempt is made to lock h->request_mutex which is already locked by the reader. Change the read thread to only exit on thread cancellation, where read_thr_exists will return 0.

Signature · 2 lines
Daniel De Graaf
National Security Agency
tools/xenstore/xs.c +4 -8
diff -r 6bebaf40e925 tools/xenstore/xs.c
@@ -271,6 +271,7 @@
 {
 #ifdef USE_PTHREAD
 	if (h->read_thr_exists) {
+		h->read_thr_exists = 0;
 		pthread_cancel(h->read_thr);
 		pthread_join(h->read_thr, NULL);
 	}
@@ -667,7 +668,7 @@
 	mutex_lock(&h->watch_mutex);
 
 	/* Wait on the condition variable for a watch to fire. */
-	while (list_empty(&h->watch_list) && read_thread_exists(h))
+	while (list_empty(&h->watch_list) && (!read_from_thread || read_thread_exists(h)))
 		condvar_wait(&h->watch_condvar, &h->watch_mutex, h);
 	if (!read_thread_exists(h)) {
 		mutex_unlock(&h->watch_mutex);
@@ -966,13 +967,8 @@
 {
 	struct xs_handle *h = arg;
 
-	while (read_message(h) != -1)
-		continue;
-
-	/* Kick anyone waiting for a reply */
-	pthread_mutex_lock(&h->request_mutex);
-	h->read_thr_exists = 0;
-	pthread_mutex_unlock(&h->request_mutex);
+	while (h->read_thr_exists)
+		read_message(h);
 
 	pthread_mutex_lock(&h->reply_mutex);
 	pthread_cond_signal(&h->reply_condvar);

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#1 lore raw
Quoted 19 lines · Daniel De Graaf

When read_message returns -1 while a read is pending, an attempt is made to lock h->request_mutex which is already locked by the reader. Change the read thread to only exit on thread cancellation, where read_thr_exists will return 0.

diff -r 6bebaf40e925 tools/xenstore/xs.c

tools/xenstore/xs.c +1 -0
@@ -271,6 +271,7 @@
 {
 #ifdef USE_PTHREAD
 	if (h->read_thr_exists) {
+		h->read_thr_exists = 0;
 		pthread_cancel(h->read_thr);
 		pthread_join(h->read_thr, NULL);
 	}

tools/xenstore/xs.c line 674

Quoted 8 lines
tools/xenstore/xs.c +1 -1
@@ -667,7 +668,7 @@
 	mutex_lock(&h->watch_mutex);
 
 	/* Wait on the condition variable for a watch to fire. */
-	while (list_empty(&h->watch_list) && read_thread_exists(h))
+	while (list_empty(&h->watch_list) && (!read_from_thread || read_thread_exists(h)))
 		condvar_wait(&h->watch_condvar, &h->watch_mutex, h);
 	if (!read_thread_exists(h)) {
 		mutex_unlock(&h->watch_mutex);

read_from_thread is not declared in this function

#2 lore raw
Quoted 22 lines · Stefano Stabellini
Quoted 21 lines · Daniel De Graaf

When read_message returns -1 while a read is pending, an attempt is made to lock h->request_mutex which is already locked by the reader. Change the read thread to only exit on thread cancellation, where read_thr_exists will return 0.

diff -r 6bebaf40e925 tools/xenstore/xs.c

tools/xenstore/xs.c +1 -0
@@ -271,6 +271,7 @@
 {
 #ifdef USE_PTHREAD
 	if (h->read_thr_exists) {
+		h->read_thr_exists = 0;
 		pthread_cancel(h->read_thr);
 		pthread_join(h->read_thr, NULL);
 	}
@@ -667,7 +668,7 @@
 	mutex_lock(&h->watch_mutex);
 
Quoted 9 lines

tools/xenstore/xs.c line 674

Quoted 6 lines
tools/xenstore/xs.c +1 -1
 	/* Wait on the condition variable for a watch to fire. */
-	while (list_empty(&h->watch_list) && read_thread_exists(h))
+	while (list_empty(&h->watch_list) && (!read_from_thread || read_thread_exists(h)))
 		condvar_wait(&h->watch_condvar, &h->watch_mutex, h);
 	if (!read_thread_exists(h)) {
 		mutex_unlock(&h->watch_mutex);

read_from_thread is not declared in this function

Sorry about that, looks like I picked the wrong patch to send. Corrected patch attached.

tools/xenstore/xs.c +9 -7
diff -r 3c4c3d48a835 tools/xenstore/xs.c
@@ -271,6 +271,7 @@
 {
 #ifdef USE_PTHREAD
 	if (h->read_thr_exists) {
+		h->read_thr_exists = 0;
 		pthread_cancel(h->read_thr);
 		pthread_join(h->read_thr, NULL);
 	}
@@ -666,9 +667,15 @@
 
 	mutex_lock(&h->watch_mutex);
 
+#ifdef USE_PTHREAD
 	/* Wait on the condition variable for a watch to fire. */
 	while (list_empty(&h->watch_list) && read_thread_exists(h))
 		condvar_wait(&h->watch_condvar, &h->watch_mutex, h);
+#else
+	while (list_empty(&h->watch_list))
+		read_message(h);
+#endif
+	
 	if (!read_thread_exists(h)) {
 		mutex_unlock(&h->watch_mutex);
 		errno = EINVAL;
@@ -966,13 +973,8 @@
 {
 	struct xs_handle *h = arg;
 
-	while (read_message(h) != -1)
-		continue;
-
-	/* Kick anyone waiting for a reply */
-	pthread_mutex_lock(&h->request_mutex);
-	h->read_thr_exists = 0;
-	pthread_mutex_unlock(&h->request_mutex);
+	while (h->read_thr_exists)
+		read_message(h);
 
 	pthread_mutex_lock(&h->reply_mutex);
 	pthread_cond_signal(&h->reply_condvar);

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#3 lore raw
Quoted 33 lines · Daniel De Graaf
Quoted 22 lines · Stefano Stabellini
Quoted 21 lines · Daniel De Graaf

When read_message returns -1 while a read is pending, an attempt is made to lock h->request_mutex which is already locked by the reader. Change the read thread to only exit on thread cancellation, where read_thr_exists will return 0.

diff -r 6bebaf40e925 tools/xenstore/xs.c

tools/xenstore/xs.c +1 -0
@@ -271,6 +271,7 @@
 {
 #ifdef USE_PTHREAD
 	if (h->read_thr_exists) {
+		h->read_thr_exists = 0;
 		pthread_cancel(h->read_thr);
 		pthread_join(h->read_thr, NULL);
 	}
@@ -667,7 +668,7 @@
 	mutex_lock(&h->watch_mutex);
 
Quoted 9 lines

tools/xenstore/xs.c line 680

Quoted 6 lines
tools/xenstore/xs.c +1 -1
 	/* Wait on the condition variable for a watch to fire. */
-	while (list_empty(&h->watch_list) && read_thread_exists(h))
+	while (list_empty(&h->watch_list) && (!read_from_thread || read_thread_exists(h)))
 		condvar_wait(&h->watch_condvar, &h->watch_mutex, h);
 	if (!read_thread_exists(h)) {
 		mutex_unlock(&h->watch_mutex);

read_from_thread is not declared in this function

Quoted 3 lines

Sorry about that, looks like I picked the wrong patch to send. Corrected patch attached.

According to the description you gave, this patch fixes a problem occurring when read_message returns -1 while a read is pending in read_thread that is relevant only when USE_PTHREAD.
Why are you changing a code path that is only compiled when !USE_PTHREAD?
Also shouldn't you be checking the return value of read_message anyway? Considering that h->read_thr_exists is set to zero only on xs_daemon_close, and in that case pthread_cancel is called right after, I doubt that any of the operation run in read_thread after the loop will be executed.

#4 lore raw
Quoted 38 lines · Stefano Stabellini
Quoted 33 lines · Daniel De Graaf
Quoted 22 lines · Stefano Stabellini
Quoted 21 lines · Daniel De Graaf

When read_message returns -1 while a read is pending, an attempt is made to lock h->request_mutex which is already locked by the reader. Change the read thread to only exit on thread cancellation, where read_thr_exists will return 0.

diff -r 6bebaf40e925 tools/xenstore/xs.c

tools/xenstore/xs.c +1 -0
@@ -271,6 +271,7 @@
 {
 #ifdef USE_PTHREAD
 	if (h->read_thr_exists) {
+		h->read_thr_exists = 0;
 		pthread_cancel(h->read_thr);
 		pthread_join(h->read_thr, NULL);
 	}
@@ -667,7 +668,7 @@
 	mutex_lock(&h->watch_mutex);
 
Quoted 9 lines

tools/xenstore/xs.c line 680

Quoted 6 lines
tools/xenstore/xs.c +1 -1
 	/* Wait on the condition variable for a watch to fire. */
-	while (list_empty(&h->watch_list) && read_thread_exists(h))
+	while (list_empty(&h->watch_list) && (!read_from_thread || read_thread_exists(h)))
 		condvar_wait(&h->watch_condvar, &h->watch_mutex, h);
 	if (!read_thread_exists(h)) {
 		mutex_unlock(&h->watch_mutex);

read_from_thread is not declared in this function

Quoted 3 lines

Sorry about that, looks like I picked the wrong patch to send. Corrected patch attached.

Quoted 11 lines

According to the description you gave, this patch fixes a problem occurring when read_message returns -1 while a read is pending in read_thread that is relevant only when USE_PTHREAD.
Why are you changing a code path that is only compiled when !USE_PTHREAD?
Also shouldn't you be checking the return value of read_message anyway? Considering that h->read_thr_exists is set to zero only on xs_daemon_close, and in that case pthread_cancel is called right after, I doubt that any of the operation run in read_thread after the loop will be executed.

The value of read_message is not consistently checked in existing code. I have submitted a separate patch that addresses this and also fixes the deadlock, so neither of the patches in this thread are required.

Signature · 2 lines
Daniel De Graaf
National Security Agency