Skip to contentSkip to conversation

svc_run: make sure only one svc_run loop runs in one process

PATCH In review [email protected] 3 participants 6 messages Apr 9, 2019 – Jun 13, 2019

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 svc_run: make sure only one svc_run loop runs in one process

From: Xiubo Li <[email protected]>

In gluster-block project and there are 2 separate threads, both of which will run the svc_run loop, this could work well in glibc version, but in libtirpc we are hitting the random crash and stuck issues.

More detail please see:
https://github.com/gluster/gluster-block/pull/182

---
 src/svc_run.c | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
src/svc_run.c +19 -0
index f40314b..b295755 100644
@@ -38,12 +38,17 @@
 #include <string.h>
 #include <unistd.h>
 #include <sys/poll.h>
+#include <syslog.h>
+#include <stdbool.h>
 
 
 #include <rpc/rpc.h>
 #include "rpc_com.h"
 #include <sys/select.h>
 
+static bool svc_loop_running = false;
+static pthread_mutex_t svc_run_lock = PTHREAD_MUTEX_INITIALIZER;
+
 void
 svc_run()
 {
@@ -51,6 +56,16 @@ svc_run()
   struct pollfd *my_pollfd = NULL;
   int last_max_pollfd = 0;
 
+  pthread_mutex_lock(&svc_run_lock);
+  if (svc_loop_running) {
+    pthread_mutex_unlock(&svc_run_lock);
+    syslog (LOG_ERR, "svc_run: svc loop is already running in current process %d", getpid());
+    return;
+  }
+
+  svc_loop_running = true;
+  pthread_mutex_unlock(&svc_run_lock);
+
   for (;;) {
     int max_pollfd = svc_max_pollfd;
     if (max_pollfd == 0 && svc_pollfd == NULL)
@@ -111,4 +126,8 @@ svc_exit()
 	svc_pollfd = NULL;
 	svc_max_pollfd = 0;
 	rwlock_unlock(&svc_fd_lock);
+
+    pthread_mutex_lock(&svc_run_lock);
+    svc_loop_running = false;
+    pthread_mutex_unlock(&svc_run_lock);
 }
Signature · 1 line
1.8.3.1
author
#1 lore raw

Hey ping.

What's the state of this patch and will it make sense here?

Thanks
BRs

Quoted 63 lines · On 2019/4/9 19:37, [email protected] wrote:

From: Xiubo Li <[email protected]>

In gluster-block project and there are 2 separate threads, both of which will run the svc_run loop, this could work well in glibc version, but in libtirpc we are hitting the random crash and stuck issues.

More detail please see:
https://github.com/gluster/gluster-block/pull/182

---
  src/svc_run.c | 19 +++++++++++++++++++
  1 file changed, 19 insertions(+)
src/svc_run.c +19 -0
index f40314b..b295755 100644
@@ -38,12 +38,17 @@
  #include <string.h>
  #include <unistd.h>
  #include <sys/poll.h>
+#include <syslog.h>
+#include <stdbool.h>
 
 
  #include <rpc/rpc.h>
  #include "rpc_com.h"
  #include <sys/select.h>
 
+static bool svc_loop_running = false;
+static pthread_mutex_t svc_run_lock = PTHREAD_MUTEX_INITIALIZER;
+
  void
  svc_run()
  {
@@ -51,6 +56,16 @@ svc_run()
    struct pollfd *my_pollfd = NULL;
    int last_max_pollfd = 0;
 
+  pthread_mutex_lock(&svc_run_lock);
+  if (svc_loop_running) {
+    pthread_mutex_unlock(&svc_run_lock);
+    syslog (LOG_ERR, "svc_run: svc loop is already running in current process %d", getpid());
+    return;
+  }
+
+  svc_loop_running = true;
+  pthread_mutex_unlock(&svc_run_lock);
+
    for (;;) {
      int max_pollfd = svc_max_pollfd;
      if (max_pollfd == 0 && svc_pollfd == NULL)
@@ -111,4 +126,8 @@ svc_exit()
  	svc_pollfd = NULL;
  	svc_max_pollfd = 0;
  	rwlock_unlock(&svc_fd_lock);
+
+    pthread_mutex_lock(&svc_run_lock);
+    svc_loop_running = false;
+    pthread_mutex_unlock(&svc_run_lock);
  }
#2 lore raw

Sorry for the delay....

Quoted 3 lines · Xiubo Li

Hey ping.

What's the state of this patch and will it make sense here?

I'm not sure it does make sense.... Shouldn't the mutex lock be in the call of svc_run()?

steved.

Quoted 72 lines

Thanks
BRs

Quoted 51 lines · On 2019/4/9 19:37, [email protected] wrote:

From: Xiubo Li <[email protected]>

In gluster-block project and there are 2 separate threads, both of which will run the svc_run loop, this could work well in glibc version, but in libtirpc we are hitting the random crash and stuck issues.

More detail please see:
https://github.com/gluster/gluster-block/pull/182

---
  src/svc_run.c | 19 +++++++++++++++++++

  1 file changed, 19 insertions(+)
src/svc_run.c +4 -0
index f40314b..b295755 100644
@@ -38,12 +38,17 @@
   #include <string.h>
   #include <unistd.h>
   #include <sys/poll.h>
+#include <syslog.h>
+#include <stdbool.h>
       #include <rpc/rpc.h>
   #include "rpc_com.h"
   #include <sys/select.h>
   +static bool svc_loop_running = false;
+static pthread_mutex_t svc_run_lock = PTHREAD_MUTEX_INITIALIZER;
+

  void
  svc_run()
  {

src/svc_run.c +9 -0
@@ -51,6 +56,16 @@ svc_run()
     struct pollfd *my_pollfd = NULL;
     int last_max_pollfd = 0;
   +  pthread_mutex_lock(&svc_run_lock);
+  if (svc_loop_running) {
+    pthread_mutex_unlock(&svc_run_lock);
+    syslog (LOG_ERR, "svc_run: svc loop is already running in current process %d", getpid());
+    return;
+  }
+
+  svc_loop_running = true;
+  pthread_mutex_unlock(&svc_run_lock);
+
     for (;;) {
       int max_pollfd = svc_max_pollfd;
       if (max_pollfd == 0 && svc_pollfd == NULL)

src/svc_run.c svc_exit() · line 133

Quoted 8 lines
src/svc_run.c +4 -0
@@ -111,4 +126,8 @@ svc_exit()
       svc_pollfd = NULL;
       svc_max_pollfd = 0;
       rwlock_unlock(&svc_fd_lock);
+
+    pthread_mutex_lock(&svc_run_lock);
+    svc_loop_running = false;
+    pthread_mutex_unlock(&svc_run_lock);
   }

_______________________________________________
Libtirpc-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/libtirpc-devel

author
#3 lore raw
Quoted 8 lines · Steve Dickson

Sorry for the delay....

Quoted 3 lines · Xiubo Li

Hey ping.

What's the state of this patch and will it make sense here?

I'm not sure it does make sense.... Shouldn't the mutex lock be in the call of svc_run()?

Hi Steve,

Yeah, mutex lock should be in the call of svc_run(). This is exactly what I do in this change.

If the libtirpc means to allow only one svc_run() loop in each process, so IMO this change is needed. Or if we will allow more than one like the glibc version does, so this should be one bug in libtirpc.

Thanks.
BRs
Xiubo

Quoted 72 lines

steved.

Quoted 70 lines

Thanks
BRs

Quoted 51 lines · On 2019/4/9 19:37, [email protected] wrote:

From: Xiubo Li <[email protected]>

In gluster-block project and there are 2 separate threads, both of which will run the svc_run loop, this could work well in glibc version, but in libtirpc we are hitting the random crash and stuck issues.

More detail please see:
https://github.com/gluster/gluster-block/pull/182

---
   src/svc_run.c | 19 +++++++++++++++++++
   1 file changed, 19 insertions(+)
src/svc_run.c +13 -0
index f40314b..b295755 100644
@@ -38,12 +38,17 @@
   #include <string.h>
   #include <unistd.h>
   #include <sys/poll.h>
+#include <syslog.h>
+#include <stdbool.h>
       #include <rpc/rpc.h>
   #include "rpc_com.h"
   #include <sys/select.h>
   +static bool svc_loop_running = false;
+static pthread_mutex_t svc_run_lock = PTHREAD_MUTEX_INITIALIZER;
+
   void
   svc_run()
   {
@@ -51,6 +56,16 @@ svc_run()
     struct pollfd *my_pollfd = NULL;
     int last_max_pollfd = 0;
   +  pthread_mutex_lock(&svc_run_lock);
+  if (svc_loop_running) {
+    pthread_mutex_unlock(&svc_run_lock);
+    syslog (LOG_ERR, "svc_run: svc loop is already running in current process %d", getpid());
+    return;
+  }
+
+  svc_loop_running = true;
+  pthread_mutex_unlock(&svc_run_lock);
+
     for (;;) {
       int max_pollfd = svc_max_pollfd;
       if (max_pollfd == 0 && svc_pollfd == NULL)

src/svc_run.c svc_exit() · line 133

Quoted 8 lines
src/svc_run.c +4 -0
@@ -111,4 +126,8 @@ svc_exit()
       svc_pollfd = NULL;
       svc_max_pollfd = 0;
       rwlock_unlock(&svc_fd_lock);
+
+    pthread_mutex_lock(&svc_run_lock);
+    svc_loop_running = false;
+    pthread_mutex_unlock(&svc_run_lock);
   }

_______________________________________________
Libtirpc-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/libtirpc-devel

#4 lore raw
Quoted 11 lines · Xiubo Li
Quoted 8 lines · Steve Dickson

Sorry for the delay....

Quoted 3 lines · Xiubo Li

Hey ping.

What's the state of this patch and will it make sense here?

I'm not sure it does make sense.... Shouldn't the mutex lock be in the call of svc_run()?

Quoted 8 lines

Hi Steve,

Yeah, mutex lock should be in the call of svc_run(). This is exactly what I do in this change.

If the libtirpc means to allow only one svc_run() loop in each process, so IMO this change is needed. Or if we will allow more than one like the glibc version does, so this should be one bug in libtirpc.

Has there been effort into made into investigating what's causing the crashes? We perhaps should make an effort to see if svc_run() is thread-safe and examine which functions it uses and which might not be thread safe. You might be able to allow greater parallelism then 1 thread in a svc_run() function by just making some not-thread safe functions wrapped in pthread locks.

Quoted 80 lines

Thanks.
BRs
Xiubo

Quoted 72 lines

steved.

Quoted 70 lines

Thanks
BRs

Quoted 51 lines · On 2019/4/9 19:37, [email protected] wrote:

From: Xiubo Li <[email protected]>

In gluster-block project and there are 2 separate threads, both of which will run the svc_run loop, this could work well in glibc version, but in libtirpc we are hitting the random crash and stuck issues.

More detail please see:
https://github.com/gluster/gluster-block/pull/182

---
   src/svc_run.c | 19 +++++++++++++++++++
   1 file changed, 19 insertions(+)
src/svc_run.c +13 -0
index f40314b..b295755 100644
@@ -38,12 +38,17 @@
   #include <string.h>
   #include <unistd.h>
   #include <sys/poll.h>
+#include <syslog.h>
+#include <stdbool.h>
       #include <rpc/rpc.h>
   #include "rpc_com.h"
   #include <sys/select.h>
   +static bool svc_loop_running = false;
+static pthread_mutex_t svc_run_lock = PTHREAD_MUTEX_INITIALIZER;
+
   void
   svc_run()
   {
@@ -51,6 +56,16 @@ svc_run()
     struct pollfd *my_pollfd = NULL;
     int last_max_pollfd = 0;
   +  pthread_mutex_lock(&svc_run_lock);
+  if (svc_loop_running) {
+    pthread_mutex_unlock(&svc_run_lock);
+    syslog (LOG_ERR, "svc_run: svc loop is already running in current process %d", getpid());
+    return;
+  }
+
+  svc_loop_running = true;
+  pthread_mutex_unlock(&svc_run_lock);
+
     for (;;) {
       int max_pollfd = svc_max_pollfd;
       if (max_pollfd == 0 && svc_pollfd == NULL)

src/svc_run.c svc_exit() · line 133

Quoted 8 lines
src/svc_run.c +4 -0
@@ -111,4 +126,8 @@ svc_exit()
       svc_pollfd = NULL;
       svc_max_pollfd = 0;
       rwlock_unlock(&svc_fd_lock);
+
+    pthread_mutex_lock(&svc_run_lock);
+    svc_loop_running = false;
+    pthread_mutex_unlock(&svc_run_lock);
   }

_______________________________________________
Libtirpc-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/libtirpc-devel

author
#5 lore raw
Quoted 12 lines · Olga Kornievskaia
Quoted 11 lines · Xiubo Li
Quoted 8 lines · Steve Dickson

Sorry for the delay....

Quoted 3 lines · Xiubo Li

Hey ping.

What's the state of this patch and will it make sense here?

I'm not sure it does make sense.... Shouldn't the mutex lock be in the call of svc_run()?

Hi Steve,

Quoted 8 lines
Quoted 6 lines

Yeah, mutex lock should be in the call of svc_run(). This is exactly what I do in this change.

If the libtirpc means to allow only one svc_run() loop in each process, so IMO this change is needed. Or if we will allow more than one like the glibc version does, so this should be one bug in libtirpc.

Has there been effort into made into investigating what's causing the crashes?

Before as our investigation and test, it was that if we ran two svc_run() loop in one process, such as in pthread1 and pthread2, it seems that pthread1 will receive the RPC connection/request which should be handled by pthread2's svc_run loop and vice versa.

Then we can see many random crash for tons of different reasons, like use after free and double free..., and almost every time the crash will randomly in different places and different libraries, such as the libtirpc, glusterfs and gluster-block...

After switching to multi processes instead of running two svc_run loop in multi pthreads, this issue has been resolved we didn't dig it further.

Quoted 5 lines

We perhaps should make an effort to see if svc_run() is
thread-safe and examine which functions it uses and which might not be thread safe. You might be able to allow greater parallelism then 1 thread in a svc_run() function by just making some not-thread safe functions wrapped in pthread locks.

Yeah, make sense.

Thanks.

BRs

Quoted 77 lines
Quoted 77 lines

Thanks.
BRs
Xiubo

Quoted 71 lines

steved.

Quoted 69 lines

Thanks
BRs

Quoted 51 lines · On 2019/4/9 19:37, [email protected] wrote:

From: Xiubo Li <[email protected]>

In gluster-block project and there are 2 separate threads, both of which will run the svc_run loop, this could work well in glibc version, but in libtirpc we are hitting the random crash and stuck issues.

More detail please see:
https://github.com/gluster/gluster-block/pull/182

---
    src/svc_run.c | 19 +++++++++++++++++++
    1 file changed, 19 insertions(+)
src/svc_run.c +13 -0
index f40314b..b295755 100644
@@ -38,12 +38,17 @@
    #include <string.h>
    #include <unistd.h>
    #include <sys/poll.h>
+#include <syslog.h>
+#include <stdbool.h>
        #include <rpc/rpc.h>
    #include "rpc_com.h"
    #include <sys/select.h>
    +static bool svc_loop_running = false;
+static pthread_mutex_t svc_run_lock = PTHREAD_MUTEX_INITIALIZER;
+
    void
    svc_run()
    {
@@ -51,6 +56,16 @@ svc_run()
      struct pollfd *my_pollfd = NULL;
      int last_max_pollfd = 0;
    +  pthread_mutex_lock(&svc_run_lock);
+  if (svc_loop_running) {
+    pthread_mutex_unlock(&svc_run_lock);
+    syslog (LOG_ERR, "svc_run: svc loop is already running in current process %d", getpid());
+    return;
+  }
+
+  svc_loop_running = true;
+  pthread_mutex_unlock(&svc_run_lock);
+
      for (;;) {
        int max_pollfd = svc_max_pollfd;
        if (max_pollfd == 0 && svc_pollfd == NULL)

src/svc_run.c svc_exit() · line 133

Quoted 8 lines
src/svc_run.c +4 -0
@@ -111,4 +126,8 @@ svc_exit()
        svc_pollfd = NULL;
        svc_max_pollfd = 0;
        rwlock_unlock(&svc_fd_lock);
+
+    pthread_mutex_lock(&svc_run_lock);
+    svc_loop_running = false;
+    pthread_mutex_unlock(&svc_run_lock);
    }

_______________________________________________
Libtirpc-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/libtirpc-devel