Multipath io_err_stat fixes
Benjamin Marzinski
maintainerv2 cover Multipath io_err_stat fixes
I found an ABBA deadlock in the io_err_stat marginal path code, and in the process of fixing it, noticed a potential crash on shutdown. This patchset addresses both of the issues.
Changes from v1:
0002: use cleanup_mutex instead of cleanup_unlock as suggested by Martin
0003: add pthread_testcancel and use cleanup_mutex instead of cleanup_unlock as suggested by Martin. Also, make tmp_pathvec a constant pointer, since it should always equal _pathvec.
0004-0006 are new patches to deal with io_err_stat issues from Martin's review
Benjamin Marzinski (6):
libmultipath: make find_err_path_by_dev() static
multipathd: avoid io_err_stat crash during shutdown
multipathd: avoid io_err_stat ABBA deadlock
multipathd: use get_monotonic_time() in io_err_stat code
multipathd: combine free_io_err_stat_path and destroy_directio_ctx
multipathd: cleanup logging for marginal paths
libmultipath/io_err_stat.c | 216 ++++++++++++++++--------------------- multipathd/main.c | 25 +++-- 2 files changed, 105 insertions(+), 136 deletions(-)
Signature · 6 lines
2.17.2 -- dm-devel mailing list [email protected] https://www.redhat.com/mailman/listinfo/dm-devel
Benjamin Marzinski
maintainerPATCH v2 1/6 libmultipath: make find_err_path_by_dev() static
--- libmultipath/io_err_stat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
libmultipath/io_err_stat.c +1 -1
@@ -88,7 +88,7 @@ static void rcu_unregister(__attribute__((unused)) void *param)rcu_unregister_thread (); } -struct io_err_stat_path *find_err_path_by_dev (vector pathvec,char *dev) +static struct io_err_stat_path *find_err_path_by_dev (vector pathvec,char *dev) {int i;struct io_err_stat_path *pp;
Signature · 6 lines
2.17.2 -- dm-devel mailing list [email protected] https://www.redhat.com/mailman/listinfo/dm-devel
Benjamin Marzinski
maintainerPATCH v2 2/6 multipathd: avoid io_err_stat crash during shutdown
In reply to Benjamin MarzinskiThe checker thread is reponsible for enqueueing paths for the io_err_stat thread to check. During shutdown, the io_err_stat thread is shut down and cleaned up before the checker thread. There is no code to make sure that the checker thread isn't accessing the io_err_stat pathvec or its mutex while they are being freed, which can lead to memory corruption crashes.
To solve this, get rid of the io_err_stat_pathvec structure, and statically define the mutex. This means that the mutex is always valid to access, and the io_err_stat pathvec can only be accessed while holding it. If the io_err_stat thread has already been cleaned up when the checker tries to access the pathvec, it will be NULL, and the checker will simply fail to enqueue the path.
This change also fixes a bug in free_io_err_pathvec(), which previously only attempted to free the pathvec if it was not set, instead of when it was set.
--- libmultipath/io_err_stat.c | 111 ++++++++++++++----------------------- 1 file changed, 43 insertions(+), 68 deletions(-)
libmultipath/io_err_stat.c +43 -68
@@ -46,12 +46,6 @@ #define io_err_stat_log(prio, fmt, args...) \ condlog(prio, "io error statistic: " fmt, ##args) - -struct io_err_stat_pathvec { -pthread_mutex_t mutex; -vector pathvec; -}; -struct dio_ctx {struct timespec io_starttime;unsigned int blksize;
@@ -75,9 +69,10 @@ static pthread_t io_err_stat_thr;static pthread_mutex_t io_err_thread_lock =PTHREAD_MUTEX_INITIALIZER ;static pthread_cond_t io_err_thread_cond =PTHREAD_COND_INITIALIZER ; +static pthread_mutex_t io_err_pathvec_lock =PTHREAD_MUTEX_INITIALIZER ;static int io_err_thread_running =0 ; -static struct io_err_stat_pathvec *paths; +static vector io_err_pathvec;struct vectors *vecs;io_context_t ioctx;
@@ -207,46 +202,23 @@ static void free_io_err_stat_path(struct io_err_stat_path *p)FREE (p); } -static struct io_err_stat_pathvec *alloc_pathvec (void ) -{ -struct io_err_stat_pathvec *p; -int r; - - p = (struct io_err_stat_pathvec *)MALLOC (sizeof (*p)); -if (!p) -return NULL ; - p->pathvec =vector_alloc (); -if (!p->pathvec) - goto out_free_struct_pathvec; - r =pthread_mutex_init (&p->mutex,NULL ); -if (r) - goto out_free_member_pathvec; - -return p; - -out_free_member_pathvec: -vector_free (p->pathvec); -out_free_struct_pathvec: -FREE (p); -return NULL ; -} - -static void free_io_err_pathvec (struct io_err_stat_pathvec *p) +static void free_io_err_pathvec (void ) {struct io_err_stat_path *path;int i; -if (!p) -return ; -pthread_mutex_destroy (&p->mutex); -if (!p->pathvec) { - vector_foreach_slot(p->pathvec, path, i) { -destroy_directio_ctx (path); -free_io_err_stat_path (path); - } -vector_free (p->pathvec); +pthread_mutex_lock (&io_err_pathvec_lock); +pthread_cleanup_push (cleanup_mutex, &io_err_pathvec_lock); +if (!io_err_pathvec) + goto out; + vector_foreach_slot(io_err_pathvec, path, i) { +destroy_directio_ctx (path); +free_io_err_stat_path (path); } -FREE (p); +vector_free (io_err_pathvec); + io_err_pathvec =NULL ; +out: +pthread_cleanup_pop (1 ); }/*
@@ -258,13 +230,13 @@ static int enqueue_io_err_stat_by_path(struct path *path) {struct io_err_stat_path *p; -pthread_mutex_lock (&paths->mutex); - p =find_err_path_by_dev (paths->pathvec, path->dev); +pthread_mutex_lock (&io_err_pathvec_lock); + p =find_err_path_by_dev (io_err_pathvec, path->dev);if (p) { -pthread_mutex_unlock (&paths->mutex); +pthread_mutex_unlock (&io_err_pathvec_lock);return 0 ; } -pthread_mutex_unlock (&paths->mutex); +pthread_mutex_unlock (&io_err_pathvec_lock); p =alloc_io_err_stat_path ();if (!p)
@@ -276,18 +248,18 @@ static int enqueue_io_err_stat_by_path(struct path *path)if (setup_directio_ctx (p)) goto free_ioerr_path; -pthread_mutex_lock (&paths->mutex); -if (!vector_alloc_slot (paths->pathvec)) +pthread_mutex_lock (&io_err_pathvec_lock); +if (!vector_alloc_slot (io_err_pathvec)) goto unlock_destroy; -vector_set_slot (paths->pathvec, p); -pthread_mutex_unlock (&paths->mutex); +vector_set_slot (io_err_pathvec, p); +pthread_mutex_unlock (&io_err_pathvec_lock);io_err_stat_log (2 ,"%s: enqueue path %s to check" , path->mpp->alias, path->dev);return 0 ; unlock_destroy: -pthread_mutex_unlock (&paths->mutex); +pthread_mutex_unlock (&io_err_pathvec_lock);destroy_directio_ctx (p); free_ioerr_path:free_io_err_stat_path (p);
@@ -412,9 +384,9 @@ static int delete_io_err_stat_by_addr(struct io_err_stat_path *p) {int i; - i =find_slot (paths->pathvec, p); + i =find_slot (io_err_pathvec, p);if (i !=-1 ) -vector_del_slot (paths->pathvec, i); +vector_del_slot (io_err_pathvec, i);destroy_directio_ctx (p);free_io_err_stat_path (p);
@@ -585,7 +557,7 @@ static void poll_async_io_timeout(void)if (clock_gettime (CLOCK_MONOTONIC , &curr_time) !=0 )return ; - vector_foreach_slot(paths->pathvec, pp, i) { + vector_foreach_slot(io_err_pathvec, pp, i) {for (j =0 ; j <CONCUR_NR_EVENT ; j++) { rc =try_to_cancel_timeout_io (pp->dio_ctx_array + j, &curr_time, pp->devname);
@@ -631,7 +603,7 @@ static void handle_async_io_done_event(struct io_event *io_evt)int rc =PATH_UNCHECKED ;int i, j; - vector_foreach_slot(paths->pathvec, pp, i) { + vector_foreach_slot(io_err_pathvec, pp, i) {for (j =0 ; j <CONCUR_NR_EVENT ; j++) { ct = pp->dio_ctx_array + j;if (&ct->io == io_evt->obj) {
@@ -665,19 +637,14 @@ static void service_paths(void)struct io_err_stat_path *pp;int i; -pthread_mutex_lock (&paths->mutex); - vector_foreach_slot(paths->pathvec, pp, i) { +pthread_mutex_lock (&io_err_pathvec_lock); + vector_foreach_slot(io_err_pathvec, pp, i) {send_batch_async_ios (pp);process_async_ios_event (TIMEOUT_NO_IO_NSEC , pp->devname);poll_async_io_timeout ();poll_io_err_stat (vecs, pp); } -pthread_mutex_unlock (&paths->mutex); -} - -static void cleanup_unlock (void *arg) -{ -pthread_mutex_unlock ((pthread_mutex_t *) arg); +pthread_mutex_unlock (&io_err_pathvec_lock); }static void cleanup_exited (__attribute__((unused))void *arg)
@@ -736,13 +703,18 @@ int start_io_err_stat_thread(void *data)io_err_stat_log (4 ,"io_setup failed" );return 1 ; } - paths =alloc_pathvec (); -if (!paths) + +pthread_mutex_lock (&io_err_pathvec_lock); + io_err_pathvec =vector_alloc (); +if (!io_err_pathvec) { +pthread_mutex_unlock (&io_err_pathvec_lock); goto destroy_ctx; + } +pthread_mutex_unlock (&io_err_pathvec_lock);setup_thread_attr (&io_err_stat_attr,32 *1024 ,0 );pthread_mutex_lock (&io_err_thread_lock); -pthread_cleanup_push (cleanup_unlock, &io_err_thread_lock); +pthread_cleanup_push (cleanup_mutex, &io_err_thread_lock); ret =pthread_create (&io_err_stat_thr, &io_err_stat_attr, io_err_stat_loop, data);
@@ -763,7 +735,10 @@ int start_io_err_stat_thread(void *data)return 0 ; out_free: -free_io_err_pathvec (paths); +pthread_mutex_lock (&io_err_pathvec_lock); +vector_free (io_err_pathvec); + io_err_pathvec =NULL ; +pthread_mutex_unlock (&io_err_pathvec_lock); destroy_ctx:io_destroy (ioctx);io_err_stat_log (0 ,"failed to start io_error statistic thread" );
@@ -779,6 +754,6 @@ void stop_io_err_stat_thread(void)pthread_cancel (io_err_stat_thr);pthread_join (io_err_stat_thr,NULL ); -free_io_err_pathvec (paths); +free_io_err_pathvec ();io_destroy (ioctx); }
Signature · 6 lines
2.17.2 -- dm-devel mailing list [email protected] https://www.redhat.com/mailman/listinfo/dm-devel
Benjamin Marzinski
maintainerPATCH v2 3/6 multipathd: avoid io_err_stat ABBA deadlock
In reply to Benjamin MarzinskiWhen the checker thread enqueues paths for the io_err_stat thread to check, it calls enqueue_io_err_stat_by_path() with the vecs lock held. start_io_err_stat_thread() is also called with the vecs lock held. These two functions both lock io_err_pathvec_lock. When the io_err_stat thread updates the paths in vecs->pathvec in poll_io_err_stat(), it has the io_err_pathvec_lock held, and then locks the vecs lock. This can cause an ABBA deadlock.
To solve this, service_paths() no longer updates the paths in vecs->pathvec with the io_err_pathvec_lock held. It does this by moving the io_err_stat_path from io_err_pathvec to a local vector when it needs to update the path. After releasing the io_err_pathvec_lock, it goes through this temporary vector, updates the paths with the vecs lock held, and then frees everything.
This change fixes a bug in service_paths() where elements were being deleted from io_err_pathvec, without the index being decremented, causing the loop to skip elements. Also, service_paths() could be cancelled while holding the io_err_pathvec_lock, so it should have a cleanup handler.
--- libmultipath/io_err_stat.c | 56 ++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 24 deletions(-)
libmultipath/io_err_stat.c +32 -24
@@ -380,20 +380,6 @@ recover:return 0 ; } -static int delete_io_err_stat_by_addr (struct io_err_stat_path *p) -{ -int i; - - i =find_slot (io_err_pathvec, p); -if (i !=-1 ) -vector_del_slot (io_err_pathvec, i); - -destroy_directio_ctx (p); -free_io_err_stat_path (p); - -return 0 ; -} -static void account_async_io_state (struct io_err_stat_path *pp,int rc) {switch (rc) {
@@ -410,17 +396,26 @@ static void account_async_io_state(struct io_err_stat_path *pp, int rc) } } -static int poll_io_err_stat (struct vectors *vecs,struct io_err_stat_path *pp) +static int io_err_stat_time_up (struct io_err_stat_path *pp) {struct timespec currtime, difftime; -struct path *path; -double err_rate;if (clock_gettime (CLOCK_MONOTONIC , &currtime) !=0 ) -return 1 ; +return 0 ;timespecsub (&currtime, &pp->start_time, &difftime);if (difftime.tv_sec < pp->total_time)return 0 ; +return 1 ; +} + +static void end_io_err_stat (struct io_err_stat_path *pp) +{ +struct timespec currtime; +struct path *path; +double err_rate; + +if (clock_gettime (CLOCK_MONOTONIC , &currtime) !=0 ) + currtime = pp->start_time;io_err_stat_log (4 ,"%s: check end" , pp->devname);
@@ -459,10 +454,6 @@ static int poll_io_err_stat(struct vectors *vecs, struct io_err_stat_path *pp) pp->devname); }lock_cleanup_pop (vecs->lock); - -delete_io_err_stat_by_addr (pp); - -return 0 ; }static int send_each_async_io (struct dio_ctx *ct,int fd,char *dev)
@@ -622,6 +613,7 @@ static void process_async_ios_event(int timeout_nsecs, char *dev)struct timespec timeout = { .tv_nsec = timeout_nsecs }; errno =0 ; +pthread_testcancel (); n =io_getevents (ioctx,1L ,CONCUR_NR_EVENT , events, &timeout);if (n <0 ) { io_err_stat_log(3, "%s: async io events returned %d (errno=%s)",
@@ -634,17 +626,33 @@ static void process_async_ios_event(int timeout_nsecs, char *dev)static void service_paths (void ) { +struct _vector _pathvec = {0 }; +/* avoid gcc warnings that &_pathvec will never be NULL in vector ops */ +struct _vector *const tmp_pathvec = &_pathvec;struct io_err_stat_path *pp;int i;pthread_mutex_lock (&io_err_pathvec_lock); +pthread_cleanup_push (cleanup_mutex, &io_err_pathvec_lock); vector_foreach_slot(io_err_pathvec, pp, i) {send_batch_async_ios (pp);process_async_ios_event (TIMEOUT_NO_IO_NSEC , pp->devname);poll_async_io_timeout (); -poll_io_err_stat (vecs, pp); +if (io_err_stat_time_up (pp)) { +if (!vector_alloc_slot (tmp_pathvec)) +continue ; +vector_del_slot (io_err_pathvec, i--); +vector_set_slot (tmp_pathvec, pp); + } } -pthread_mutex_unlock (&io_err_pathvec_lock); +pthread_cleanup_pop (1 ); + vector_foreach_slot_backwards(tmp_pathvec, pp, i) { +end_io_err_stat (pp); +vector_del_slot (tmp_pathvec, i); +destroy_directio_ctx (pp); +free_io_err_stat_path (pp); + } +vector_reset (tmp_pathvec); }static void cleanup_exited (__attribute__((unused))void *arg)
Signature · 6 lines
2.17.2 -- dm-devel mailing list [email protected] https://www.redhat.com/mailman/listinfo/dm-devel
Benjamin Marzinski
maintainerPATCH v2 4/6 multipathd: use get_monotonic_time() in io_err_stat code
In reply to Benjamin MarzinskiInstead of calling clock_gettime(), and dealing with failure conditions, just call get_monotonic_time().
--- libmultipath/io_err_stat.c | 34 +++++++++++----------------------- 1 file changed, 11 insertions(+), 23 deletions(-)
libmultipath/io_err_stat.c +11 -23
@@ -295,8 +295,7 @@ int io_err_stat_handle_pathfail(struct path *path)* the repeated count threshold and time frame, we assume a path * which fails at least twice within 60 seconds is flaky. */ -if (clock_gettime (CLOCK_MONOTONIC , &curr_time) !=0 ) -return 1 ; +get_monotonic_time (&curr_time);if (path->io_err_pathfail_cnt ==0 ) { path->io_err_pathfail_cnt++; path->io_err_pathfail_starttime = curr_time.tv_sec;
@@ -352,9 +351,9 @@ int need_io_err_check(struct path *pp) }if (pp->io_err_pathfail_cnt !=PATH_IO_ERR_WAITING_TO_CHECK )return 1 ; -if (clock_gettime (CLOCK_MONOTONIC , &curr_time) !=0 || - (curr_time.tv_sec - pp->io_err_dis_reinstate_time) > - pp->mpp->marginal_path_err_recheck_gap_time) { +get_monotonic_time (&curr_time); +if ((curr_time.tv_sec - pp->io_err_dis_reinstate_time) > + pp->mpp->marginal_path_err_recheck_gap_time) {io_err_stat_log (4 ,"%s: reschedule checking after %d seconds" , pp->dev, pp->mpp->marginal_path_err_recheck_gap_time);
@@ -400,8 +399,7 @@ static int io_err_stat_time_up(struct io_err_stat_path *pp) {struct timespec currtime, difftime; -if (clock_gettime (CLOCK_MONOTONIC , &currtime) !=0 ) -return 0 ; +get_monotonic_time (&currtime);timespecsub (&currtime, &pp->start_time, &difftime);if (difftime.tv_sec < pp->total_time)return 0 ;
@@ -414,8 +412,7 @@ static void end_io_err_stat(struct io_err_stat_path *pp)struct path *path;double err_rate; -if (clock_gettime (CLOCK_MONOTONIC , &currtime) !=0 ) - currtime = pp->start_time; +get_monotonic_time (&currtime);io_err_stat_log (4 ,"%s: check end" , pp->devname);
@@ -464,11 +461,7 @@ static int send_each_async_io(struct dio_ctx *ct, int fd, char *dev) ct->io_starttime.tv_sec == 0) {struct iocb *ios[1 ] = { &ct->io }; -if (clock_gettime (CLOCK_MONOTONIC , &ct->io_starttime) !=0 ) { - ct->io_starttime.tv_sec =0 ; - ct->io_starttime.tv_nsec =0 ; -return rc; - } +get_monotonic_time (&ct->io_starttime);io_prep_pread (&ct->io, fd, ct->buf, ct->blksize,0 );if (io_submit (ioctx,1 , ios) !=1 ) { io_err_stat_log(5, "%s: io_submit error %i",
@@ -487,8 +480,7 @@ static void send_batch_async_ios(struct io_err_stat_path *pp)struct dio_ctx *ct;struct timespec currtime, difftime; -if (clock_gettime (CLOCK_MONOTONIC , &currtime) !=0 ) -return ; +get_monotonic_time (&currtime);/* * Give a free time for all IO to complete or timeout */
@@ -503,11 +495,8 @@ static void send_batch_async_ios(struct io_err_stat_path *pp)if (!send_each_async_io (ct, pp->fd, pp->devname)) pp->io_nr++; } -if (pp->start_time.tv_sec ==0 && pp->start_time.tv_nsec ==0 && -clock_gettime (CLOCK_MONOTONIC , &pp->start_time)) { - pp->start_time.tv_sec =0 ; - pp->start_time.tv_nsec =0 ; - } +if (pp->start_time.tv_sec ==0 && pp->start_time.tv_nsec ==0 ) +get_monotonic_time (&pp->start_time); } static int try_to_cancel_timeout_io(struct dio_ctx *ct, struct timespec *t,
@@ -546,8 +535,7 @@ static void poll_async_io_timeout(void)int rc =PATH_UNCHECKED ;int i, j; -if (clock_gettime (CLOCK_MONOTONIC , &curr_time) !=0 ) -return ; +get_monotonic_time (&curr_time); vector_foreach_slot(io_err_pathvec, pp, i) {for (j =0 ; j <CONCUR_NR_EVENT ; j++) { rc = try_to_cancel_timeout_io(pp->dio_ctx_array + j,
Signature · 6 lines
2.17.2 -- dm-devel mailing list [email protected] https://www.redhat.com/mailman/listinfo/dm-devel
Benjamin Marzinski
maintainerPATCH v2 5/6 multipathd: combine free_io_err_stat_path and destroy_directio_ctx
In reply to Benjamin Marzinskidestroy_directio_ctx() is only called from free_io_err_stat_path(), and free_io_err_stat_path() is very short, so combine them.
--- libmultipath/io_err_stat.c | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-)
libmultipath/io_err_stat.c +10 -14
@@ -161,12 +161,15 @@ fail_close:return 1 ; } -static void destroy_directio_ctx (struct io_err_stat_path *p) +static void free_io_err_stat_path (struct io_err_stat_path *p) {int i; -if (!p || !p->dio_ctx_array) +if (!p)return ; +if (!p->dio_ctx_array) + goto free_path; +cancel_inflight_io (p);for (i =0 ; i <CONCUR_NR_EVENT ; i++)
@@ -175,6 +178,8 @@ static void destroy_directio_ctx(struct io_err_stat_path *p)if (p->fd >0 )close (p->fd); +free_path: +FREE (p); }static struct io_err_stat_path *alloc_io_err_stat_path (void )
@@ -197,11 +202,6 @@ static struct io_err_stat_path *alloc_io_err_stat_path(void)return p; } -static void free_io_err_stat_path (struct io_err_stat_path *p) -{ -FREE (p); -} -static void free_io_err_pathvec (void ) {struct io_err_stat_path *path;
@@ -211,10 +211,8 @@ static void free_io_err_pathvec(void)pthread_cleanup_push (cleanup_mutex, &io_err_pathvec_lock);if (!io_err_pathvec) goto out; - vector_foreach_slot(io_err_pathvec, path, i) { -destroy_directio_ctx (path); + vector_foreach_slot(io_err_pathvec, path, i)free_io_err_stat_path (path); - }vector_free (io_err_pathvec); io_err_pathvec =NULL ; out:
@@ -250,7 +248,7 @@ static int enqueue_io_err_stat_by_path(struct path *path) goto free_ioerr_path;pthread_mutex_lock (&io_err_pathvec_lock);if (!vector_alloc_slot (io_err_pathvec)) - goto unlock_destroy; + goto unlock_pathvec;vector_set_slot (io_err_pathvec, p);pthread_mutex_unlock (&io_err_pathvec_lock);
@@ -258,9 +256,8 @@ static int enqueue_io_err_stat_by_path(struct path *path) path->mpp->alias, path->dev);return 0 ; -unlock_destroy: +unlock_pathvec:pthread_mutex_unlock (&io_err_pathvec_lock); -destroy_directio_ctx (p); free_ioerr_path:free_io_err_stat_path (p);
@@ -637,7 +634,6 @@ static void service_paths(void) vector_foreach_slot_backwards(tmp_pathvec, pp, i) {end_io_err_stat (pp);vector_del_slot (tmp_pathvec, i); -destroy_directio_ctx (pp);free_io_err_stat_path (pp); }vector_reset (tmp_pathvec);
Signature · 6 lines
2.17.2 -- dm-devel mailing list [email protected] https://www.redhat.com/mailman/listinfo/dm-devel
Benjamin Marzinski
maintainerPATCH v2 6/6 multipathd: cleanup logging for marginal paths
In reply to Benjamin Marzinskiio_err_stat logged at level 2 whenever it enqueued a path to check, which could happen multiple times while a path was marginal. On the other hand if marginal_pathgroups wasn't set, multipathd didn't log when paths were set to marginal. Now io_err_stat only logs at level 2 when something unexpected happens, but multipathd will always log when a path switches its marginal state.
--- libmultipath/io_err_stat.c | 7 +++---- multipathd/main.c | 25 ++++++++++++++----------- 2 files changed, 17 insertions(+), 15 deletions(-)
libmultipath/io_err_stat.c +3 -4
@@ -252,7 +252,7 @@ static int enqueue_io_err_stat_by_path(struct path *path)vector_set_slot (io_err_pathvec, p);pthread_mutex_unlock (&io_err_pathvec_lock); -io_err_stat_log (2 ,"%s: enqueue path %s to check" , +io_err_stat_log (3 ,"%s: enqueue path %s to check" , path->mpp->alias, path->dev);return 0 ;
@@ -343,7 +343,7 @@ int need_io_err_check(struct path *pp)if (uatomic_read (&io_err_thread_running) ==0 )return 0 ;if (count_active_paths (pp->mpp) <=0 ) { -io_err_stat_log (2 ,"%s: recover path early" , pp->dev); +io_err_stat_log (2 ,"%s: no paths. recovering early" , pp->dev); goto recover; }if (pp->io_err_pathfail_cnt !=PATH_IO_ERR_WAITING_TO_CHECK )
@@ -361,8 +361,7 @@ int need_io_err_check(struct path *pp)* Or else, return 1 to set path state to PATH_SHAKY */ if (r ==1 ) { -io_err_stat_log (3 ,"%s: enqueue fails, to recover" , - pp->dev); +io_err_stat_log (2 ,"%s: enqueue failed. recovering early" , pp->dev); goto recover; }else pp->io_err_pathfail_cnt =PATH_IO_ERR_IN_CHECKING ;
multipathd/main.c +14 -11
@@ -2132,8 +2132,8 @@ check_path (struct vectors * vecs, struct path * pp, unsigned int ticks)pathinfo (pp, conf,0 );pthread_cleanup_pop (1 );return 1 ; - }else if ((newstate !=PATH_UP && newstate !=PATH_GHOST ) && - (pp->state ==PATH_DELAYED )) { + }else if ((newstate !=PATH_UP && newstate !=PATH_GHOST && + newstate !=PATH_PENDING ) && (pp->state ==PATH_DELAYED )) {/* If path state become failed again cancel path delay state */ pp->state = newstate;/*
@@ -2200,8 +2200,9 @@ check_path (struct vectors * vecs, struct path * pp, unsigned int ticks)if ((newstate ==PATH_UP || newstate ==PATH_GHOST ) && (san_path_check_enabled (pp->mpp) ||marginal_path_check_enabled (pp->mpp))) { -int was_marginal = pp->marginal;if (should_skip_path (pp)) { +if (!pp->marginal && pp->state !=PATH_DELAYED ) +condlog (2 ,"%s: path is now marginal" , pp->dev);if (!marginal_pathgroups) {if (marginal_path_check_enabled (pp->mpp))/* to reschedule as soon as possible,
@@ -2211,13 +2212,18 @@ check_path (struct vectors * vecs, struct path * pp, unsigned int ticks) pp->state =PATH_DELAYED ;return 1 ; } -if (!was_marginal) { +if (!pp->marginal) { pp->marginal =1 ; marginal_changed =1 ; } - }else if (marginal_pathgroups && was_marginal) { - pp->marginal =0 ; - marginal_changed =1 ; + }else { +if (pp->marginal || pp->state ==PATH_DELAYED ) +condlog (2 ,"%s: path is no longer marginal" , + pp->dev); +if (marginal_pathgroups && pp->marginal) { + pp->marginal =0 ; + marginal_changed =1 ; + } } }
@@ -2343,11 +2349,8 @@ check_path (struct vectors * vecs, struct path * pp, unsigned int ticks)*/ condlog (4 ,"path prio refresh" ); -if (marginal_changed) { -condlog (2 ,"%s: path is %s marginal" , pp->dev, - (pp->marginal)?"now" :"no longer" ); +if (marginal_changed)reload_and_sync_map (pp->mpp, vecs,1 ); - }else if (update_prio (pp, new_path_up) && (pp->mpp->pgpolicyfn == (pgpolicyfn *)group_by_prio) && pp->mpp->pgfailback == -FAILBACK_IMMEDIATE ) {
Signature · 6 lines
2.17.2 -- dm-devel mailing list [email protected] https://www.redhat.com/mailman/listinfo/dm-devel
Re: [dm-devel] [PATCH v2 6/6] multipathd: cleanup logging for marginal paths
Quoted 57 lines · Benjamin Marzinski
io_err_statlogged at level 2 whenever it enqueued a path to check, which could happen multiple times while a path was marginal. On the other hand ifmarginal_pathgroupswasn't set, multipathd didn't log when
paths were set to marginal. Nowio_err_statonly logs at level 2 when something unexpected happens, but multipathd will always log when a path switches its marginal state.Signed-off-by Benjamin Marzinski <[email protected]>---
libmultipath/io_err_stat.c | 7 +++---- multipathd/main.c | 25 ++++++++++++++-----------2 files changed, 17 insertions(+), 15 deletions(-)libmultipath/io_err_stat.c +0
-0@@ -252,7 +252,7 @@ static int enqueue_io_err_stat_by_path(struct path *path) vector_set_slot(io_err_pathvec, p); pthread_mutex_unlock(&io_err_pathvec_lock);
- io_err_stat_log(2, "%s: enqueue path %s to check", + io_err_stat_log(3, "%s: enqueue path %s to check",path->mpp->alias,path->dev); return 0;libmultipath/io_err_stat.c +1
-1@@ -343,7 +343,7 @@ int need_io_err_check(struct path *pp) if (uatomic_read(&io_err_thread_running) == 0) return 0; if (count_active_paths(pp->mpp) <= 0) { - io_err_stat_log(2, "%s: recover path early", pp- >dev); + io_err_stat_log(2, "%s: no paths. recovering early", pp->dev); goto recover;}
if (pp->io_err_pathfail_cnt!=PATH_IO_ERR_WAITING_TO_CHECK)libmultipath/io_err_stat.c +1
-2@@ -361,8 +361,7 @@ int need_io_err_check(struct path *pp)* Or else, return 1 to set path state to PATH_SHAKY */ if (r == 1) { - io_err_stat_log(3, "%s: enqueue fails, to recover", - pp->dev); + io_err_stat_log(2, "%s: enqueue failed. recovering early", pp->dev); goto recover;} else
pp->io_err_pathfail_cnt =PATH_IO_ERR_IN_CHECKING;multipathd/main.c +0
-0
multipathd/main.c check_path (struct vectors * vecs, struct path · line 2138
Quoted 9 lines
multipathd/main.c +2
-2@@ -2132,8 +2132,8 @@ check_path (struct vectors * vecs, struct path * pp, unsigned int ticks) pathinfo(pp, conf, 0); pthread_cleanup_pop(1); return 1; - } else if ((newstate != PATH_UP && newstate != PATH_GHOST) && - (pp->state == PATH_DELAYED)) { + } else if ((newstate != PATH_UP && newstate != PATH_GHOST && + newstate != PATH_PENDING) && (pp->state ==PATH_DELAYED )) {
I think this is correct, but it needs to be mentioned in the commit message (or go into a separate patch).
Regards,
Martin
Signature · 10 lines
Dr. Martin Wilck <[email protected]>, Tel. +49 (0)911 74053 2107 SUSE Software Solutions Germany GmbH HRB 36809, AG Nürnberg GF: Felix Imendörffer -- dm-devel mailing list [email protected] https://www.redhat.com/mailman/listinfo/dm-devel
Benjamin Marzinski
maintainerRe: [dm-devel] [PATCH v2 6/6] multipathd: cleanup logging for marginal paths
Quoted 20 lines · Martin Wilck
Quoted 19 lines · Benjamin Marzinski
io_err_statlogged at level 2 whenever it enqueued a path to check, which could happen multiple times while a path was marginal. On the other hand ifmarginal_pathgroupswasn't set, multipathd didn't log when
paths were set to marginal. Nowio_err_statonly logs at level 2 when something unexpected happens, but multipathd will always log when a path switches its marginal state.Signed-off-by Benjamin Marzinski <[email protected]>multipathd/main.c +0
-0@@ -2132,8 +2132,8 @@ check_path (struct vectors * vecs, struct path * pp, unsigned int ticks) pathinfo(pp, conf, 0); pthread_cleanup_pop(1); return 1;
Quoted 8 lines
multipathd/main.c check_path (struct vectors * vecs, struct path * pp, unsigned int ticks)
Quoted 5 lines
multipathd/main.c +2
-2- } else if ((newstate != PATH_UP && newstate != PATH_GHOST) && - (pp->state == PATH_DELAYED)) { + } else if ((newstate != PATH_UP && newstate != PATH_GHOST && + newstate != PATH_PENDING) && (pp->state ==PATH_DELAYED )) {I think this is correct, but it needs to be mentioned in the commit message (or go into a separate patch).
It needs to go in this patch. Without it, devices in the delayed state can change to pending, and then back again. The code logs another message every time this happens. But I can add an explanation to the commit message.
I'll make a v3 release with just this patch.
-Ben
Quoted 9 lines
Regards,
MartinSignature · 3 lines
Dr. Martin Wilck <[email protected]>, Tel. +49 (0)911 74053 2107 SUSE Software Solutions Germany GmbH HRB 36809, AG Nürnberg GF: Felix Imendörffer
--
dm-devel mailing list
[email protected]
https://www.redhat.com/mailman/listinfo/dm-devel
Quoted 27 lines · Benjamin Marzinski
I found an ABBA deadlock in the
io_err_statmarginal path code, and in
the process of fixing it, noticed a potential crash on shutdown. This patchset addresses both of the issues.Changes from v1:
0002: use
cleanup_mutexinstead ofcleanup_unlockas suggested by Martin0003: add
pthread_testcanceland usecleanup_mutexinstead ofcleanup_unlockas suggested by Martin. Also, maketmp_pathveca constant
pointer, since it should always equal_pathvec.0004-0006 are new patches to deal with
io_err_statissues from Martin's
reviewBenjamin Marzinski (6):
libmultipath: makefind_err_path_by_dev()static
multipathd: avoidio_err_statcrash during shutdown
multipathd: avoidio_err_statABBA deadlock
multipathd: useget_monotonic_time()inio_err_statcode
multipathd: combinefree_io_err_stat_pathanddestroy_directio_ctx
multipathd: cleanup logging for marginal paths
Quoted 4 lines
libmultipath/io_err_stat.c | 216 ++++++++++++++++------------------- --
multipathd/main.c | 25 +++--2 files changed, 105 insertions(+), 136 deletions(-)
For the series, except 6/6 where I had a nitpick:
Signature · 10 lines
Dr. Martin Wilck <[email protected]>, Tel. +49 (0)911 74053 2107 SUSE Software Solutions Germany GmbH HRB 36809, AG Nürnberg GF: Felix Imendörffer -- dm-devel mailing list [email protected] https://www.redhat.com/mailman/listinfo/dm-devel