fs: dlm: fix sock release if listen fails
Subsystems (2)
Alexander Aring
maintainerPATCH 1/18 fs: dlm: fix sock release if listen fails
This patch fixes a double sock_release() call when the listen() is called for the dlm lowcomms listen socket. The caller of dlm_listen_for_all should never care about releasing the socket if dlm_listen_for_all() fails, it's done now only once if listen() fails.
--- fs/dlm/lowcomms.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
fs/dlm/lowcomms.c +1 -2
@@ -1820,7 +1820,7 @@ static int dlm_listen_for_all(void) result = sock->ops->listen (sock,5 );if (result <0 ) {dlm_close_sock (&listen_con.sock); - goto out; +return result; }return 0 ;
@@ -2023,7 +2023,6 @@ int dlm_lowcomms_start(void) dlm_proto_ops =NULL ; fail_proto_ops: dlm_allow_conn =0 ; -dlm_close_sock (&listen_con.sock);work_stop (); fail_local:deinit_local ();
Signature · 1 line
2.31.1
Alexander Aring
maintainerPATCH 2/18 fs: dlm: retry accept() until -EAGAIN or error returns
This patch fixes a race if we get two times an socket data ready event while the listen connection worker is queued. Currently it will be served only once but we need to do it (in this case twice) until we hit -EAGAIN which tells us there is no pending accept going on.
This patch wraps an do while loop until we receive a return value which is different than 0 as it was done before commit d11ccd451b65 ("fs: dlm:
listen socket out of connection hash").
--- fs/dlm/lowcomms.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-)
fs/dlm/lowcomms.c +5 -1
@@ -1543,7 +1543,11 @@ static void process_recv_sockets(struct work_struct *work)static void process_listen_recv_socket (struct work_struct *work) { -accept_from_sock (&listen_con); +int ret; + +do { + ret =accept_from_sock (&listen_con); + }while (!ret); }static void dlm_connect (struct connection *con)
Signature · 1 line
2.31.1
Alexander Aring
maintainerPATCH 3/18 fs: dlm: remove send repeat remove handling
In reply to Alexander AringThis patch removes the send repeat remove handling. This handling is there to repeatingly DLM_MSG_REMOVE messages in cases the dlm stack thinks it was not received at the first time. In cases of message drops this functionality is necessary, but since the DLM midcomms layer guarantees there are no messages drops between cluster nodes this feature became not strict necessary anymore. Due message delays/processing it could be that two send_repeat_remove() are sent out while the other should be still on it's way. We remove the repeat remove handling because we are sure that the message cannot be dropped due communication errors.
--- fs/dlm/lock.c | 74 --------------------------------------------------- 1 file changed, 74 deletions(-)
fs/dlm/lock.c +0 -74
@@ -4044,66 +4044,6 @@ static int validate_message(struct dlm_lkb *lkb, struct dlm_message *ms)return error; } -static void send_repeat_remove (struct dlm_ls *ls,char *ms_name,int len) -{ -char name[DLM_RESNAME_MAXLEN +1 ]; -struct dlm_message *ms; -struct dlm_mhandle *mh; -struct dlm_rsb *r; -uint32_t hash, b; -int rv, dir_nodeid; - -memset (name,0 ,sizeof (name)); -memcpy (name, ms_name, len); - - hash =jhash (name, len,0 ); - b = hash & (ls->ls_rsbtbl_size -1 ); - - dir_nodeid =dlm_hash2nodeid (ls, hash); - -log_error (ls,"send_repeat_remove dir %d %s" , dir_nodeid, name); - -spin_lock (&ls->ls_rsbtbl[b].lock); - rv =dlm_search_rsb_tree (&ls->ls_rsbtbl[b].keep, name, len, &r); -if (!rv) { -spin_unlock (&ls->ls_rsbtbl[b].lock); -log_error (ls,"repeat_remove on keep %s" , name); -return ; - } - - rv =dlm_search_rsb_tree (&ls->ls_rsbtbl[b].toss, name, len, &r); -if (!rv) { -spin_unlock (&ls->ls_rsbtbl[b].lock); -log_error (ls,"repeat_remove on toss %s" , name); -return ; - } - -/* use ls->remove_name2 to avoid conflict with shrink? */ - -spin_lock (&ls->ls_remove_spin); - ls->ls_remove_len = len; -memcpy (ls->ls_remove_name, name,DLM_RESNAME_MAXLEN ); -spin_unlock (&ls->ls_remove_spin); -spin_unlock (&ls->ls_rsbtbl[b].lock); - - rv =_create_message (ls,sizeof (struct dlm_message ) + len, - dir_nodeid,DLM_MSG_REMOVE , &ms, &mh); -if (rv) - goto out; - -memcpy (ms->m_extra, name, len); - ms->m_hash =cpu_to_le32 (hash); - -send_message (mh, ms); - -out: -spin_lock (&ls->ls_remove_spin); - ls->ls_remove_len =0 ; -memset (ls->ls_remove_name,0 ,DLM_RESNAME_MAXLEN ); -spin_unlock (&ls->ls_remove_spin); -wake_up (&ls->ls_remove_wait); -} -static int receive_request (struct dlm_ls *ls,struct dlm_message *ms) {struct dlm_lkb *lkb;
@@ -4173,25 +4113,11 @@ static int receive_request(struct dlm_ls *ls, struct dlm_message *ms)ENOTBLK request failures when the lookup reply designating us as master is delayed. */ -/* We could repeatedly return -EBADR here if our send_remove() is -delayed in being sent/arriving/being processed on the dir node. -Another node would repeatedly lookup up the master, and the dir -node would continue returning our nodeid until our send_remove -took effect. - -We send another remove message in case our previous send_remove -was lost/ignored/missed somehow. */ -if (error != -ENOTBLK ) {log_limit (ls,"receive_request %x from %d %d" ,le32_to_cpu (ms->m_lkid), from_nodeid, error); } -if (namelen && error == -EBADR ) { -send_repeat_remove (ls, ms->m_extra, namelen); -msleep (1000 ); - } -setup_stub_lkb (ls, ms);send_request_reply (&ls->ls_stub_rsb, &ls->ls_stub_lkb, error);return error;
Signature · 1 line
2.31.1
Alexander Aring
maintainerPATCH 4/18 fs: dlm: use packet in dlm_mhandle
In reply to Alexander AringTo allow more than just dereferencing the inner header we directly point to the inner dlm packet which allows us to dereference the header, rcom or message structure.
--- fs/dlm/midcomms.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
fs/dlm/midcomms.c +3 -3
@@ -194,7 +194,7 @@ struct midcomms_node { };struct dlm_mhandle { -const struct dlm_header *inner_hd; +const union dlm_packet *inner_p;struct midcomms_node *node;struct dlm_opts *opts;struct dlm_msg *msg;
@@ -1055,7 +1055,7 @@ static struct dlm_msg *dlm_midcomms_get_msg_3_2(struct dlm_mhandle *mh, int nodedlm_fill_opts_header (opts, len, mh->seq); *ppc +=sizeof (*opts); - mh->inner_hd = (const struct dlm_header *)*ppc; + mh->inner_p = (const union dlm_packet *)*ppc;return msg; }
@@ -1133,7 +1133,7 @@ struct dlm_mhandle *dlm_midcomms_get_mhandle(int nodeid, int len,static void dlm_midcomms_commit_msg_3_2 (struct dlm_mhandle *mh) {/* nexthdr chain for fast lookup */ - mh->opts->o_nextcmd = mh->inner_hd->h_cmd; + mh->opts->o_nextcmd = mh->inner_p->header.h_cmd; mh->committed = true;dlm_lowcomms_commit_msg (mh->msg); }
Signature · 1 line
2.31.1
Alexander Aring
maintainerPATCH 5/18 fd: dlm: trace send/recv of dlm message and rcom
In reply to Alexander AringThis patch adds tracepoints for send and recv cases of dlm messages and dlm rcom messages. In case of send and dlm message we add the dlm rsb resource name this dlm messages belongs to. This has the advantage to follow dlm messages on a per lock basis. In case of recv message the resource name can be extracted by follow the send message sequence number.
The dlm message DLM_MSG_PURGE doesn't belong to a lock request and will not set the resource name in a dlm_message trace. The same for all rcom messages.
There is additional handling required for this debugging functionality which is tried to be small as possible. Also the midcomms layer gets aware of lock resource names, for now this is required to make a connection between sequence number and lock resource names. It is for debugging purpose only.
--- fs/dlm/lock.c | 21 +-- fs/dlm/midcomms.c | 45 +++++- fs/dlm/midcomms.h | 3 +- fs/dlm/rcom.c | 4 +- include/trace/events/dlm.h | 297 +++++++++++++++++++++++++++++++++++++ 5 files changed, 353 insertions(+), 17 deletions(-)
fs/dlm/lock.c +11 -10
@@ -3611,9 +3611,10 @@ static int create_message(struct dlm_rsb *r, struct dlm_lkb *lkb,/* further lowcomms enhancements or alternate implementations may make the return value from this function useful at some point */ -static int send_message (struct dlm_mhandle *mh,struct dlm_message *ms) +static int send_message (struct dlm_mhandle *mh,struct dlm_message *ms, +const void *name,int namelen) { -dlm_midcomms_commit_mhandle (mh); +dlm_midcomms_commit_mhandle (mh, name, namelen);return 0 ; }
@@ -3679,7 +3680,7 @@ static int send_common(struct dlm_rsb *r, struct dlm_lkb *lkb, int mstype)send_args (r, lkb, ms); - error =send_message (mh, ms); + error =send_message (mh, ms, r->res_name, r->res_length);if (error) goto fail;return 0 ;
@@ -3742,7 +3743,7 @@ static int send_grant(struct dlm_rsb *r, struct dlm_lkb *lkb) ms->m_result =0 ; - error =send_message (mh, ms); + error =send_message (mh, ms, r->res_name, r->res_length); out:return error; }
@@ -3763,7 +3764,7 @@ static int send_bast(struct dlm_rsb *r, struct dlm_lkb *lkb, int mode) ms->m_bastmode =cpu_to_le32 (mode); - error =send_message (mh, ms); + error =send_message (mh, ms, r->res_name, r->res_length); out:return error; }
@@ -3786,7 +3787,7 @@ static int send_lookup(struct dlm_rsb *r, struct dlm_lkb *lkb)send_args (r, lkb, ms); - error =send_message (mh, ms); + error =send_message (mh, ms, r->res_name, r->res_length);if (error) goto fail;return 0 ;
@@ -3811,7 +3812,7 @@ static int send_remove(struct dlm_rsb *r)memcpy (ms->m_extra, r->res_name, r->res_length); ms->m_hash =cpu_to_le32 (r->res_hash); - error =send_message (mh, ms); + error =send_message (mh, ms, r->res_name, r->res_length); out:return error; }
@@ -3833,7 +3834,7 @@ static int send_common_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, ms->m_result =cpu_to_le32 (to_dlm_errno (rv)); - error =send_message (mh, ms); + error =send_message (mh, ms, r->res_name, r->res_length); out:return error; }
@@ -3874,7 +3875,7 @@ static int send_lookup_reply(struct dlm_ls *ls, struct dlm_message *ms_in, ms->m_result =cpu_to_le32 (to_dlm_errno (rv)); ms->m_nodeid =cpu_to_le32 (ret_nodeid); - error =send_message (mh, ms); + error =send_message (mh, ms, ms_in->m_extra,receive_extralen (ms_in)); out:return error; }
@@ -6300,7 +6301,7 @@ static int send_purge(struct dlm_ls *ls, int nodeid, int pid) ms->m_nodeid =cpu_to_le32 (nodeid); ms->m_pid =cpu_to_le32 (pid); -return send_message (mh, ms); +return send_message (mh, ms,NULL ,0 ); } int dlm_user_purge(struct dlm_ls *ls, struct dlm_user_proc *proc,
fs/dlm/midcomms.c +41 -4
92 lines not on this page · raw message
fs/dlm/midcomms.h +2 -1
@@ -17,7 +17,8 @@ struct midcomms_node;int dlm_process_incoming_buffer (int nodeid,unsigned char *buf,int buflen);struct dlm_mhandle *dlm_midcomms_get_mhandle (int nodeid,int len,gfp_t allocation,char **ppc); -void dlm_midcomms_commit_mhandle (struct dlm_mhandle *mh); +void dlm_midcomms_commit_mhandle (struct dlm_mhandle *mh,const void *name, +int namelen);int dlm_midcomms_close (int nodeid);int dlm_midcomms_start (void );void dlm_midcomms_shutdown (void );
fs/dlm/rcom.c +2 -2
@@ -91,7 +91,7 @@ static int create_rcom_stateless(struct dlm_ls *ls, int to_nodeid, int type,static void send_rcom (struct dlm_mhandle *mh,struct dlm_rcom *rc) { -dlm_midcomms_commit_mhandle (mh); +dlm_midcomms_commit_mhandle (mh,NULL ,0 ); }static void send_rcom_stateless (struct dlm_msg *msg,struct dlm_rcom *rc)
@@ -516,7 +516,7 @@ int dlm_send_ls_not_ready(int nodeid, struct dlm_rcom *rc_in) rf = (struct rcom_config *) rc->rc_buf; rf->rf_lvblen =cpu_to_le32 (~0U ); -dlm_midcomms_commit_mhandle (mh); +dlm_midcomms_commit_mhandle (mh,NULL ,0 );return 0 ; }
include/trace/events/dlm.h +297 -0
309 lines not on this page · raw message
Signature · 1 line
2.31.1
Alexander Aring
maintainerPATCH 6/18 fs: dlm: let dlm_add_cb queue work after resume only
In reply to Alexander AringWe should allow dlm_add_cb() to call queue_work() only after the recovery queued pending for delayed lkbs. This patch will move the switch LSFL_CB_DELAY after the delayed lkb work was processed.
--- fs/dlm/ast.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
fs/dlm/ast.c +2 -2
@@ -308,8 +308,6 @@ void dlm_callback_resume(struct dlm_ls *ls)if (!ls->ls_callback_wq)return ; -clear_bit (LSFL_CB_DELAY , &ls->ls_flags); - more:mutex_lock (&ls->ls_cb_mutex);list_for_each_entry_safe (lkb, safe, &ls->ls_cb_delay, lkb_cb_list) {
@@ -320,6 +318,8 @@ void dlm_callback_resume(struct dlm_ls *ls)break ; } empty =list_empty (&ls->ls_cb_delay); +if (empty) +clear_bit (LSFL_CB_DELAY , &ls->ls_flags);mutex_unlock (&ls->ls_cb_mutex); sum += count;
Signature · 1 line
2.31.1
Alexander Aring
maintainerPATCH 7/18 fs: dlm: use list_first_entry marco
In reply to Alexander AringInstead of using list_entry() this patch moves to using the list_first_entry() macro.
--- fs/dlm/user.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
fs/dlm/user.c +1 -1
@@ -857,7 +857,7 @@ static ssize_t device_read(struct file *file, char __user *buf, size_t count,without removing lkb_cb_list; so empty lkb_cb_list is always consistent with empty lkb_callbacks */ - lkb = list_entry(proc->asts.next, struct dlm_lkb, lkb_cb_list); + lkb = list_first_entry(&proc->asts, struct dlm_lkb, lkb_cb_list);/* rem_lkb_callback sets a new lkb_last_cast */ old_mode = lkb->lkb_last_cast.mode;
Signature · 1 line
2.31.1
Alexander Aring
maintainerPATCH 8/18 fs: dlm: convert ls_cb_mutex mutex to spinlock
In reply to Alexander AringThis patch converts the ls_cb_mutex mutex to a spinlock, there is no sleepable context when this lock is held.
--- fs/dlm/ast.c | 12 ++++++------ fs/dlm/dlm_internal.h | 2 +- fs/dlm/lockspace.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-)
fs/dlm/ast.c +6 -6
@@ -200,13 +200,13 @@ void dlm_add_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, int status,if (!prev_seq) {kref_get (&lkb->lkb_ref); -mutex_lock (&ls->ls_cb_mutex); +spin_lock (&ls->ls_cb_lock);if (test_bit (LSFL_CB_DELAY , &ls->ls_flags)) {list_add (&lkb->lkb_cb_list, &ls->ls_cb_delay); }else {queue_work (ls->ls_callback_wq, &lkb->lkb_cb_work); } -mutex_unlock (&ls->ls_cb_mutex); +spin_unlock (&ls->ls_cb_lock); } out:mutex_unlock (&lkb->lkb_cb_mutex);
@@ -289,9 +289,9 @@ void dlm_callback_stop(struct dlm_ls *ls)void dlm_callback_suspend (struct dlm_ls *ls) {if (ls->ls_callback_wq) { -mutex_lock (&ls->ls_cb_mutex); +spin_lock (&ls->ls_cb_lock);set_bit (LSFL_CB_DELAY , &ls->ls_flags); -mutex_unlock (&ls->ls_cb_mutex); +spin_unlock (&ls->ls_cb_lock);flush_workqueue (ls->ls_callback_wq); }
@@ -309,7 +309,7 @@ void dlm_callback_resume(struct dlm_ls *ls)return ; more: -mutex_lock (&ls->ls_cb_mutex); +spin_lock (&ls->ls_cb_lock);list_for_each_entry_safe (lkb, safe, &ls->ls_cb_delay, lkb_cb_list) {list_del_init (&lkb->lkb_cb_list);queue_work (ls->ls_callback_wq, &lkb->lkb_cb_work);
@@ -320,7 +320,7 @@ void dlm_callback_resume(struct dlm_ls *ls) empty =list_empty (&ls->ls_cb_delay);if (empty)clear_bit (LSFL_CB_DELAY , &ls->ls_flags); -mutex_unlock (&ls->ls_cb_mutex); +spin_unlock (&ls->ls_cb_lock); sum += count;if (!empty) {
fs/dlm/dlm_internal.h +1 -1
@@ -631,7 +631,7 @@ struct dlm_ls {/* recovery related */ -struct mutex ls_cb_mutex; +spinlock_t ls_cb_lock;struct list_head ls_cb_delay;/* save for queue_work later */ struct timer_list ls_timer;struct task_struct *ls_recoverd_task;
fs/dlm/lockspace.c +1 -1
@@ -567,7 +567,7 @@ static int new_lockspace(const char *name, const char *cluster,init_completion (&ls->ls_recovery_done); ls->ls_recovery_result =-1 ; -mutex_init (&ls->ls_cb_mutex); +spin_lock_init (&ls->ls_cb_lock);INIT_LIST_HEAD (&ls->ls_cb_delay); ls->ls_recoverd_task =NULL ;
Signature · 1 line
2.31.1
Alexander Aring
maintainerPATCH 9/18 fs: dlm: use spin lock instead of mutex
In reply to Alexander AringThere is no need to use a mutex in those hot path sections. We change it to spin lock to serve callbacks more faster by not allowing schedule. The locked sections will not be locked for a long time.
--- fs/dlm/ast.c | 8 ++++---- fs/dlm/dlm_internal.h | 2 +- fs/dlm/lock.c | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-)
fs/dlm/ast.c +4 -4
@@ -190,7 +190,7 @@ void dlm_add_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, int status,return ; } -mutex_lock (&lkb->lkb_cb_mutex); +spin_lock (&lkb->lkb_cb_lock); prev_seq = lkb->lkb_callbacks[0 ].seq; rv =dlm_add_lkb_callback (lkb, flags, mode, status, sbflags, new_seq);
@@ -209,7 +209,7 @@ void dlm_add_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, int status,spin_unlock (&ls->ls_cb_lock); } out: -mutex_unlock (&lkb->lkb_cb_mutex); +spin_unlock (&lkb->lkb_cb_lock); }void dlm_callback_work (struct work_struct *work)
@@ -223,7 +223,7 @@ void dlm_callback_work(struct work_struct *work)memset (&callbacks,0 ,sizeof (callbacks)); -mutex_lock (&lkb->lkb_cb_mutex); +spin_lock (&lkb->lkb_cb_lock);if (!lkb->lkb_callbacks[0 ].seq) {/* no callback work exists, shouldn't happen */ log_error (ls,"dlm_callback_work %x no work" , lkb->lkb_id);
@@ -244,7 +244,7 @@ void dlm_callback_work(struct work_struct *work)dlm_print_lkb (lkb);dlm_dump_lkb_callbacks (lkb); } -mutex_unlock (&lkb->lkb_cb_mutex); +spin_unlock (&lkb->lkb_cb_lock); castfn = lkb->lkb_astfn; bastfn = lkb->lkb_bastfn;
fs/dlm/dlm_internal.h +1 -1
@@ -268,7 +268,7 @@ struct dlm_lkb {unsigned long lkb_timeout_cs;#endif -struct mutex lkb_cb_mutex; +spinlock_t lkb_cb_lock;struct work_struct lkb_cb_work;struct list_head lkb_cb_list;/* for ls_cb_delay or proc->asts */ struct dlm_callback lkb_callbacks[DLM_CALLBACKS_SIZE ];
fs/dlm/lock.c +1 -1
@@ -1218,7 +1218,7 @@ static int _create_lkb(struct dlm_ls *ls, struct dlm_lkb **lkb_ret,INIT_LIST_HEAD (&lkb->lkb_time_list);#endif INIT_LIST_HEAD (&lkb->lkb_cb_list); -mutex_init (&lkb->lkb_cb_mutex); +spin_lock_init (&lkb->lkb_cb_lock);INIT_WORK (&lkb->lkb_cb_work, dlm_callback_work);idr_preload (GFP_NOFS );
Signature · 1 line
2.31.1
Alexander Aring
maintainerPATCH 10/18 fs: dlm: move last cast bast time to function call
In reply to Alexander AringThis patch moves the debugging information of the last cast and bast time when calling the last and bast function call.
--- fs/dlm/ast.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-)
fs/dlm/ast.c +4 -6
@@ -158,15 +158,11 @@ int dlm_rem_lkb_callback(struct dlm_ls *ls, struct dlm_lkb *lkb, } } -if (cb->flags &DLM_CB_CAST ) { +if (cb->flags &DLM_CB_CAST )memcpy (&lkb->lkb_last_cast, cb,sizeof (struct dlm_callback )); - lkb->lkb_last_cast_time =ktime_get (); - } -if (cb->flags &DLM_CB_BAST ) { +if (cb->flags &DLM_CB_BAST )memcpy (&lkb->lkb_last_bast, cb,sizeof (struct dlm_callback )); - lkb->lkb_last_bast_time =ktime_get (); - } rv =0 ; out:return rv;
@@ -256,11 +252,13 @@ void dlm_callback_work(struct work_struct *work)continue ; }else if (callbacks[i].flags &DLM_CB_BAST ) {trace_dlm_bast (ls, lkb, callbacks[i].mode); + lkb->lkb_last_bast_time =ktime_get ();bastfn (lkb->lkb_astparam, callbacks[i].mode); }else if (callbacks[i].flags &DLM_CB_CAST ) { lkb->lkb_lksb->sb_status = callbacks[i].sb_status; lkb->lkb_lksb->sb_flags = callbacks[i].sb_flags;trace_dlm_ast (ls, lkb); + lkb->lkb_last_cast_time =ktime_get ();castfn (lkb->lkb_astparam); } }
Signature · 1 line
2.31.1
Alexander Aring
maintainerPATCH 11/18 fs: dlm: use a non-static queue for callbacks
In reply to Alexander AringThis patch will introducde a queue implementation for callbacks by using the Linux lists. The current callback queue handling is implemented by a static limit of 6 entries, see DLM_CALLBACKS_SIZE. The sequence number inside the callback structure was used to see if the entries inside the static entry is valid or not. We don't need any sequence numbers anymore with a dynamic datastructure with grows and shrinks during runtime to offer such functionality.
We assume that every callback will be delivered to the DLM user if once queued. Therefore the callback flag DLM_CB_SKIP was dropped and the check for skipping bast was moved before worker handling and not skip while the callback worker executes. This will reduce unnecessary queues of the callback worker.
All last callback saves are pointers now and don't need to copied over. There is a reference counter for callback structures which will care about to free the callback structures at the right time if they are not referenced anymore.
--- fs/dlm/ast.c | 294 ++++++++++++++++++------------------------ fs/dlm/ast.h | 17 ++- fs/dlm/debug_fs.c | 2 +- fs/dlm/dlm_internal.h | 15 ++- fs/dlm/lock.c | 8 +- fs/dlm/memory.c | 26 ++++ fs/dlm/memory.h | 2 + fs/dlm/user.c | 73 ++++++----- fs/dlm/user.h | 2 +- 9 files changed, 222 insertions(+), 217 deletions(-)
fs/dlm/ast.c +124 -170
@@ -12,55 +12,68 @@#include <trace/events/dlm.h> #include "dlm_internal.h" +#include "memory.h" #include "lock.h" #include "user.h" #include "ast.h" -static uint64_t dlm_cb_seq; -static DEFINE_SPINLOCK (dlm_cb_seq_spin); +void dlm_release_callback (struct kref *ref) +{ + struct dlm_callback *cb = container_of(ref, struct dlm_callback, ref); + +dlm_free_cb (cb); +} + +void dlm_callback_set_last_ptr (struct dlm_callback **from, +struct dlm_callback *to) +{ +if (*from) +kref_put (&(*from)->ref, dlm_release_callback); + +if (to) +kref_get (&to->ref); + + *from = to; +} -static void dlm_dump_lkb_callbacks (struct dlm_lkb *lkb) +void dlm_purge_lkb_callbacks (struct dlm_lkb *lkb) { -int i; - -log_print ("last_bast %x %llu flags %x mode %d sb %d %x" , - lkb->lkb_id, - (unsigned long long )lkb->lkb_last_bast.seq, - lkb->lkb_last_bast.flags, - lkb->lkb_last_bast.mode, - lkb->lkb_last_bast.sb_status, - lkb->lkb_last_bast.sb_flags); - -log_print ("last_cast %x %llu flags %x mode %d sb %d %x" , - lkb->lkb_id, - (unsigned long long )lkb->lkb_last_cast.seq, - lkb->lkb_last_cast.flags, - lkb->lkb_last_cast.mode, - lkb->lkb_last_cast.sb_status, - lkb->lkb_last_cast.sb_flags); - -for (i =0 ; i <DLM_CALLBACKS_SIZE ; i++) { -log_print ("cb %x %llu flags %x mode %d sb %d %x" , - lkb->lkb_id, - (unsigned long long )lkb->lkb_callbacks[i].seq, - lkb->lkb_callbacks[i].flags, - lkb->lkb_callbacks[i].mode, - lkb->lkb_callbacks[i].sb_status, - lkb->lkb_callbacks[i].sb_flags); +struct dlm_callback *cb, *safe; + +list_for_each_entry_safe (cb, safe, &lkb->lkb_callbacks, list) { +list_del (&cb->list); +kref_put (&cb->ref, dlm_release_callback); } + +/* TODO */ + lkb->lkb_flags &= ~DLM_IFL_NEED_SCHED ; + +/* invalidate */ +dlm_callback_set_last_ptr (&lkb->lkb_last_cast,NULL ); +dlm_callback_set_last_ptr (&lkb->lkb_last_cb,NULL ); + lkb->lkb_last_bast_mode =-1 ; } -int dlm_add_lkb_callback (struct dlm_lkb *lkb,uint32_t flags,int mode, -int status,uint32_t sbflags,uint64_t seq) +int dlm_enqueue_lkb_callback (struct dlm_lkb *lkb,uint32_t flags,int mode, +int status,uint32_t sbflags) {struct dlm_ls *ls = lkb->lkb_resource->res_ls; -uint64_t prev_seq; +int rv =DLM_ENQUEUE_CALLBACK_SUCCESS ; +struct dlm_callback *cb;int prev_mode; -int i, rv; -for (i =0 ; i <DLM_CALLBACKS_SIZE ; i++) { -if (lkb->lkb_callbacks[i].seq) -continue ; +if (flags &DLM_CB_BAST ) { +/* if cb is a bast, it should be skipped if the blocking mode is +* compatible with the last granted mode +*/ +if (lkb->lkb_last_cast) { +if (dlm_modes_compat (mode, lkb->lkb_last_cast->mode)) { +log_debug (ls,"skip %x bast mode %d for cast mode %d" , + lkb->lkb_id, mode, + lkb->lkb_last_cast->mode); + goto out; + } + }/* * Suppress some redundant basts here, do more on removal.
@@ -68,132 +81,75 @@ int dlm_add_lkb_callback(struct dlm_lkb *lkb, uint32_t flags, int mode,* is a bast for the same mode or a more restrictive mode. * (the addional > PR check is needed for PR/CW inversion) */ - -if ((i >0 ) && (flags &DLM_CB_BAST ) && - (lkb->lkb_callbacks[i-1 ].flags &DLM_CB_BAST )) { - - prev_seq = lkb->lkb_callbacks[i-1 ].seq; - prev_mode = lkb->lkb_callbacks[i-1 ].mode; +if (lkb->lkb_last_cb && lkb->lkb_last_cb->flags &DLM_CB_BAST ) { + prev_mode = lkb->lkb_last_cb->mode;if ((prev_mode == mode) || (prev_mode > mode && prev_mode >DLM_LOCK_PR )) { - -log_debug (ls,"skip %x add bast %llu mode %d " -"for bast %llu mode %d" , - lkb->lkb_id, - (unsigned long long )seq, - mode, - (unsigned long long )prev_seq, - prev_mode); - rv =0 ; +log_debug (ls,"skip %x add bast mode %d for bast mode %d" , + lkb->lkb_id, mode, prev_mode); goto out; } } - - lkb->lkb_callbacks[i].seq = seq; - lkb->lkb_callbacks[i].flags = flags; - lkb->lkb_callbacks[i].mode = mode; - lkb->lkb_callbacks[i].sb_status = status; - lkb->lkb_callbacks[i].sb_flags = (sbflags &0x000000FF ); - rv =0 ; -break ; } -if (i ==DLM_CALLBACKS_SIZE ) { -log_error (ls,"no callbacks %x %llu flags %x mode %d sb %d %x" , - lkb->lkb_id, (unsigned long long )seq, - flags, mode, status, sbflags); -dlm_dump_lkb_callbacks (lkb); - rv =-1 ; + cb =dlm_allocate_cb (); +if (!cb) { + rv =DLM_ENQUEUE_CALLBACK_FAILURE ; goto out; } - out: -return rv; -} - -int dlm_rem_lkb_callback (struct dlm_ls *ls,struct dlm_lkb *lkb, -struct dlm_callback *cb,int *resid) -{ -int i, rv; - - *resid =0 ; - -if (!lkb->lkb_callbacks[0 ].seq) { - rv = -ENOENT ; - goto out; - } - -/* oldest undelivered cb is callbacks[0] */ - -memcpy (cb, &lkb->lkb_callbacks[0 ],sizeof (struct dlm_callback )); -memset (&lkb->lkb_callbacks[0 ],0 ,sizeof (struct dlm_callback )); -/* shift others down */ - -for (i =1 ; i <DLM_CALLBACKS_SIZE ; i++) { -if (!lkb->lkb_callbacks[i].seq) -break ; -memcpy (&lkb->lkb_callbacks[i-1 ], &lkb->lkb_callbacks[i], -sizeof (struct dlm_callback )); -memset (&lkb->lkb_callbacks[i],0 ,sizeof (struct dlm_callback )); - (*resid)++; + cb->flags = flags; + cb->mode = mode; + cb->sb_status = status; + cb->sb_flags = (sbflags &0x000000FF ); +kref_init (&cb->ref); +if (!(lkb->lkb_flags &DLM_IFL_NEED_SCHED )) { + lkb->lkb_flags |=DLM_IFL_NEED_SCHED ; + rv =DLM_ENQUEUE_CALLBACK_NEED_SCHED ; } +list_add_tail (&cb->list, &lkb->lkb_callbacks); -/* if cb is a bast, it should be skipped if the blocking mode is -compatible with the last granted mode */ - -if ((cb->flags &DLM_CB_BAST ) && lkb->lkb_last_cast.seq) { -if (dlm_modes_compat (cb->mode, lkb->lkb_last_cast.mode)) { - cb->flags |=DLM_CB_SKIP ; - -log_debug (ls,"skip %x bast %llu mode %d " -"for cast %llu mode %d" , - lkb->lkb_id, - (unsigned long long )cb->seq, - cb->mode, - (unsigned long long )lkb->lkb_last_cast.seq, - lkb->lkb_last_cast.mode); - rv =0 ; - goto out; - } - } +if (flags &DLM_CB_CAST ) +dlm_callback_set_last_ptr (&lkb->lkb_last_cast, cb); -if (cb->flags &DLM_CB_CAST ) -memcpy (&lkb->lkb_last_cast, cb,sizeof (struct dlm_callback )); +dlm_callback_set_last_ptr (&lkb->lkb_last_cb, cb); -if (cb->flags &DLM_CB_BAST ) -memcpy (&lkb->lkb_last_bast, cb,sizeof (struct dlm_callback )); - rv =0 ; out:return rv; } +int dlm_dequeue_lkb_callback (struct dlm_lkb *lkb,struct dlm_callback **cb) +{ +/* oldest undelivered cb is callbacks first entry */ + *cb =list_first_entry_or_null (&lkb->lkb_callbacks, + struct dlm_callback, list); +if (!*cb) +return DLM_DEQUEUE_CALLBACK_EMPTY ; + +/* remove it from callbacks so shift others down */ +list_del (&(*cb)->list); +if (list_empty (&lkb->lkb_callbacks)) +return DLM_DEQUEUE_CALLBACK_LAST ; + +return DLM_DEQUEUE_CALLBACK_SUCCESS ; +} +void dlm_add_cb (struct dlm_lkb *lkb,uint32_t flags,int mode,int status,uint32_t sbflags) {struct dlm_ls *ls = lkb->lkb_resource->res_ls; -uint64_t new_seq, prev_seq;int rv; -spin_lock (&dlm_cb_seq_spin); - new_seq = ++dlm_cb_seq; -if (!dlm_cb_seq) - new_seq = ++dlm_cb_seq; -spin_unlock (&dlm_cb_seq_spin); -if (lkb->lkb_flags &DLM_IFL_USER ) { -dlm_user_add_ast (lkb, flags, mode, status, sbflags, new_seq); +dlm_user_add_ast (lkb, flags, mode, status, sbflags);return ; }spin_lock (&lkb->lkb_cb_lock); - prev_seq = lkb->lkb_callbacks[0 ].seq; - - rv =dlm_add_lkb_callback (lkb, flags, mode, status, sbflags, new_seq); -if (rv <0 ) - goto out; - -if (!prev_seq) { + rv =dlm_enqueue_lkb_callback (lkb, flags, mode, status, sbflags); +switch (rv) { +case DLM_ENQUEUE_CALLBACK_NEED_SCHED :kref_get (&lkb->lkb_ref);spin_lock (&ls->ls_cb_lock);
@@ -203,8 +159,16 @@ void dlm_add_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, int status,queue_work (ls->ls_callback_wq, &lkb->lkb_cb_work); }spin_unlock (&ls->ls_cb_lock); +break ; +case DLM_ENQUEUE_CALLBACK_FAILURE : +WARN_ON (1 ); +break ; +case DLM_ENQUEUE_CALLBACK_SUCCESS : +break ; +default : +WARN_ON (1 ); +break ; } - out:spin_unlock (&lkb->lkb_cb_lock); }
@@ -214,53 +178,43 @@ void dlm_callback_work(struct work_struct *work)struct dlm_ls *ls = lkb->lkb_resource->res_ls;void (*castfn) (void *astparam);void (*bastfn) (void *astparam,int mode); -struct dlm_callback callbacks[DLM_CALLBACKS_SIZE ]; -int i, rv, resid; - -memset (&callbacks,0 ,sizeof (callbacks)); +struct dlm_callback *cb; +int rv;spin_lock (&lkb->lkb_cb_lock); -if (!lkb->lkb_callbacks[0 ].seq) { -/* no callback work exists, shouldn't happen */ -log_error (ls,"dlm_callback_work %x no work" , lkb->lkb_id); -dlm_print_lkb (lkb); -dlm_dump_lkb_callbacks (lkb); - } - -for (i =0 ; i <DLM_CALLBACKS_SIZE ; i++) { - rv =dlm_rem_lkb_callback (ls, lkb, &callbacks[i], &resid); -if (rv <0 ) -break ; - } - -if (resid) { -/* cbs remain, loop should have removed all, shouldn't happen */ -log_error (ls,"dlm_callback_work %x resid %d" , lkb->lkb_id, - resid); -dlm_print_lkb (lkb); -dlm_dump_lkb_callbacks (lkb); - } + rv =dlm_dequeue_lkb_callback (lkb, &cb);spin_unlock (&lkb->lkb_cb_lock); - castfn = lkb->lkb_astfn; - bastfn = lkb->lkb_bastfn; +if (WARN_ON (rv ==DLM_DEQUEUE_CALLBACK_EMPTY )) +return ; -for (i =0 ; i <DLM_CALLBACKS_SIZE ; i++) { -if (!callbacks[i].seq) -break ; -if (callbacks[i].flags &DLM_CB_SKIP ) { -continue ; - }else if (callbacks[i].flags &DLM_CB_BAST ) { -trace_dlm_bast (ls, lkb, callbacks[i].mode); +for (;;) { + castfn = lkb->lkb_astfn; + bastfn = lkb->lkb_bastfn; + +if (cb->flags &DLM_CB_BAST ) { +trace_dlm_bast (ls, lkb, cb->mode); lkb->lkb_last_bast_time =ktime_get (); -bastfn (lkb->lkb_astparam, callbacks[i].mode); - }else if (callbacks[i].flags &DLM_CB_CAST ) { - lkb->lkb_lksb->sb_status = callbacks[i].sb_status; - lkb->lkb_lksb->sb_flags = callbacks[i].sb_flags; + lkb->lkb_last_bast_mode = cb->mode; +bastfn (lkb->lkb_astparam, cb->mode); + }else if (cb->flags &DLM_CB_CAST ) { + lkb->lkb_lksb->sb_status = cb->sb_status; + lkb->lkb_lksb->sb_flags = cb->sb_flags;trace_dlm_ast (ls, lkb); lkb->lkb_last_cast_time =ktime_get ();castfn (lkb->lkb_astparam); } + +kref_put (&cb->ref, dlm_release_callback); + +spin_lock (&lkb->lkb_cb_lock); + rv =dlm_dequeue_lkb_callback (lkb, &cb); +if (rv ==DLM_DEQUEUE_CALLBACK_EMPTY ) { + lkb->lkb_flags &= ~DLM_IFL_NEED_SCHED ; +spin_unlock (&lkb->lkb_cb_lock); +break ; + } +spin_unlock (&lkb->lkb_cb_lock); }/* undo kref_get from dlm_add_callback, may cause lkb to be freed */
fs/dlm/ast.h +13 -4
@@ -11,13 +11,22 @@#ifndef __ASTD_DOT_H__#define __ASTD_DOT_H__ -int dlm_add_lkb_callback (struct dlm_lkb *lkb,uint32_t flags,int mode, -int status,uint32_t sbflags,uint64_t seq); -int dlm_rem_lkb_callback (struct dlm_ls *ls,struct dlm_lkb *lkb, -struct dlm_callback *cb,int *resid); +#define DLM_ENQUEUE_CALLBACK_NEED_SCHED 1 +#define DLM_ENQUEUE_CALLBACK_SUCCESS 0 +#define DLM_ENQUEUE_CALLBACK_FAILURE -1 +int dlm_enqueue_lkb_callback (struct dlm_lkb *lkb,uint32_t flags,int mode, +int status,uint32_t sbflags); +#define DLM_DEQUEUE_CALLBACK_EMPTY 2 +#define DLM_DEQUEUE_CALLBACK_LAST 1 +#define DLM_DEQUEUE_CALLBACK_SUCCESS 0 +int dlm_dequeue_lkb_callback (struct dlm_lkb *lkb,struct dlm_callback **cb);void dlm_add_cb (struct dlm_lkb *lkb,uint32_t flags,int mode,int status,uint32_t sbflags); +void dlm_callback_set_last_ptr (struct dlm_callback **from, +struct dlm_callback *to); +void dlm_release_callback (struct kref *ref); +void dlm_purge_lkb_callbacks (struct dlm_lkb *lkb);void dlm_callback_work (struct work_struct *work);int dlm_callback_start (struct dlm_ls *ls);void dlm_callback_stop (struct dlm_ls *ls);
fs/dlm/debug_fs.c +1 -1
@@ -246,7 +246,7 @@ static void print_format3_lock(struct seq_file *s, struct dlm_lkb *lkb, lkb->lkb_status, lkb->lkb_grmode, lkb->lkb_rqmode, - lkb->lkb_last_bast.mode, + lkb->lkb_last_bast_mode, rsb_lookup, lkb->lkb_wait_type, lkb->lkb_lvbseq,
fs/dlm/dlm_internal.h +8 -7
@@ -211,6 +211,7 @@ struct dlm_args {#endif #define DLM_IFL_DEADLOCK_CANCEL 0x01000000#define DLM_IFL_STUB_MS 0x02000000/* magic number for m_flags */ +#define DLM_IFL_NEED_SCHED 0x04000000/* least significant 2 bytes are message changed, they are full transmitted * but at receive side only the 2 bytes LSB will be set. *
@@ -222,18 +223,17 @@ struct dlm_args {#define DLM_IFL_USER 0x00000001#define DLM_IFL_ORPHAN 0x00000002 -#define DLM_CALLBACKS_SIZE 6 -#define DLM_CB_CAST 0x00000001#define DLM_CB_BAST 0x00000002 -#define DLM_CB_SKIP 0x00000004struct dlm_callback { -uint64_t seq;uint32_t flags;/* DLM_CBF_ */ int sb_status;/* copy to lksb status */ uint8_t sb_flags;/* copy to lksb flags */ int8_t mode;/* rq mode of bast, gr mode of cast */ + +struct list_head list; +struct kref ref; };struct dlm_lkb {
@@ -271,9 +271,10 @@ struct dlm_lkb {spinlock_t lkb_cb_lock;struct work_struct lkb_cb_work;struct list_head lkb_cb_list;/* for ls_cb_delay or proc->asts */ -struct dlm_callback lkb_callbacks[DLM_CALLBACKS_SIZE ]; -struct dlm_callback lkb_last_cast; -struct dlm_callback lkb_last_bast; +struct list_head lkb_callbacks; +struct dlm_callback *lkb_last_cast; +struct dlm_callback *lkb_last_cb; +int lkb_last_bast_mode;ktime_t lkb_last_cast_time;/* for debugging */ ktime_t lkb_last_bast_time;/* for debugging */
fs/dlm/lock.c +4 -4
@@ -1209,6 +1209,7 @@ static int _create_lkb(struct dlm_ls *ls, struct dlm_lkb **lkb_ret,if (!lkb)return -ENOMEM ; + lkb->lkb_last_bast_mode =-1 ; lkb->lkb_nodeid =-1 ; lkb->lkb_grmode =DLM_LOCK_IV ;kref_init (&lkb->lkb_ref);
@@ -1218,6 +1219,7 @@ static int _create_lkb(struct dlm_ls *ls, struct dlm_lkb **lkb_ret,INIT_LIST_HEAD (&lkb->lkb_time_list);#endif INIT_LIST_HEAD (&lkb->lkb_cb_list); +INIT_LIST_HEAD (&lkb->lkb_callbacks);spin_lock_init (&lkb->lkb_cb_lock);INIT_WORK (&lkb->lkb_cb_work, dlm_callback_work);
@@ -6221,8 +6223,7 @@ void dlm_clear_proc_locks(struct dlm_ls *ls, struct dlm_user_proc *proc) }list_for_each_entry_safe (lkb, safe, &proc->asts, lkb_cb_list) { -memset (&lkb->lkb_callbacks,0 , -sizeof (struct dlm_callback ) *DLM_CALLBACKS_SIZE ); +dlm_purge_lkb_callbacks (lkb);list_del_init (&lkb->lkb_cb_list);dlm_put_lkb (lkb); }
@@ -6263,8 +6264,7 @@ static void purge_proc_locks(struct dlm_ls *ls, struct dlm_user_proc *proc)spin_lock (&proc->asts_spin);list_for_each_entry_safe (lkb, safe, &proc->asts, lkb_cb_list) { -memset (&lkb->lkb_callbacks,0 , -sizeof (struct dlm_callback ) *DLM_CALLBACKS_SIZE ); +dlm_purge_lkb_callbacks (lkb);list_del_init (&lkb->lkb_cb_list);dlm_put_lkb (lkb); }
fs/dlm/memory.c +26 -0
@@ -14,12 +14,14 @@#include "lowcomms.h" #include "config.h" #include "memory.h" +#include "ast.h" static struct kmem_cache *writequeue_cache;static struct kmem_cache *mhandle_cache;static struct kmem_cache *msg_cache;static struct kmem_cache *lkb_cache;static struct kmem_cache *rsb_cache; +static struct kmem_cache *cb_cache;int __init dlm_memory_init (void )
@@ -46,8 +48,16 @@ int __init dlm_memory_init(void)if (!rsb_cache) goto rsb; + cb_cache =kmem_cache_create ("dlm_cb" ,sizeof (struct dlm_callback ), + __alignof__(struct dlm_callback ),0 , +NULL ); +if (!rsb_cache) + goto cb; +return 0 ; +cb: +kmem_cache_destroy (rsb_cache); rsb:kmem_cache_destroy (msg_cache); msg:
@@ -67,6 +77,7 @@ void dlm_memory_exit(void)kmem_cache_destroy (msg_cache);kmem_cache_destroy (lkb_cache);kmem_cache_destroy (rsb_cache); +kmem_cache_destroy (cb_cache); }char *dlm_allocate_lvb (struct dlm_ls *ls)
@@ -115,6 +126,11 @@ void dlm_free_lkb(struct dlm_lkb *lkb)kfree (ua); } } + +/* drop references if they are set */ +dlm_callback_set_last_ptr (&lkb->lkb_last_cast,NULL ); +dlm_callback_set_last_ptr (&lkb->lkb_last_cb,NULL ); +kmem_cache_free (lkb_cache, lkb); }
@@ -147,3 +163,13 @@ void dlm_free_msg(struct dlm_msg *msg) {kmem_cache_free (msg_cache, msg); } + +struct dlm_callback *dlm_allocate_cb (void ) +{ +return kmem_cache_alloc (cb_cache,GFP_ATOMIC ); +} + +void dlm_free_cb (struct dlm_callback *cb) +{ +kmem_cache_free (cb_cache, cb); +}
fs/dlm/memory.h +2 -0
@@ -26,6 +26,8 @@ struct writequeue_entry *dlm_allocate_writequeue(void);void dlm_free_writequeue (struct writequeue_entry *writequeue);struct dlm_msg *dlm_allocate_msg (gfp_t allocation);void dlm_free_msg (struct dlm_msg *msg); +struct dlm_callback *dlm_allocate_cb (void ); +void dlm_free_cb (struct dlm_callback *cb);#endif /* __MEMORY_DOT_H__ */
fs/dlm/user.c +43 -30
130 lines not on this page · raw message
fs/dlm/user.h +1 -1
@@ -7,7 +7,7 @@#define __USER_DOT_H__void dlm_user_add_ast (struct dlm_lkb *lkb,uint32_t flags,int mode, -int status,uint32_t sbflags,uint64_t seq); +int status,uint32_t sbflags);int dlm_user_init (void );void dlm_user_exit (void );int dlm_device_deregister (struct dlm_ls *ls);
Signature · 1 line
2.31.1
Alexander Aring
maintainerPATCH 11/18 fs: dlm: use a non-static queue for callbacks
Hi,
On Thu, Oct 27, 2022 at 4:45 PM Alexander Aring <[email protected]> wrote:
...
fs/dlm/ast.c line 49
Quoted 3 lines
fs/dlm/ast.c +3
-0+ +/* TODO */ + lkb->lkb_flags &= ~DLM_IFL_NEED_SCHED ;
sorry, I think I put a TODO there because I wanted to rename this flag with DLM_IFL_CB_PENDING.
- Alex
Alexander Aring
maintainerPATCH 12/18 fs: dlm: allow different allocation context per _create_message
In reply to Alexander AringThis patch allows to give the use control about the allocation context based on a per message basis. Currently all messages forced to be created under GFP_NOFS context.
--- fs/dlm/lock.c | 31 +++++++++++++++++++------------ fs/dlm/memory.c | 4 ++-- fs/dlm/memory.h | 2 +- fs/dlm/midcomms.c | 2 +- 4 files changed, 23 insertions(+), 16 deletions(-)
fs/dlm/lock.c +19 -12
@@ -3554,7 +3554,8 @@ int dlm_unlock(dlm_lockspace_t *lockspace,static int _create_message (struct dlm_ls *ls,int mb_len,int to_nodeid,int mstype,struct dlm_message **ms_ret, -struct dlm_mhandle **mh_ret) +struct dlm_mhandle **mh_ret, +gfp_t allocation) {struct dlm_message *ms;struct dlm_mhandle *mh;
@@ -3564,7 +3565,7 @@ static int _create_message(struct dlm_ls *ls, int mb_len,pass into midcomms_commit and a message buffer (mb) that we write our data into */ - mh =dlm_midcomms_get_mhandle (to_nodeid, mb_len,GFP_NOFS , &mb); + mh =dlm_midcomms_get_mhandle (to_nodeid, mb_len, allocation, &mb);if (!mh)return -ENOBUFS ;
@@ -3586,7 +3587,8 @@ static int _create_message(struct dlm_ls *ls, int mb_len,static int create_message (struct dlm_rsb *r,struct dlm_lkb *lkb,int to_nodeid,int mstype,struct dlm_message **ms_ret, -struct dlm_mhandle **mh_ret) +struct dlm_mhandle **mh_ret, +gfp_t allocation) {int mb_len =sizeof (struct dlm_message );
@@ -3607,7 +3609,7 @@ static int create_message(struct dlm_rsb *r, struct dlm_lkb *lkb, }return _create_message (r->res_ls, mb_len, to_nodeid, mstype, - ms_ret, mh_ret); + ms_ret, mh_ret, allocation); }/* further lowcomms enhancements or alternate implementations may make
@@ -3676,7 +3678,7 @@ static int send_common(struct dlm_rsb *r, struct dlm_lkb *lkb, int mstype)if (error)return error; - error =create_message (r, lkb, to_nodeid, mstype, &ms, &mh); + error =create_message (r, lkb, to_nodeid, mstype, &ms, &mh,GFP_NOFS );if (error) goto fail;
@@ -3737,7 +3739,8 @@ static int send_grant(struct dlm_rsb *r, struct dlm_lkb *lkb) to_nodeid = lkb->lkb_nodeid; - error =create_message (r, lkb, to_nodeid,DLM_MSG_GRANT , &ms, &mh); + error =create_message (r, lkb, to_nodeid,DLM_MSG_GRANT , &ms, &mh, +GFP_NOFS );if (error) goto out;
@@ -3758,7 +3761,8 @@ static int send_bast(struct dlm_rsb *r, struct dlm_lkb *lkb, int mode) to_nodeid = lkb->lkb_nodeid; - error =create_message (r,NULL , to_nodeid,DLM_MSG_BAST , &ms, &mh); + error =create_message (r,NULL , to_nodeid,DLM_MSG_BAST , &ms, &mh, +GFP_NOFS );if (error) goto out;
@@ -3783,7 +3787,8 @@ static int send_lookup(struct dlm_rsb *r, struct dlm_lkb *lkb)if (error)return error; - error =create_message (r,NULL , to_nodeid,DLM_MSG_LOOKUP , &ms, &mh); + error =create_message (r,NULL , to_nodeid,DLM_MSG_LOOKUP , &ms, &mh, +GFP_NOFS );if (error) goto fail;
@@ -3807,7 +3812,8 @@ static int send_remove(struct dlm_rsb *r) to_nodeid =dlm_dir_nodeid (r); - error =create_message (r,NULL , to_nodeid,DLM_MSG_REMOVE , &ms, &mh); + error =create_message (r,NULL , to_nodeid,DLM_MSG_REMOVE , &ms, &mh, +GFP_NOFS );if (error) goto out;
@@ -3828,7 +3834,7 @@ static int send_common_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, to_nodeid = lkb->lkb_nodeid; - error =create_message (r, lkb, to_nodeid, mstype, &ms, &mh); + error =create_message (r, lkb, to_nodeid, mstype, &ms, &mh,GFP_NOFS );if (error) goto out;
@@ -3869,7 +3875,8 @@ static int send_lookup_reply(struct dlm_ls *ls, struct dlm_message *ms_in,struct dlm_mhandle *mh;int error, nodeid =le32_to_cpu (ms_in->m_header.h_nodeid); - error =create_message (r,NULL , nodeid,DLM_MSG_LOOKUP_REPLY , &ms, &mh); + error =create_message (r,NULL , nodeid,DLM_MSG_LOOKUP_REPLY , &ms, &mh, +GFP_NOFS );if (error) goto out;
@@ -6295,7 +6302,7 @@ static int send_purge(struct dlm_ls *ls, int nodeid, int pid)int error; error =_create_message (ls,sizeof (struct dlm_message ), nodeid, -DLM_MSG_PURGE , &ms, &mh); +DLM_MSG_PURGE , &ms, &mh,GFP_NOFS );if (error)return error; ms->m_nodeid =cpu_to_le32 (nodeid);
fs/dlm/memory.c +2 -2
@@ -134,9 +134,9 @@ void dlm_free_lkb(struct dlm_lkb *lkb)kmem_cache_free (lkb_cache, lkb); } -struct dlm_mhandle *dlm_allocate_mhandle (void ) +struct dlm_mhandle *dlm_allocate_mhandle (gfp_t allocation) { -return kmem_cache_alloc (mhandle_cache,GFP_NOFS ); +return kmem_cache_alloc (mhandle_cache, allocation); }void dlm_free_mhandle (struct dlm_mhandle *mhandle)
fs/dlm/memory.h +1 -1
@@ -20,7 +20,7 @@ struct dlm_lkb *dlm_allocate_lkb(struct dlm_ls *ls);void dlm_free_lkb (struct dlm_lkb *l);char *dlm_allocate_lvb (struct dlm_ls *ls);void dlm_free_lvb (char *l); -struct dlm_mhandle *dlm_allocate_mhandle (void ); +struct dlm_mhandle *dlm_allocate_mhandle (gfp_t allocation);void dlm_free_mhandle (struct dlm_mhandle *mhandle);struct writequeue_entry *dlm_allocate_writequeue (void );void dlm_free_writequeue (struct writequeue_entry *writequeue);
fs/dlm/midcomms.c +1 -1
@@ -1097,7 +1097,7 @@ struct dlm_mhandle *dlm_midcomms_get_mhandle(int nodeid, int len,/* this is a bug, however we going on and hope it will be resolved */ WARN_ON (test_bit (DLM_NODE_FLAG_STOP_TX , &node->flags)); - mh =dlm_allocate_mhandle (); + mh =dlm_allocate_mhandle (allocation);if (!mh) goto err;
Signature · 1 line
2.31.1
Alexander Aring
maintainerPATCH 13/18 fs: dlm: remove ls_remove_wait waitqueue
In reply to Alexander AringThis patch removes the ls_remove_wait waitqueue handling. The current handling tries to wait before a lookup is send out for a identically resource name which is going to be removed. Hereby the remove message should be send out before the new lookup message. The reason is that after a lookup request and response will actually use the specific remote rsb. A followed remove message would delete the rsb on the remote side but it's still being used.
To reach a similar behaviour we simple send the remove message out while the rsb lookup lock is held and the rsb is removed from the toss list. Other find_rsb() calls would never have the change to get a rsb back to live while a remove message will be send out (without holding the lock).
This behaviour requires a non-sleepable context which should be provided now and might be the reason why it was not implemented so in the first place.
--- fs/dlm/dlm_internal.h | 4 ---- fs/dlm/lock.c | 56 ++----------------------------------------- fs/dlm/lockspace.c | 3 --- 3 files changed, 2 insertions(+), 61 deletions(-)
fs/dlm/dlm_internal.h +0 -4
@@ -592,11 +592,7 @@ struct dlm_ls {int ls_new_rsb_count;struct list_head ls_new_rsb;/* new rsb structs */ -spinlock_t ls_remove_spin; -wait_queue_head_t ls_remove_wait; -char ls_remove_name[DLM_RESNAME_MAXLEN +1 ];char *ls_remove_names[DLM_REMOVE_NAMES_MAX ]; -int ls_remove_len;int ls_remove_lens[DLM_REMOVE_NAMES_MAX ];struct list_head ls_nodes;/* current nodes in ls */
fs/dlm/lock.c +2 -54
87 lines not on this page · raw message
fs/dlm/lockspace.c +0 -3
@@ -524,9 +524,6 @@ static int new_lockspace(const char *name, const char *cluster,spin_lock_init (&ls->ls_rsbtbl[i].lock); } -spin_lock_init (&ls->ls_remove_spin); -init_waitqueue_head (&ls->ls_remove_wait); -for (i =0 ; i <DLM_REMOVE_NAMES_MAX ; i++) { ls->ls_remove_names[i] =kzalloc (DLM_RESNAME_MAXLEN +1 ,GFP_KERNEL );
Signature · 1 line
2.31.1
Alexander Aring
maintainerPATCH 14/18 fs: dlm: relax sending to allow receiving
In reply to Alexander AringThis patch drops additionally the sock_mutex when there is a sending message burst. Since we have acknowledge handling we free sending buffers only when we receive an ack back, but if we are stuck in send_to_sock() looping because dlm sends a lot of messages and we never leave the loop the sending buffer fill up very quickly. We can't receive during this iteration because the sock_mutex is held. This patch will unlock the sock_mutex so it should be possible to receive messages when a burst of sending messages happens. This will allow to free up memory because acks which are already received can be processed.
--- fs/dlm/lowcomms.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-)
fs/dlm/lowcomms.c +10 -5
@@ -1418,7 +1418,10 @@ static void send_to_sock(struct connection *con)const int msg_flags =MSG_DONTWAIT |MSG_NOSIGNAL ;struct writequeue_entry *e;int len, offset, ret; -int count =0 ; +int count; + +again: + count =0 ;mutex_lock (&con->sock_mutex);if (con->sock ==NULL )
@@ -1453,14 +1456,16 @@ static void send_to_sock(struct connection *con) }else if (ret <0 ) goto out; +spin_lock (&con->writequeue_lock); +writequeue_entry_complete (e, ret); +/* Don't starve people filling buffers */ if (++count >=MAX_SEND_MSG_COUNT ) { +spin_unlock (&con->writequeue_lock); +mutex_unlock (&con->sock_mutex);cond_resched (); - count =0 ; + goto again; } - -spin_lock (&con->writequeue_lock); -writequeue_entry_complete (e, ret); }spin_unlock (&con->writequeue_lock);
Signature · 1 line
2.31.1
Alexander Aring
maintainerPATCH 15/18 fs: dlm: catch dlm_add_member() error
In reply to Alexander AringThis patch will catch a possible dlm_add_member() and delivers it to the dlm recovery handling.
--- fs/dlm/member.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
fs/dlm/member.c +4 -1
@@ -573,7 +573,10 @@ int dlm_recover_members(struct dlm_ls *ls, struct dlm_recover *rv, int *neg_out) node = &rv->nodes[i];if (dlm_is_member (ls, node->nodeid))continue ; -dlm_add_member (ls, node); + error =dlm_add_member (ls, node); +if (error) +return error; +log_rinfo (ls,"add member %d" , node->nodeid); }
Signature · 1 line
2.31.1
Alexander Aring
maintainerPATCH 16/18 fs: dlm: fix log of lowcomms vs midcomms
In reply to Alexander AringThis patch will fix a small issue when printing out that dlm_midcomms_start() failed to start and it was printing out that the dlm subcomponent lowcomms was failed but lowcomms is behind the midcomms layer.
--- fs/dlm/lockspace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
fs/dlm/lockspace.c +1 -1
@@ -391,7 +391,7 @@ static int threads_start(void)/* Thread for sending/receiving messages for all lockspace's */ error =dlm_midcomms_start ();if (error) { -log_print ("cannot start dlm lowcomms %d" , error); +log_print ("cannot start dlm midcomms %d" , error); goto scand_fail; }
Signature · 1 line
2.31.1
Alexander Aring
maintainerPATCH 17/18 fs: dlm: use WARN_ON_ONCE() instead of WARN_ON()
In reply to Alexander AringTo not get the console spammed about WARN_ON() of invalid states in the dlm midcomms hot path handling we switch to WARN_ON_ONCE() to get it only once that there might be an issue with the midcomms state handling.
--- fs/dlm/midcomms.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-)
fs/dlm/midcomms.c +9 -9
@@ -469,7 +469,7 @@ static void dlm_pas_fin_ack_rcv(struct midcomms_node *node)spin_unlock (&node->state_lock);log_print ("%s: unexpected state: %d\n" , __func__, node->state); -WARN_ON (1 ); +WARN_ON_ONCE (1 );return ; }spin_unlock (&node->state_lock);
@@ -540,7 +540,7 @@ static void dlm_midcomms_receive_buffer(union dlm_packet *p,spin_unlock (&node->state_lock);log_print ("%s: unexpected state: %d\n" , __func__, node->state); -WARN_ON (1 ); +WARN_ON_ONCE (1 );return ; }spin_unlock (&node->state_lock);
@@ -548,7 +548,7 @@ static void dlm_midcomms_receive_buffer(union dlm_packet *p,set_bit (DLM_NODE_FLAG_STOP_RX , &node->flags);break ;default : -WARN_ON (test_bit (DLM_NODE_FLAG_STOP_RX , &node->flags)); +WARN_ON_ONCE (test_bit (DLM_NODE_FLAG_STOP_RX , &node->flags));dlm_receive_buffer_3_2_trace (seq, p);dlm_receive_buffer (p, node->nodeid);set_bit (DLM_NODE_ULP_DELIVERED , &node->flags);
@@ -770,7 +770,7 @@ static void dlm_midcomms_receive_buffer_3_2(union dlm_packet *p, int nodeid) goto out; } -WARN_ON (test_bit (DLM_NODE_FLAG_STOP_RX , &node->flags)); +WARN_ON_ONCE (test_bit (DLM_NODE_FLAG_STOP_RX , &node->flags));dlm_receive_buffer (p, nodeid);break ;case DLM_OPTS :
@@ -1095,7 +1095,7 @@ struct dlm_mhandle *dlm_midcomms_get_mhandle(int nodeid, int len, }/* this is a bug, however we going on and hope it will be resolved */ -WARN_ON (test_bit (DLM_NODE_FLAG_STOP_TX , &node->flags)); +WARN_ON_ONCE (test_bit (DLM_NODE_FLAG_STOP_TX , &node->flags)); mh =dlm_allocate_mhandle (allocation);if (!mh)
@@ -1127,7 +1127,7 @@ struct dlm_mhandle *dlm_midcomms_get_mhandle(int nodeid, int len,break ;default :dlm_free_mhandle (mh); -WARN_ON (1 ); +WARN_ON_ONCE (1 ); goto err; }
@@ -1196,7 +1196,7 @@ void dlm_midcomms_commit_mhandle(struct dlm_mhandle *mh,break ;default :srcu_read_unlock (&nodes_srcu, mh->idx); -WARN_ON (1 ); +WARN_ON_ONCE (1 );break ; } }
@@ -1238,7 +1238,7 @@ static void dlm_act_fin_ack_rcv(struct midcomms_node *node)spin_unlock (&node->state_lock);log_print ("%s: unexpected state: %d\n" , __func__, node->state); -WARN_ON (1 ); +WARN_ON_ONCE (1 );return ; }spin_unlock (&node->state_lock);
@@ -1356,7 +1356,7 @@ static void midcomms_node_release(struct rcu_head *rcu) { struct midcomms_node *node = container_of(rcu, struct midcomms_node, rcu); -WARN_ON (atomic_read (&node->send_queue_cnt)); +WARN_ON_ONCE (atomic_read (&node->send_queue_cnt));kfree (node); }
Signature · 1 line
2.31.1
Alexander Aring
maintainerPATCH 18/18 fs: dlm: rework lowcomms handling
In reply to Alexander AringThis patch is rework of lowcomms handling, the main goal was here to handle recvmsg() and sendpage() to run parallel. Parallel in two sense:
1. per connection and 2. that recvmsg()/sendpage() doesn't block each other.
Currently recvmsg()/sendpage() cannot run parallel because two workqueues "dlm_recv" and "dlm_send" are ordered workqeues. That's said and we having two workqueues for receive and send, they block each if both workqueues schedule the same connection at the same time because a per connection mutex to prevent that the socket gets released by being used.
To make it more parallel we introduce one "dlm_io" workqueue which is not an ordered workqueue, the amount of workers are not limited. Due per connection flags SEND/RECV pending we schedule workers ordered per connection and per send and receive task. To get rid of the mutex blocking same workers to do socket handling we switched to a semaphore which handles socket operations as read lock and sock releases as write operations, to prevent sock_release() being called while the socket is being used.
There might be more optimization removing the semaphore and replacing it with rcu, however due other circumstances e.g. othercon behaviour it seems complicated to doing this change. I added comments to remove the othercon handling and moving to rcu handling. We need to do that to the next dlm major version upgrade because it is not backwards compatible with the current connect mechanism.
The processing of dlm messages need to be still handled by a ordered workqueue. An dlm_process ordered workqeue was introduced which gets filled by the receive worker. This is probably the next bottleneck of DLM but the application can't currently parse dlm messages parallel. A comment was introduced to lift the workqueue context of dlm processing in a non-sleepable softirq.
The lifetime of a connection struct is now different because the dlm_node_addr structure was merged into the connection struct. The shutdown handling for TCP was removed to handle half-open sockets. We have a application handling for dealing with half-opened connection which is required by dlm and cluster manager join/leave events. SCTP doesn't support half-closed sockets and it's already handled by the application to never run into such case, therefore we removed it.
To check if lowcomms is currently running and it cannot be configured at this time we introduced a function dlm_lowcomms_is_running() and remove the dlm_allow_conn variable. The synchronization of stopping all listen socket workers goes now of it's data ready socket callback.
--- fs/dlm/config.c | 4 +- fs/dlm/lockspace.c | 5 +- fs/dlm/lowcomms.c | 1453 ++++++++++++++++++++------------------------ fs/dlm/lowcomms.h | 6 +- fs/dlm/main.c | 7 +- fs/dlm/midcomms.c | 68 ++- fs/dlm/midcomms.h | 4 + 7 files changed, 730 insertions(+), 817 deletions(-)
fs/dlm/config.c +2 -2
@@ -183,7 +183,7 @@ static int dlm_check_protocol_and_dlm_running(unsigned int x)return -EINVAL ; } -if (dlm_allow_conn) +if (dlm_lowcomms_is_running ())return -EBUSY ;return 0 ;
@@ -194,7 +194,7 @@ static int dlm_check_zero_and_dlm_running(unsigned int x)if (!x)return -EINVAL ; -if (dlm_allow_conn) +if (dlm_lowcomms_is_running ())return -EBUSY ;return 0 ;
fs/dlm/lockspace.c +2 -3
@@ -17,7 +17,6 @@#include "recoverd.h" #include "dir.h" #include "midcomms.h" -#include "lowcomms.h" #include "config.h" #include "memory.h" #include "lock.h"
@@ -723,7 +722,7 @@ static int __dlm_new_lockspace(const char *name, const char *cluster,if (!ls_count) {dlm_scand_stop ();dlm_midcomms_shutdown (); -dlm_lowcomms_stop (); +dlm_midcomms_stop (); } out:mutex_unlock (&ls_lock);
@@ -926,7 +925,7 @@ int dlm_release_lockspace(void *lockspace, int force)if (!error) ls_count--;if (!ls_count) -dlm_lowcomms_stop (); +dlm_midcomms_stop ();mutex_unlock (&ls_lock);return error;
fs/dlm/lowcomms.c +660 -793
2,088 lines not on this page · raw message
fs/dlm/lowcomms.h +4 -2
@@ -29,12 +29,14 @@ static inline int nodeid_hash(int nodeid)return nodeid & (CONN_HASH_SIZE -1 ); } -/* switch to check if dlm is running */ -extern int dlm_allow_conn; +/* check if dlm is running */ +bool dlm_lowcomms_is_running (void );int dlm_lowcomms_start (void );void dlm_lowcomms_shutdown (void ); +void dlm_lowcomms_shutdown_node (int nodeid,bool force);void dlm_lowcomms_stop (void ); +void dlm_lowcomms_init (void );void dlm_lowcomms_exit (void );int dlm_lowcomms_close (int nodeid); struct dlm_msg *dlm_lowcomms_new_msg(int nodeid, int len, gfp_t allocation,
fs/dlm/main.c +5 -2
@@ -17,7 +17,7 @@#include "user.h" #include "memory.h" #include "config.h" -#include "lowcomms.h" +#include "midcomms.h" #define CREATE_TRACE_POINTS #include <trace/events/dlm.h>
@@ -30,6 +30,8 @@ static int __init init_dlm(void)if (error) goto out; +dlm_midcomms_init (); + error =dlm_lockspace_init ();if (error) goto out_mem;
@@ -66,6 +68,7 @@ static int __init init_dlm(void) out_lockspace:dlm_lockspace_exit (); out_mem: +dlm_midcomms_exit ();dlm_memory_exit (); out:return error;
@@ -79,7 +82,7 @@ static void __exit exit_dlm(void)dlm_config_exit ();dlm_memory_exit ();dlm_lockspace_exit (); -dlm_lowcomms_exit (); +dlm_midcomms_exit ();dlm_unregister_debugfs (); }
fs/dlm/midcomms.c +53 -15
143 lines not on this page · raw message
fs/dlm/midcomms.h +4 -0
@@ -14,6 +14,7 @@struct midcomms_node ; +int dlm_validate_incoming_buffer (int nodeid,unsigned char *buf,int len);int dlm_process_incoming_buffer (int nodeid,unsigned char *buf,int buflen);struct dlm_mhandle *dlm_midcomms_get_mhandle (int nodeid,int len,gfp_t allocation,char **ppc);
@@ -21,6 +22,9 @@ void dlm_midcomms_commit_mhandle(struct dlm_mhandle *mh, const void *name, int namelen);int dlm_midcomms_close (int nodeid);int dlm_midcomms_start (void ); +void dlm_midcomms_stop (void ); +void dlm_midcomms_init (void ); +void dlm_midcomms_exit (void );void dlm_midcomms_shutdown (void );void dlm_midcomms_add_member (int nodeid);void dlm_midcomms_remove_member (int nodeid);
Signature · 1 line
2.31.1
Alexander Aring
maintainerPATCH 18/18 fs: dlm: rework lowcomms handling
Hi,
Quoted 69 lines · Alexander Aring
This patch is rework of lowcomms handling, the main goal was here to handle
recvmsg()andsendpage()to run parallel. Parallel in two sense:
1. per connection and 2. that recvmsg()/sendpage() doesn't block each other.Currently recvmsg()/sendpage() cannot run parallel because two workqueues "
dlm_recv" and "dlm_send" are ordered workqeues. That's said and we having two workqueues for receive and send, they block each if both workqueues schedule the same connection at the same time because a per connection mutex to prevent that the socket gets released by being used.To make it more parallel we introduce one "
dlm_io" workqueue which is not an ordered workqueue, the amount of workers are not limited. Due per connection flags SEND/RECV pending we schedule workers ordered per connection and per send and receive task. To get rid of the mutex blocking same workers to do socket handling we switched to a semaphore which handles socket operations as read lock and sock releases as write operations, to preventsock_release()being called while the socket is being used.There might be more optimization removing the semaphore and replacing it with rcu, however due other circumstances e.g. othercon behaviour it seems complicated to doing this change. I added comments to remove the othercon handling and moving to rcu handling. We need to do that to the next dlm major version upgrade because it is not backwards compatible with the current connect mechanism.
The processing of dlm messages need to be still handled by a ordered workqueue. An
dlm_processordered workqeue was introduced which gets filled by the receive worker. This is probably the next bottleneck of DLM but the application can't currently parse dlm messages parallel. A comment was introduced to lift the workqueue context of dlm processing in a non-sleepable softirq.The lifetime of a connection struct is now different because the
dlm_node_addrstructure was merged into the connection struct. The shutdown handling for TCP was removed to handle half-open sockets. We have a application handling for dealing with half-opened connection which is required by dlm and cluster manager join/leave events. SCTP doesn't support half-closed sockets and it's already handled by the application to never run into such case, therefore we removed it.To check if lowcomms is currently running and it cannot be configured at this time we introduced a function
dlm_lowcomms_is_running()and remove thedlm_allow_connvariable. The synchronization of stopping all listen socket workers goes now of it's data ready socket callback.Signed-off-by Alexander Aring <[email protected]>--- fs/dlm/config.c | 4 +- fs/dlm/lockspace.c | 5 +- fs/dlm/lowcomms.c | 1453 ++++++++++++++++++++------------------------ fs/dlm/lowcomms.h | 6 +- fs/dlm/main.c | 7 +- fs/dlm/midcomms.c | 68 ++- fs/dlm/midcomms.h | 4 + 7 files changed, 730 insertions(+), 817 deletions(-)fs/dlm/config.c +0
-1@@ -183,7 +183,7 @@ static int dlm_check_protocol_and_dlm_running(unsigned int x)return -EINVAL ; } -if (dlm_allow_conn)
...
Quoted 7 lines
fs/dlm/config.c +7
-0+ +switch (ret) { +case DLM_IO_END : +dlm_midcomms_receive_done (con->nodeid); +break ; +case DLM_IO_EOF : +close_connection (con, false);
fs/dlm/config.c
Quoted 10 lines
fs/dlm/config.c +10
-0+break ; +case DLM_IO_RESCHED : +cond_resched (); +queue_work (io_workqueue, &con->rwork); +break ; +default : +if (ret <0 ) { +log_print ("node %d recv error %d" , con->nodeid, ret); +close_connection (con, false); +dlm_midcomms_unack_msg_resend (con->nodeid);
I will change it to only call close_connection() on the send side and put only a dlm_midcomms_unack_msg_resend() in here. It is difficult to synchronize it with the sending worker. As we retransmit not acked dlm messages we will hopefully get an error on the sending try and it will close the socket.
Because the othercon and if we have the othercon it's only receiving we will call close_connection() if it's a "othercon".
I currently try to run a test with tcpkill to see how the close/reconnect behaviour acts.
I know the patch is really really big. I will try to split them somehow into smaller ones as possible.
- Alex