fs: dlm: debug and tracepoints
Subsystems (2)
Alexander Aring
maintainercover fs: dlm: debug and tracepoints
Hi,
again a resend of the tracepoints patches and fixed that the lkb reference is hold during tracepoint call. We need to do that because we access the structure during tracing. There is some code duplication to convert the return error for dlm_lock_end trace now. I didn't wanted to change the existing code and introduce more overhead if tracing is not enabled.
- Alex
Alexander Aring (3):
fs: dlm: debug improvements print nodeid
fs: dlm: initial support for tracepoints
fs: dlm: trace socket handling
fs/dlm/ast.c | 4 + fs/dlm/lock.c | 10 ++ fs/dlm/lowcomms.c | 4 + fs/dlm/main.c | 3 + fs/dlm/midcomms.c | 4 +- include/trace/events/dlm.h | 260 +++++++++++++++++++++++++++++++++++++ 6 files changed, 283 insertions(+), 2 deletions(-) create mode 100644 include/trace/events/dlm.h
Signature · 1 line
2.27.0
Alexander Aring
maintainerPATCH 1/3 fs: dlm: debug improvements print nodeid
This patch improves the debug output for midcomms layer by also printing out the nodeid where users counter belongs to.
--- fs/dlm/midcomms.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
fs/dlm/midcomms.c +2 -2
@@ -1231,7 +1231,7 @@ void dlm_midcomms_add_member(int nodeid) } node->users++; -pr_debug ("users inc count %d\n" , node->users); +pr_debug ("node %d users inc count %d\n" , nodeid, node->users);spin_unlock (&node->state_lock);srcu_read_unlock (&nodes_srcu, idx);
@@ -1254,7 +1254,7 @@ void dlm_midcomms_remove_member(int nodeid)spin_lock (&node->state_lock); node->users--; -pr_debug ("users dec count %d\n" , node->users); +pr_debug ("node %d users dec count %d\n" , nodeid, node->users);/* hitting users count to zero means the * other side is running dlm_midcomms_stop()
Signature · 1 line
2.27.0
Alexander Aring
maintainerPATCH 2/3 fs: dlm: initial support for tracepoints
In reply to Alexander AringThis patch adds initial support for dlm tracepoints. It will introduce tracepoints to dlm main functionality dlm_lock()/dlm_unlock() and their complete ast() callback or blocking bast() callback.
The lock/unlock functionality has a start and end tracepoint, this is because there exists a race in case if would have a tracepoint at the end position only the complete/blocking callbacks could occur before. To work with eBPF tracing and using their lookup hash functionality there could be problems that an entry was not inserted yet. However use the start functionality for hash insert and check again in end functionality if there was an dlm internal error so there is no ast callback. In further it might also that locks with local masters will occur those callbacks immediately so we must have such functionality.
I did not make everything accessible yet, although it seems eBPF can be used to access a lot of internal datastructures if it's aware of the struct definitions of the running kernel instance. We still can change it, if you do eBPF experiments e.g. time measurements between lock and callback functionality you can simple use the local lkb_id field as hash value in combination with the lockspace id if you have multiple lockspaces. Otherwise you can simple use trace-cmd for some functionality, e.g. trace-cmd record -e dlm and trace-cmd report afterwards.
--- fs/dlm/ast.c | 4 + fs/dlm/lock.c | 10 ++ fs/dlm/main.c | 3 + include/trace/events/dlm.h | 220 +++++++++++++++++++++++++++++++++++++ 4 files changed, 237 insertions(+) create mode 100644 include/trace/events/dlm.h
fs/dlm/ast.c +4 -0
@@ -9,6 +9,8 @@******************************************************************************* ******************************************************************************/ +#include <trace/events/dlm.h> +#include "dlm_internal.h" #include "lock.h" #include "user.h"
@@ -254,10 +256,12 @@ void dlm_callback_work(struct work_struct *work)continue ; }else if (callbacks[i].flags &DLM_CB_BAST ) {bastfn (lkb->lkb_astparam, callbacks[i].mode); +trace_dlm_bast (ls, lkb, callbacks[i].mode); }else if (callbacks[i].flags &DLM_CB_CAST ) { lkb->lkb_lksb->sb_status = callbacks[i].sb_status; lkb->lkb_lksb->sb_flags = callbacks[i].sb_flags;castfn (lkb->lkb_astparam); +trace_dlm_ast (ls, lkb, lkb->lkb_lksb); } }
fs/dlm/lock.c +10 -0
@@ -53,6 +53,8 @@R: do_xxxx() L: receive_xxxx_reply() <- R: send_xxxx_reply() */ +#include <trace/events/dlm.h> +#include <linux/types.h> #include <linux/rbtree.h> #include <linux/slab.h>
@@ -3437,6 +3439,8 @@ int dlm_lock(dlm_lockspace_t *lockspace,if (error) goto out; +trace_dlm_lock_start (ls, lkb, mode, flags); + error =set_lock_args (mode, lksb, flags, namelen,0 , ast, astarg, bast, &args);if (error)
@@ -3450,6 +3454,8 @@ int dlm_lock(dlm_lockspace_t *lockspace,if (error == -EINPROGRESS ) error =0 ; out_put: +trace_dlm_lock_end (ls, lkb, mode, flags, error); +if (convert || error)__put_lkb (ls, lkb);if (error == -EAGAIN || error == -EDEADLK )
@@ -3481,6 +3487,8 @@ int dlm_unlock(dlm_lockspace_t *lockspace,if (error) goto out; +trace_dlm_unlock_start (ls, lkb, flags); + error =set_unlock_args (flags, astarg, &args);if (error) goto out_put;
@@ -3495,6 +3503,8 @@ int dlm_unlock(dlm_lockspace_t *lockspace,if (error == -EBUSY && (flags & (DLM_LKF_CANCEL |DLM_LKF_FORCEUNLOCK ))) error =0 ; out_put: +trace_dlm_unlock_end (ls, lkb, flags, error); +dlm_put_lkb (lkb); out:dlm_unlock_recovery (ls);
fs/dlm/main.c +3 -0
@@ -19,6 +19,9 @@#include "config.h" #include "lowcomms.h" +#define CREATE_TRACE_POINTS +#include <trace/events/dlm.h> +static int __init init_dlm (void ) {int error;
include/trace/events/dlm.h new file +220 -0
220 lines not on this page · raw message
Signature · 1 line
2.27.0
Alexander Aring
maintainerPATCH 3/3 fs: dlm: trace socket handling
In reply to Alexander AringThis patch adds tracepoints for dlm socket receive and send functionality. We can use it to track how much data was send or received to or from a specific nodeid.
--- fs/dlm/lowcomms.c | 4 ++++ include/trace/events/dlm.h | 40 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+)
fs/dlm/lowcomms.c +4 -0
@@ -53,6 +53,8 @@#include <net/sctp/sctp.h> #include <net/ipv6.h> +#include <trace/events/dlm.h> +#include "dlm_internal.h" #include "lowcomms.h" #include "midcomms.h"
@@ -925,6 +927,7 @@ static int receive_from_sock(struct connection *con) msg.msg_flags =MSG_DONTWAIT |MSG_NOSIGNAL ; ret =kernel_recvmsg (con->sock, &msg, &iov,1 , iov.iov_len, msg.msg_flags); +trace_dlm_recv (con->nodeid, ret);if (ret == -EAGAIN )break ;else if (ret <=0 )
@@ -1411,6 +1414,7 @@ static void send_to_sock(struct connection *con) ret =kernel_sendpage (con->sock, e->page, offset, len, msg_flags); +trace_dlm_send (con->nodeid, ret);if (ret == -EAGAIN || ret ==0 ) { if (ret == -EAGAIN && test_bit(SOCKWQ_ASYNC_NOSPACE, &con->sock->flags) &&
include/trace/events/dlm.h +40 -0
@@ -214,6 +214,46 @@ TRACE_EVENT(dlm_unlock_end, ); +TRACE_EVENT (dlm_send, + + TP_PROTO(int nodeid, int ret), + +TP_ARGS (nodeid, ret), + +TP_STRUCT__entry ( + __field(int, nodeid) +__field (int, ret) + ), + +TP_fast_assign ( + __entry->nodeid = nodeid; + __entry->ret = ret; + ), + +TP_printk ("nodeid=%d ret=%d" , __entry->nodeid, __entry->ret) + +); + +TRACE_EVENT (dlm_recv, + + TP_PROTO(int nodeid, int ret), + +TP_ARGS (nodeid, ret), + +TP_STRUCT__entry ( + __field(int, nodeid) +__field (int, ret) + ), + +TP_fast_assign ( + __entry->nodeid = nodeid; + __entry->ret = ret; + ), + +TP_printk ("nodeid=%d ret=%d" , __entry->nodeid, __entry->ret) + +); +#endif /* if !defined(_TRACE_DLM_H) || defined(TRACE_HEADER_MULTI_READ) */ /* This part must be outside protection */
Signature · 1 line
2.27.0