O2CB global heartbeat - hopefully final drop!
All,
This is hopefully the final drop of the patches for adding global heartbeat to the o2cb stack.
The diff from the previous set is here:
http://oss.oracle.com/~smushran/global-hb-diff-2010-10-07
Implemented most of the suggestions provided by Joel and Wengang. The most important one was to activate the feature only at the end,
Also, got mostly a clean run with checkpatch.pl.
Sunil
PATCH 1/22 ocfs2/cluster: Add heartbeat mode configfs parameter
Add heartbeat mode parameter to the configfs tree. This will be used to set/show the heartbeat mode. The user is free to toggle the mode between local and global as long as there is no active heartbeat region.
--- fs/ocfs2/cluster/heartbeat.c | 72 ++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 72 insertions(+), 0 deletions(-)
fs/ocfs2/cluster/heartbeat.c +72 -0
@@ -77,7 +77,19 @@ static struct o2hb_callback *hbcall_from_type(enum o2hb_callback_type type);#define O2HB_DEFAULT_BLOCK_BITS 9 +enum o2hb_heartbeat_modes { +O2HB_HEARTBEAT_LOCAL =0 , +O2HB_HEARTBEAT_GLOBAL , +O2HB_HEARTBEAT_NUM_MODES , +}; + +char *o2hb_heartbeat_mode_desc[O2HB_HEARTBEAT_NUM_MODES ] = { +"local" ,/* O2HB_HEARTBEAT_LOCAL */ +"global" ,/* O2HB_HEARTBEAT_GLOBAL */ +}; +unsigned int o2hb_dead_threshold =O2HB_DEFAULT_DEAD_THRESHOLD ; +unsigned int o2hb_heartbeat_mode =O2HB_HEARTBEAT_LOCAL ;/* Only sets a new threshold if there are no active regions. *
@@ -94,6 +106,22 @@ static void o2hb_dead_threshold_set(unsigned int threshold) } } +static int o2hb_global_hearbeat_mode_set (unsigned int hb_mode) +{ +int ret =-1 ; + +if (hb_mode <O2HB_HEARTBEAT_NUM_MODES ) { +spin_lock (&o2hb_live_lock); +if (list_empty (&o2hb_all_regions)) { + o2hb_heartbeat_mode = hb_mode; + ret =0 ; + } +spin_unlock (&o2hb_live_lock); + } + +return ret; +} +struct o2hb_node_event {struct list_head hn_item;enum o2hb_callback_type hn_event_type;
@@ -1688,6 +1716,41 @@ static ssize_t o2hb_heartbeat_group_threshold_store(struct o2hb_heartbeat_groupreturn count; } +static +ssize_t o2hb_heartbeat_group_mode_show (struct o2hb_heartbeat_group *group, +char *page) +{ +return sprintf (page,"%s\n" , + o2hb_heartbeat_mode_desc[o2hb_heartbeat_mode]); +} + +static +ssize_t o2hb_heartbeat_group_mode_store (struct o2hb_heartbeat_group *group, +const char *page,size_t count) +{ +unsigned int i; +int ret; +size_t len; + + len = (page[count -1 ] =='\n' ) ? count -1 : count; +if (!len) +return -EINVAL ; + +for (i =0 ; i <O2HB_HEARTBEAT_NUM_MODES ; ++i) { +if (strnicmp (page, o2hb_heartbeat_mode_desc[i], len)) +continue ; + + ret =o2hb_global_hearbeat_mode_set (i); +if (!ret) +printk (KERN_NOTICE "ocfs2: Heartbeat mode set to %s\n" , + o2hb_heartbeat_mode_desc[i]); +return count; + } + +return -EINVAL ; + +} +static struct o2hb_heartbeat_group_attribute o2hb_heartbeat_group_attr_threshold = { .attr = { .ca_owner =THIS_MODULE , .ca_name ="dead_threshold" ,
@@ -1696,8 +1759,17 @@ static struct o2hb_heartbeat_group_attribute o2hb_heartbeat_group_attr_threshold .store = o2hb_heartbeat_group_threshold_store, }; +static struct o2hb_heartbeat_group_attribute o2hb_heartbeat_group_attr_mode = { + .attr = { .ca_owner =THIS_MODULE , + .ca_name ="mode" , + .ca_mode =S_IRUGO |S_IWUSR }, + .show = o2hb_heartbeat_group_mode_show, + .store = o2hb_heartbeat_group_mode_store, +}; +static struct configfs_attribute *o2hb_heartbeat_group_attrs[] = { &o2hb_heartbeat_group_attr_threshold.attr, + &o2hb_heartbeat_group_attr_mode.attr,NULL , };
Signature · 1 line
1.7.0.4
PATCH 2/22 ocfs2: Add an incompat feature flag OCFS2_FEATURE_INCOMPAT_CLUSTERINFO
In reply to Sunil MushranOCFS2_FEATURE_INCOMPAT_CLUSTERINFO allows us to use sb->s_cluster_info for both userspace and o2cb cluster stacks. It also allows us to extend cluster info to include stack flags.
This patch also adds stackflags to sb->s_clusterinfo. It also introduces a clusterinfo flag OCFS2_CLUSTER_O2CB_GLOBAL_HEARTBEAT to denote the enabled global heartbeat mode.
This incompat flag can be set/cleared using tunefs.ocfs2 --fs-features. The clusterinfo flag is set/cleared using tunefs.ocfs2 --update-cluster-stack.
--- fs/ocfs2/ocfs2.h | 31 +++++++++++++++++++++++++++++-- fs/ocfs2/ocfs2_fs.h | 40 ++++++++++++++++++++++++++++++++++------ fs/ocfs2/super.c | 4 +++- 3 files changed, 66 insertions(+), 9 deletions(-)
fs/ocfs2/ocfs2.h +29 -2
@@ -368,6 +368,8 @@ struct ocfs2_superstruct ocfs2_alloc_stats alloc_stats;char dev_str[20 ];/* "major,minor" of the device */ +u8 osb_stackflags; +char osb_cluster_stack[OCFS2_STACK_LABEL_LEN +1 ];struct ocfs2_cluster_connection *cconn;struct ocfs2_lock_res osb_super_lockres;
@@ -601,10 +603,35 @@ static inline int ocfs2_is_soft_readonly(struct ocfs2_super *osb)return ret; } -static inline int ocfs2_userspace_stack (struct ocfs2_super *osb) +static inline int ocfs2_clusterinfo_valid (struct ocfs2_super *osb) {return (osb->s_feature_incompat & -OCFS2_FEATURE_INCOMPAT_USERSPACE_STACK ); + (OCFS2_FEATURE_INCOMPAT_USERSPACE_STACK | +OCFS2_FEATURE_INCOMPAT_CLUSTERINFO )); +} + +static inline int ocfs2_userspace_stack (struct ocfs2_super *osb) +{ +if (ocfs2_clusterinfo_valid (osb) && +memcmp (osb->osb_cluster_stack,OCFS2_CLASSIC_CLUSTER_STACK , +OCFS2_STACK_LABEL_LEN )) +return 1 ; +return 0 ; +} + +static inline int ocfs2_o2cb_stack (struct ocfs2_super *osb) +{ +if (ocfs2_clusterinfo_valid (osb) && + !memcmp (osb->osb_cluster_stack,OCFS2_CLASSIC_CLUSTER_STACK , +OCFS2_STACK_LABEL_LEN )) +return 1 ; +return 0 ; +} + +static inline int ocfs2_cluster_o2cb_global_heartbeat (struct ocfs2_super *osb) +{ +return ocfs2_o2cb_stack (osb) && + (osb->osb_stackflags &OCFS2_CLUSTER_O2CB_GLOBAL_HEARTBEAT ); }static inline int ocfs2_mount_local (struct ocfs2_super *osb)
fs/ocfs2/ocfs2_fs.h +34 -6
@@ -101,7 +101,8 @@ | OCFS2_FEATURE_INCOMPAT_META_ECC \ | OCFS2_FEATURE_INCOMPAT_INDEXED_DIRS \ | OCFS2_FEATURE_INCOMPAT_REFCOUNT_TREE \ - | OCFS2_FEATURE_INCOMPAT_DISCONTIG_BG) + | OCFS2_FEATURE_INCOMPAT_DISCONTIG_BG \ + | OCFS2_FEATURE_INCOMPAT_CLUSTERINFO)#define OCFS2_FEATURE_RO_COMPAT_SUPP (OCFS2_FEATURE_RO_COMPAT_UNWRITTEN \ | OCFS2_FEATURE_RO_COMPAT_USRQUOTA \ | OCFS2_FEATURE_RO_COMPAT_GRPQUOTA)
@@ -170,6 +171,13 @@#define OCFS2_FEATURE_INCOMPAT_DISCONTIG_BG 0x2000/* +* Incompat bit to indicate useable clusterinfo with stackflags for all +* cluster stacks (userspace adnd o2cb). If this bit is set, +* INCOMPAT_USERSPACE_STACK becomes superfluous and thus should not be set. +*/ +#define OCFS2_FEATURE_INCOMPAT_CLUSTERINFO 0x4000 + +/* * backup superblock flag is used to indicate that this volume * has backup superblocks. */
@@ -292,10 +300,13 @@#define OCFS2_VOL_UUID_LEN 16#define OCFS2_MAX_VOL_LABEL_LEN 64 -/* The alternate, userspace stack fields */ +/* The cluster stack fields */ #define OCFS2_STACK_LABEL_LEN 4#define OCFS2_CLUSTER_NAME_LEN 16 +/* Classic (historically speaking) cluster stack */ +#define OCFS2_CLASSIC_CLUSTER_STACK "o2cb" +/* Journal limits (in bytes) */ #define OCFS2_MIN_JOURNAL_SIZE (4 * 1024 * 1024)
@@ -305,6 +316,11 @@*/ #define OCFS2_MIN_XATTR_INLINE_SIZE 256 +/* +* Cluster info flags (ocfs2_cluster_info.ci_stackflags) +*/ +#define OCFS2_CLUSTER_O2CB_GLOBAL_HEARTBEAT (0x01) +struct ocfs2_system_inode_info {char *si_name;int si_iflags;
@@ -566,9 +582,21 @@ struct ocfs2_slot_map_extended {*/ }; +/* +* ci_stackflags is only valid if the incompat bit +* OCFS2_FEATURE_INCOMPAT_CLUSTERINFO is set. +*/ struct ocfs2_cluster_info {/*00*/ __u8 ci_stack[OCFS2_STACK_LABEL_LEN ]; -__le32 ci_reserved; +union { +__le32 ci_reserved; +struct { +__u8 ci_reserved1; +__u8 ci_reserved2; +__u8 ci_reserved3; +__u8 ci_stackflags; + }; + };/*08*/ __u8 ci_cluster[OCFS2_CLUSTER_NAME_LEN ];/*18*/ };
@@ -605,9 +633,9 @@ struct ocfs2_super_block {* group header */ /*50*/ __u8 s_label[OCFS2_MAX_VOL_LABEL_LEN ];/* Label for mounting, etc. */ /*90*/ __u8 s_uuid[OCFS2_VOL_UUID_LEN ];/* 128-bit uuid */ -/*A0*/ struct ocfs2_cluster_info s_cluster_info;/* Selected userspace -stack. Only valid -with INCOMPAT flag. */ +/*A0*/ struct ocfs2_cluster_info s_cluster_info;/* Only valid if either +userspace or clusterinfo +INCOMPAT flag set. */ /*B8*/ __le16 s_xattr_inline_size;/* extended attribute inline size for this fs*/ __le16 s_reserved0;
fs/ocfs2/super.c +3 -1
@@ -2149,7 +2149,9 @@ static int ocfs2_initialize_super(struct super_block *sb, goto bail; } -if (ocfs2_userspace_stack (osb)) { +if (ocfs2_clusterinfo_valid (osb)) { + osb->osb_stackflags = +OCFS2_RAW_SB (di)->s_cluster_info.ci_stackflags;memcpy (osb->osb_cluster_stack,OCFS2_RAW_SB (di)->s_cluster_info.ci_stack,OCFS2_STACK_LABEL_LEN );
Signature · 1 line
1.7.0.4
PATCH 2/22 ocfs2: Add an incompat feature flag OCFS2_FEATURE_INCOMPAT_CLUSTERINFO
Quoted 132 lines · Sunil Mushran
OCFS2_FEATURE_INCOMPAT_CLUSTERINFOallows us to usesb->s_cluster_infofor both userspace and o2cb cluster stacks. It also allows us to extend cluster info to include stack flags.This patch also adds stackflags to
sb->s_clusterinfo. It also introduces a clusterinfo flagOCFS2_CLUSTER_O2CB_GLOBAL_HEARTBEATto denote the enabled global heartbeat mode.This incompat flag can be set/cleared using tunefs.ocfs2 --fs-features. The clusterinfo flag is set/cleared using tunefs.ocfs2 --update-cluster-stack.
Signed-off-by Sunil Mushran <[email protected]>--- fs/ocfs2/ocfs2.h | 31 +++++++++++++++++++++++++++++-- fs/ocfs2/ocfs2_fs.h | 40 ++++++++++++++++++++++++++++++++++------ fs/ocfs2/super.c | 4 +++- 3 files changed, 66 insertions(+), 9 deletions(-)fs/ocfs2/ocfs2.h +29
-2@@ -368,6 +368,8 @@ struct ocfs2_superstruct ocfs2_alloc_stats alloc_stats;char dev_str[20 ];/* "major,minor" of the device */ +u8 osb_stackflags; +char osb_cluster_stack[OCFS2_STACK_LABEL_LEN +1 ];struct ocfs2_cluster_connection *cconn;struct ocfs2_lock_res osb_super_lockres;@@ -601,10 +603,35 @@ static inline int ocfs2_is_soft_readonly(struct ocfs2_super *osb)return ret; } -static inline int ocfs2_userspace_stack (struct ocfs2_super *osb) +static inline int ocfs2_clusterinfo_valid (struct ocfs2_super *osb) {return (osb->s_feature_incompat & -OCFS2_FEATURE_INCOMPAT_USERSPACE_STACK ); + (OCFS2_FEATURE_INCOMPAT_USERSPACE_STACK | +OCFS2_FEATURE_INCOMPAT_CLUSTERINFO )); +} + +static inline int ocfs2_userspace_stack (struct ocfs2_super *osb) +{ +if (ocfs2_clusterinfo_valid (osb) && +memcmp (osb->osb_cluster_stack,OCFS2_CLASSIC_CLUSTER_STACK , +OCFS2_STACK_LABEL_LEN )) +return 1 ; +return 0 ; +} + +static inline int ocfs2_o2cb_stack (struct ocfs2_super *osb) +{ +if (ocfs2_clusterinfo_valid (osb) && + !memcmp (osb->osb_cluster_stack,OCFS2_CLASSIC_CLUSTER_STACK , +OCFS2_STACK_LABEL_LEN )) +return 1 ; +return 0 ; +} + +static inline int ocfs2_cluster_o2cb_global_heartbeat (struct ocfs2_super *osb) +{ +return ocfs2_o2cb_stack (osb) && + (osb->osb_stackflags &OCFS2_CLUSTER_O2CB_GLOBAL_HEARTBEAT ); }static inline int ocfs2_mount_local (struct ocfs2_super *osb)fs/ocfs2/ocfs2_fs.h +22
-2@@ -101,7 +101,8 @@ | OCFS2_FEATURE_INCOMPAT_META_ECC \ | OCFS2_FEATURE_INCOMPAT_INDEXED_DIRS \ | OCFS2_FEATURE_INCOMPAT_REFCOUNT_TREE \ - | OCFS2_FEATURE_INCOMPAT_DISCONTIG_BG) + | OCFS2_FEATURE_INCOMPAT_DISCONTIG_BG \ + | OCFS2_FEATURE_INCOMPAT_CLUSTERINFO)#define OCFS2_FEATURE_RO_COMPAT_SUPP (OCFS2_FEATURE_RO_COMPAT_UNWRITTEN \ | OCFS2_FEATURE_RO_COMPAT_USRQUOTA \ | OCFS2_FEATURE_RO_COMPAT_GRPQUOTA)@@ -170,6 +171,13 @@#define OCFS2_FEATURE_INCOMPAT_DISCONTIG_BG 0x2000/* +* Incompat bit to indicate useable clusterinfo with stackflags for all +* cluster stacks (userspace adnd o2cb). If this bit is set, +* INCOMPAT_USERSPACE_STACK becomes superfluous and thus should not be set. +*/ +#define OCFS2_FEATURE_INCOMPAT_CLUSTERINFO 0x4000 + +/* * backup superblock flag is used to indicate that this volume * has backup superblocks. */ @@ -292,10 +300,13 @@#define OCFS2_VOL_UUID_LEN 16#define OCFS2_MAX_VOL_LABEL_LEN 64 -/* The alternate, userspace stack fields */ +/* The cluster stack fields */ #define OCFS2_STACK_LABEL_LEN 4#define OCFS2_CLUSTER_NAME_LEN 16 +/* Classic (historically speaking) cluster stack */ +#define OCFS2_CLASSIC_CLUSTER_STACK "o2cb" +/* Journal limits (in bytes) */ #define OCFS2_MIN_JOURNAL_SIZE (4 * 1024 * 1024)@@ -305,6 +316,11 @@*/ #define OCFS2_MIN_XATTR_INLINE_SIZE 256 +/* +* Cluster info flags (ocfs2_cluster_info.ci_stackflags) +*/ +#define OCFS2_CLUSTER_O2CB_GLOBAL_HEARTBEAT (0x01) +struct ocfs2_system_inode_info {char *si_name;int si_iflags;@@ -566,9 +582,21 @@ struct ocfs2_slot_map_extended {*/ }; +/* +* ci_stackflags is only valid if the incompat bit +* OCFS2_FEATURE_INCOMPAT_CLUSTERINFO is set. +*/
fs/ocfs2/ocfs2_fs.h struct ocfs2_slot_map_extended { · line 597
Quoted 10 lines
fs/ocfs2/ocfs2_fs.h +7
-1struct ocfs2_cluster_info { /*00*/ __u8 ci_stack[OCFS2_STACK_LABEL_LEN]; -__le32 ci_reserved; + union { + __le32 ci_reserved; + struct { + __u8 ci_reserved1; + __u8 ci_reserved2; + __u8 ci_reserved3; + __u8 ci_stackflags;
Just curious, why stick ci_stackflags at the bottom of this structure?
Wouldn't putting it at the top allow us to grow the field in the future...
Otherwise the patch looks good to me.
--Mark
--
Mark Fasheh
PATCH 2/22 ocfs2: Add an incompat feature flag OCFS2_FEATURE_INCOMPAT_CLUSTERINFO
Quoted 12 lines · Mark Fasheh
fs/ocfs2/ocfs2_fs.h struct ocfs2_slot_map_extended { · line 597
Quoted 10 lines
/*00*/ __u8 ci_stack[OCFS2_STACK_LABEL_LEN];fs/ocfs2/ocfs2_fs.h +7
-1-__le32 ci_reserved; +union { +__le32 ci_reserved; +struct { +__u8 ci_reserved1; +__u8 ci_reserved2; +__u8 ci_reserved3; +__u8 ci_stackflags;Just curious, why stick
ci_stackflagsat the bottom of this structure?
Quoted 4 lines
Wouldn't putting it at the top allow us to grow the field in the future...
Otherwise the patch looks good to me.
Grow stackflags? Hmm... did not think of that.
I don't have a good reason for putting it at the end. Well, when I put it, I thought I did. Not anymore.
If we have to make the change, I'll have to do it asap. We have a very short window. Let me know if you feel that the change is a must-do.
Sunil
PATCH 2/22 ocfs2: Add an incompat feature flag OCFS2_FEATURE_INCOMPAT_CLUSTERINFO
Quoted 15 lines · Sunil Mushran
Quoted 14 lines · Mark Fasheh
fs/ocfs2/ocfs2_fs.h struct ocfs2_slot_map_extended { · line 597
Quoted 10 lines
/*00*/ __u8 ci_stack[OCFS2_STACK_LABEL_LEN];fs/ocfs2/ocfs2_fs.h +7
-1-__le32 ci_reserved; +union { +__le32 ci_reserved; +struct { +__u8 ci_reserved1; +__u8 ci_reserved2; +__u8 ci_reserved3; +__u8 ci_stackflags;Just curious, why stick
ci_stackflagsat the bottom of this structure?Wouldn't putting it at the top allow us to grow the field in the future...
Quoted 7 lines
Quoted 2 lines
Otherwise the patch looks good to me.
Grow stackflags? Hmm... did not think of that.
I don't have a good reason for putting it at the end. Well, when I put it, I thought I did. Not anymore.
Do it. I don't know that we'll ever grow it, but having the
field at the beginning reads better too.
Joel
Signature · 10 lines
"Always give your best, never get discouraged, never be petty; always remember, others may hate you. Those who hate you don't win unless you hate them. And then you destroy yourself." - Richard M. Nixon Joel Becker Consulting Software Developer Oracle E-mail: joel.becker at oracle.com Phone: (650) 506-8127
PATCH 2/22 ocfs2: Add an incompat feature flag OCFS2_FEATURE_INCOMPAT_CLUSTERINFO
Any other changes wrt disk format. Kurt's team is testing this thing. And I would like to make the changes in one go to minimize the pain.
Quoted 41 lines · Joel Becker
Quoted 15 lines · Sunil Mushran
Quoted 14 lines · Mark Fasheh
fs/ocfs2/ocfs2_fs.h struct ocfs2_slot_map_extended { · line 597
Quoted 10 lines
/*00*/
__u8ci_stack[OCFS2_STACK_LABEL_LEN];fs/ocfs2/ocfs2_fs.h +7
-1-__le32 ci_reserved; +union { +__le32 ci_reserved; +struct { +__u8 ci_reserved1; +__u8 ci_reserved2; +__u8 ci_reserved3; +__u8 ci_stackflags;Just curious, why stick
ci_stackflagsat the bottom of this structure?Wouldn't putting it at the top allow us to grow the field in the future...
Quoted 7 lines
Quoted 2 lines
Otherwise the patch looks good to me.
Grow stackflags? Hmm... did not think of that.
I don't have a good reason for putting it at the end. Well, when I put it, I thought I did. Not anymore.
Do it. I don't know that we'll ever grow it, but having the
field at the beginning reads better too.Joel
Signature · 10 lines
"Always give your best, never get discouraged, never be petty; always remember, others may hate you. Those who hate you don't win unless you hate them. And then you destroy yourself." - Richard M. Nixon Joel Becker Consulting Software Developer Oracle E-mail: joel.becker at oracle.com Phone: (650) 506-8127
PATCH 2/22 ocfs2: Add an incompat feature flag OCFS2_FEATURE_INCOMPAT_CLUSTERINFO
http://oss.oracle.com/git/?p=smushran/linux-2.6.git;a=summary
So there is a new branch "global-heartbeat-2" that has this change.
G/./ocfs2/ocfs2_fs.h 2010-10-09 10:24:53.000000000 -0700 renamed from ocfs2/ocfs2_fs.h 2010-10-08 14:12:55.000000000 -0700 +1 -1
@@ -592,10 +592,10 @@ union { __le32 ci_reserved; struct { + __u8 ci_stackflags; __u8 ci_reserved1; __u8 ci_reserved2; __u8 ci_reserved3; - __u8 ci_stackflags; }; }; /*08*/ __u8 ci_cluster[OCFS2_CLUSTER_NAME_LEN];
PATCH 2/22 ocfs2: Add an incompat feature flag OCFS2_FEATURE_INCOMPAT_CLUSTERINFO
Quoted 3 lines · Sunil Mushran
http://oss.oracle.com/git/?p=smushran/linux-2.6.git;a=summary
So there is a new branch "global-heartbeat-2" that has this change.
I couldn't find that, but I put the branch globalheartbeat-2
into merge-window. ;-)
Joel
Signature · 9 lines
Life's Little Instruction Book #444 "Never underestimate the power of a kind word or deed." Joel Becker Consulting Software Developer Oracle E-mail: joel.becker at oracle.com Phone: (650) 506-8127
PATCH 3/22 ocfs2: Add support for heartbeat=global mount option
In reply to Sunil MushranAdds support for heartbeat=global mount option. It ensures that the heartbeat mode passed matches the one enabled on disk.
--- fs/ocfs2/ocfs2.h | 4 ++- fs/ocfs2/ocfs2_fs.h | 1 + fs/ocfs2/super.c | 55 ++++++++++++++++++++++++++++++++++++++------------- 3 files changed, 45 insertions(+), 15 deletions(-)
fs/ocfs2/ocfs2.h +3 -1
@@ -243,7 +243,7 @@ enum ocfs2_local_alloc_stateenum ocfs2_mount_options { -OCFS2_MOUNT_HB_LOCAL =1 <<0 ,/* Heartbeat started in local mode */ +OCFS2_MOUNT_HB_LOCAL =1 <<0 ,/* Local heartbeat */ OCFS2_MOUNT_BARRIER =1 <<1 ,/* Use block barriers */ OCFS2_MOUNT_NOINTR =1 <<2 ,/* Don't catch signals */ OCFS2_MOUNT_ERRORS_PANIC =1 <<3 ,/* Panic on errors */
@@ -256,6 +256,8 @@ enum ocfs2_mount_optionscontrol lists */ OCFS2_MOUNT_USRQUOTA =1 <<10 ,/* We support user quotas */ OCFS2_MOUNT_GRPQUOTA =1 <<11 ,/* We support group quotas */ +OCFS2_MOUNT_HB_NONE =1 <<12 ,/* No heartbeat */ + OCFS2_MOUNT_HB_GLOBAL = 1 << 13, /* Global heartbeat */ };#define OCFS2_OSB_SOFT_RO 0x0001
fs/ocfs2/ocfs2_fs.h +1 -0
@@ -376,6 +376,7 @@ static struct ocfs2_system_inode_info ocfs2_system_inodes[NUM_SYSTEM_INODES] = {/* Parameter passed from mount.ocfs2 to module */ #define OCFS2_HB_NONE "heartbeat=none"#define OCFS2_HB_LOCAL "heartbeat=local" +#define OCFS2_HB_GLOBAL "heartbeat=global"/* * OCFS2 directory file types. Only the low 3 bits are used. The
fs/ocfs2/super.c +41 -14
119 lines not on this page · raw message
Signature · 1 line
1.7.0.4
PATCH 4/22 ocfs2/dlm: Expose dlm_protocol in dlm_state
In reply to Sunil MushranAdd dlm_protocol to the list of info shown by the debugfs file, dlm_state.
--- fs/ocfs2/dlm/dlmdebug.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-)
fs/ocfs2/dlm/dlmdebug.c +3 -1
@@ -782,7 +782,9 @@ static int debug_state_print(struct dlm_ctxt *dlm, struct debug_buffer *db)/* Domain: xxxxxxxxxx Key: 0xdfbac769 */ out +=snprintf (db->buf + out, db->len - out, -"Domain: %s Key: 0x%08x\n" , dlm->name, dlm->key); +"Domain: %s Key: 0x%08x Protocol: %d.%d\n" , + dlm->name, dlm->key, dlm->dlm_locking_proto.pv_major, + dlm->dlm_locking_proto.pv_minor);/* Thread Pid: xxx Node: xxx State: xxxxx */ out += snprintf(db->buf + out, db->len - out,
Signature · 1 line
1.7.0.4
PATCH 5/22 ocfs2/cluster: Get all heartbeat regions
In reply to Sunil MushranExport function in o2hb to get a list of heartbeat regions. It also adds an upper limit to the length of the heartbeat region name.
o2hb_global_heartbeat_active() currently disables global heartbeat. It will be enabled in a later patch after all the code is added.
--- fs/ocfs2/cluster/heartbeat.c | 34 ++++++++++++++++++++++++++++++++++ fs/ocfs2/cluster/heartbeat.h | 4 ++++ 2 files changed, 38 insertions(+), 0 deletions(-)
fs/ocfs2/cluster/heartbeat.c +34 -0
@@ -1623,6 +1623,9 @@ static struct config_item *o2hb_heartbeat_group_make_item(struct config_group *gif (reg ==NULL )return ERR_PTR (-ENOMEM ); +if (strlen (name) >O2HB_MAX_REGION_NAME_LEN ) +return ERR_PTR (-ENAMETOOLONG ); +config_item_init_type_name (®->hr_item, name, &o2hb_region_type);spin_lock (&o2hb_live_lock);
@@ -2035,3 +2038,34 @@ void o2hb_stop_all_regions(void)spin_unlock (&o2hb_live_lock); }EXPORT_SYMBOL_GPL (o2hb_stop_all_regions); + +int o2hb_get_all_regions (char *region_uuids,u8 max_regions) +{ +struct o2hb_region *reg; +int numregs =0 ; +char *p; + +spin_lock (&o2hb_live_lock); + + p = region_uuids; +list_for_each_entry (reg, &o2hb_all_regions, hr_all_item) { +mlog (0 ,"Region: %s\n" ,config_item_name (®->hr_item)); +if (numregs < max_regions) { +memcpy (p,config_item_name (®->hr_item), +O2HB_MAX_REGION_NAME_LEN ); + p +=O2HB_MAX_REGION_NAME_LEN ; + } + numregs++; + } + +spin_unlock (&o2hb_live_lock); + +return numregs; +} +EXPORT_SYMBOL_GPL (o2hb_get_all_regions); + +int o2hb_global_heartbeat_active (void ) +{ +return 0 ; +} +EXPORT_SYMBOL (o2hb_global_heartbeat_active);
fs/ocfs2/cluster/heartbeat.h +4 -0
@@ -31,6 +31,8 @@#define O2HB_REGION_TIMEOUT_MS 2000 +#define O2HB_MAX_REGION_NAME_LEN 32 +/* number of changes to be seen as live */ #define O2HB_LIVE_THRESHOLD 2/* number of equal samples to be seen as dead */
@@ -81,5 +83,7 @@ int o2hb_check_node_heartbeating(u8 node_num);int o2hb_check_node_heartbeating_from_callback (u8 node_num);int o2hb_check_local_node_heartbeating (void );void o2hb_stop_all_regions (void ); +int o2hb_get_all_regions (char *region_uuids,u8 numregions); +int o2hb_global_heartbeat_active (void );#endif /* O2CLUSTER_HEARTBEAT_H */
Signature · 1 line
1.7.0.4
PATCH 6/22 ocfs2/dlm: Add message DLM_QUERY_REGION
In reply to Sunil MushranAdds new dlm message DLM_QUERY_REGION that sends the names of all active heartbeat regions. This message is only sent in the global heartbeat mode. If the regions in the joining node do not fully match the ones in the active nodes, the join domain request is rejected.
The dlm_protocol is bumped upto 1.1.
--- fs/ocfs2/cluster/ocfs2_nodemanager.h | 6 + fs/ocfs2/dlm/dlmcommon.h | 12 ++- fs/ocfs2/dlm/dlmdomain.c | 218 ++++++++++++++++++++++++++++++++++ 3 files changed, 235 insertions(+), 1 deletions(-)
fs/ocfs2/cluster/ocfs2_nodemanager.h +6 -0
@@ -36,4 +36,10 @@/* host name, group name, cluster name all 64 bytes */ #define O2NM_MAX_NAME_LEN 64 // __NEW_UTS_LEN +/* +* Maximum number of global heartbeat regions allowed. +* **CAUTION** Changing this number will break dlm compatibility. +*/ +#define O2NM_MAX_REGIONS 32 +#endif /* _OCFS2_NODEMANAGER_H */
fs/ocfs2/dlm/dlmcommon.h +11 -1
@@ -445,7 +445,8 @@ enum {DLM_LOCK_REQUEST_MSG ,/* 515 */ DLM_RECO_DATA_DONE_MSG ,/* 516 */ DLM_BEGIN_RECO_MSG ,/* 517 */ - DLM_FINALIZE_RECO_MSG /* 518 */ +DLM_FINALIZE_RECO_MSG ,/* 518 */ + DLM_QUERY_REGION, /* 519 */ };struct dlm_reco_node_data
@@ -727,6 +728,15 @@ struct dlm_cancel_joinu8 domain[O2NM_MAX_NAME_LEN ]; }; +struct dlm_query_region { +u8 qr_node; +u8 qr_numregions; +u8 qr_namelen; +u8 pad1; +u8 qr_domain[O2NM_MAX_NAME_LEN ]; +u8 qr_regions[O2HB_MAX_REGION_NAME_LEN *O2NM_MAX_REGIONS ]; +}; +struct dlm_exit_domain {u8 node_idx;
fs/ocfs2/dlm/dlmdomain.c +218 -0
248 lines not on this page · raw message
Signature · 1 line
1.7.0.4
PATCH 7/22 ocfs2: Print message if user mounts without starting global heartbeat
In reply to Sunil MushranIn global heartbeat mode, the heartbeat is started by the user. This patch prints an error if the user attempts to mount a volume without starting the heartbeat.
--- fs/ocfs2/stack_o2cb.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-)
fs/ocfs2/stack_o2cb.c +2 -0
@@ -283,6 +283,8 @@ static int o2cb_cluster_connect(struct ocfs2_cluster_connection *conn)/* for now we only have one cluster/node, make sure we see it * in the heartbeat universe */ if (!o2hb_check_local_node_heartbeating ()) { +if (o2hb_global_heartbeat_active ()) +mlog (ML_ERROR ,"Global heartbeat not started\n" ); rc = -EINVAL ; goto out; }
Signature · 1 line
1.7.0.4
PATCH 8/22 ocfs2/dlm: Add message DLM_QUERY_NODEINFO
In reply to Sunil MushranAdds new dlm message DLM_QUERY_NODEINFO that sends the attributes of all registered nodes. This message is sent if the negotiated dlm protocol is 1.1 or higher. If the information of the joining node does not match that of any existing nodes, the join domain request is rejected.
--- fs/ocfs2/dlm/dlmcommon.h | 17 ++++ fs/ocfs2/dlm/dlmdomain.c | 182 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 198 insertions(+), 1 deletions(-)
fs/ocfs2/dlm/dlmcommon.h +17 -0
@@ -447,6 +447,7 @@ enum {DLM_BEGIN_RECO_MSG ,/* 517 */ DLM_FINALIZE_RECO_MSG ,/* 518 */ DLM_QUERY_REGION ,/* 519 */ + DLM_QUERY_NODEINFO, /* 520 */ };struct dlm_reco_node_data
@@ -737,6 +738,22 @@ struct dlm_query_region {u8 qr_regions[O2HB_MAX_REGION_NAME_LEN *O2NM_MAX_REGIONS ]; }; +struct dlm_node_info { +u8 ni_nodenum; +u8 pad1; +u16 ni_ipv4_port; +u32 ni_ipv4_address; +}; + +struct dlm_query_nodeinfo { +u8 qn_nodenum; +u8 qn_numnodes; +u8 qn_namelen; +u8 pad1; +u8 qn_domain[O2NM_MAX_NAME_LEN ]; +struct dlm_node_info qn_nodes[O2NM_MAX_NODES ]; +}; +struct dlm_exit_domain {u8 node_idx;
fs/ocfs2/dlm/dlmdomain.c +181 -1
207 lines not on this page · raw message
Signature · 1 line
1.7.0.4
PATCH 9/22 ocfs2/cluster: Print messages when adding/removing heartbeat regions
In reply to Sunil MushranPrints messages when the user adds or removes heartbeat regions in global heartbeat mode. These messages are useful when debugging cluster related issues.
--- fs/ocfs2/cluster/heartbeat.c | 9 ++++++++- 1 files changed, 8 insertions(+), 1 deletions(-)
fs/ocfs2/cluster/heartbeat.c +8 -1
@@ -1476,6 +1476,10 @@ static ssize_t o2hb_region_dev_write(struct o2hb_region *reg,else ret = -EIO ; +if (hb_task &&o2hb_global_heartbeat_active ()) +printk (KERN_NOTICE "o2hb: Heartbeat started on region %s\n" , +config_item_name (®->hr_item)); + out:if (filp)fput (filp);
@@ -1659,6 +1663,9 @@ static void o2hb_heartbeat_group_drop_item(struct config_group *group,wake_up (&o2hb_steady_queue); } +if (o2hb_global_heartbeat_active ()) +printk (KERN_NOTICE "o2hb: Heartbeat stopped on region %s\n" , +config_item_name (®->hr_item));config_item_put (item); }
@@ -1745,7 +1752,7 @@ ssize_t o2hb_heartbeat_group_mode_store(struct o2hb_heartbeat_group *group, ret =o2hb_global_hearbeat_mode_set (i);if (!ret) -printk (KERN_NOTICE "ocfs2: Heartbeat mode set to %s\n" , +printk (KERN_NOTICE "o2hb: Heartbeat mode set to %s\n" , o2hb_heartbeat_mode_desc[i]);return count; }
Signature · 1 line
1.7.0.4
PATCH 10/22 ocfs2/cluster: Print messages when adding/removing nodes
In reply to Sunil MushranPrints messages when the user adds or removes nodes.
--- fs/ocfs2/cluster/masklog.h | 3 ++- fs/ocfs2/cluster/nodemanager.c | 5 +++++ 2 files changed, 7 insertions(+), 1 deletions(-)
fs/ocfs2/cluster/masklog.h +2 -1
@@ -119,7 +119,8 @@#define ML_ERROR 0x0000000100000000ULL/* sent to KERN_ERR */ #define ML_NOTICE 0x0000000200000000ULL/* setn to KERN_NOTICE */ #define ML_KTHREAD 0x0000000400000000ULL/* kernel thread activity */ -#define ML_RESERVATIONS 0x0000000800000000ULL/* ocfs2 alloc reservations */ +#define ML_RESERVATIONS 0x0000000800000000ULL/* ocfs2 alloc reservations */ +#define ML_O2CB 0x0000001000000000ULL/* o2cb cluster */ #define MLOG_INITIAL_AND_MASK (ML_ERROR|ML_NOTICE)#define MLOG_INITIAL_NOT_MASK (ML_ENTRY|ML_EXIT)
fs/ocfs2/cluster/nodemanager.c +5 -0
@@ -711,6 +711,8 @@ static struct config_item *o2nm_node_group_make_item(struct config_group *group,config_item_init_type_name (&node->nd_item, name, &o2nm_node_type);spin_lock_init (&node->nd_lock); +mlog (ML_O2CB ,"o2nm: Registering node %s\n" , name); +return &node->nd_item; }
@@ -744,6 +746,9 @@ static void o2nm_node_group_drop_item(struct config_group *group, }write_unlock (&cluster->cl_nodes_lock); +mlog (ML_O2CB ,"o2nm: Unregistered node %s\n" , +config_item_name (&node->nd_item)); +config_item_put (item); }
Signature · 1 line
1.7.0.4
PATCH 10/22 ocfs2/cluster: Print messages when adding/removing nodes
So I am going to change ML_O2CB to ML_CLUSTER.
Quoted 46 lines · Sunil Mushran
Prints messages when the user adds or removes nodes.
Signed-off-by Sunil Mushran<[email protected]>--- fs/ocfs2/cluster/masklog.h | 3 ++- fs/ocfs2/cluster/nodemanager.c | 5 +++++ 2 files changed, 7 insertions(+), 1 deletions(-)fs/ocfs2/cluster/masklog.h +2
-1@@ -119,7 +119,8 @@#define ML_ERROR 0x0000000100000000ULL/* sent to KERN_ERR */ #define ML_NOTICE 0x0000000200000000ULL/* setn to KERN_NOTICE */ #define ML_KTHREAD 0x0000000400000000ULL/* kernel thread activity */ -#define ML_RESERVATIONS 0x0000000800000000ULL/* ocfs2 alloc reservations */ +#define ML_RESERVATIONS 0x0000000800000000ULL/* ocfs2 alloc reservations */ +#define ML_O2CB 0x0000001000000000ULL/* o2cb cluster */ #define MLOG_INITIAL_AND_MASK (ML_ERROR|ML_NOTICE)#define MLOG_INITIAL_NOT_MASK (ML_ENTRY|ML_EXIT)fs/ocfs2/cluster/nodemanager.c +5
-0@@ -711,6 +711,8 @@ static struct config_item *o2nm_node_group_make_item(struct config_group *group,config_item_init_type_name (&node->nd_item, name,&o2nm_node_type);spin_lock_init (&node->nd_lock); +mlog (ML_O2CB ,"o2nm: Registering node %s\n" , name); +return &node->nd_item; }@@ -744,6 +746,9 @@ static void o2nm_node_group_drop_item(struct config_group *group, }write_unlock (&cluster->cl_nodes_lock); +mlog (ML_O2CB ,"o2nm: Unregistered node %s\n" , +config_item_name (&node->nd_item)); +config_item_put (item); }
PATCH 11/22 ocfs2/cluster: Check slots for unconfigured live nodes
In reply to Sunil Mushrano2hb currently checks slots for configured nodes only. This patch makes it check the slots for the live nodes too to take care of a race in which a node is removed from the configuration but not from the live map.
--- fs/ocfs2/cluster/heartbeat.c | 38 +++++++++++++++++++++++++++++++------- fs/ocfs2/cluster/tcp.c | 5 +++++ 2 files changed, 36 insertions(+), 7 deletions(-)
fs/ocfs2/cluster/heartbeat.c +31 -7
@@ -541,6 +541,8 @@ static void o2hb_queue_node_event(struct o2hb_node_event *event, {assert_spin_locked (&o2hb_live_lock); +BUG_ON ((!node) && (type !=O2HB_NODE_DOWN_CB )); + event->hn_event_type = type; event->hn_node = node; event->hn_node_num = node_num;
@@ -593,14 +595,22 @@ static int o2hb_check_slot(struct o2hb_region *reg,u64 cputime;unsigned int dead_ms = o2hb_dead_threshold *O2HB_REGION_TIMEOUT_MS ;unsigned int slot_dead_ms; +int tmp;memcpy (hb_block, slot->ds_raw_block, reg->hr_block_bytes); -/* Is this correct? Do we assume that the node doesn't exist -* if we're not configured for him? */ +/* +* If a node is no longer configured but is still in the livemap, we +* may need to clear that bit from the livemap. +*/ node =o2nm_get_node_by_num (slot->ds_node_num); -if (!node) -return 0 ; +if (!node) { +spin_lock (&o2hb_live_lock); + tmp =test_bit (slot->ds_node_num, o2hb_live_node_bitmap); +spin_unlock (&o2hb_live_lock); +if (!tmp) +return 0 ; + }if (!o2hb_verify_crc (reg, hb_block)) {/* all paths from here will drop o2hb_live_lock for
@@ -717,8 +727,9 @@ fire_callbacks:if (list_empty (&o2hb_live_slots[slot->ds_node_num])) {clear_bit (slot->ds_node_num, o2hb_live_node_bitmap); -o2hb_queue_node_event (&event,O2HB_NODE_DOWN_CB , node, - slot->ds_node_num); +/* node can be null */ +o2hb_queue_node_event (&event,O2HB_NODE_DOWN_CB , + node, slot->ds_node_num); changed =1 ; }
@@ -738,7 +749,8 @@ out:o2hb_run_event_list (&event); -o2nm_node_put (node); +if (node) +o2nm_node_put (node);return changed; }
@@ -765,6 +777,7 @@ static int o2hb_do_disk_heartbeat(struct o2hb_region *reg) {int i, ret, highest_node, change =0 ;unsigned long configured_nodes[BITS_TO_LONGS (O2NM_MAX_NODES )]; +unsigned long live_node_bitmap[BITS_TO_LONGS (O2NM_MAX_NODES )];struct o2hb_bio_wait_ctxt write_wc; ret = o2nm_configured_node_map(configured_nodes,
@@ -774,6 +787,17 @@ static int o2hb_do_disk_heartbeat(struct o2hb_region *reg)return ret; } +/* +* If a node is not configured but is in the livemap, we still need +* to read the slot so as to be able to remove it from the livemap. +*/ +o2hb_fill_node_map (live_node_bitmap,sizeof (live_node_bitmap)); + i =-1 ; +while ((i =find_next_bit (live_node_bitmap, +O2NM_MAX_NODES , i +1 )) <O2NM_MAX_NODES ) { +set_bit (i, configured_nodes); + } + highest_node =o2hb_highest_node (configured_nodes,O2NM_MAX_NODES );if (highest_node >=O2NM_MAX_NODES ) {mlog (ML_NOTICE ,"ocfs2_heartbeat: no configured nodes found!\n" );
fs/ocfs2/cluster/tcp.c +5 -0
@@ -1696,6 +1696,9 @@ static void o2net_hb_node_down_cb(struct o2nm_node *node, int node_num, {o2quo_hb_down (node_num); +if (!node) +return ; +if (node_num !=o2nm_this_node ())o2net_disconnect_node (node);
@@ -1709,6 +1712,8 @@ static void o2net_hb_node_up_cb(struct o2nm_node *node, int node_num,o2quo_hb_up (node_num); +BUG_ON (!node); +/* ensure an immediate connect attempt */ nn->nn_last_connect_attempt = jiffies - (msecs_to_jiffies (o2net_reconnect_delay ()) +1 );
Signature · 1 line
1.7.0.4
PATCH 12/22 ocfs2/cluster: Reorganize o2hb debugfs init
In reply to Sunil Mushrano2hb debugfs handling is reorganized to allow for easy expansion.
--- fs/ocfs2/cluster/heartbeat.c | 101 ++++++++++++++++++++++++++++++++---------- 1 files changed, 78 insertions(+), 23 deletions(-)
fs/ocfs2/cluster/heartbeat.c +78 -23
@@ -62,8 +62,19 @@ static unsigned long o2hb_live_node_bitmap[BITS_TO_LONGS(O2NM_MAX_NODES)];static LIST_HEAD (o2hb_node_events);static DECLARE_WAIT_QUEUE_HEAD (o2hb_steady_queue); +#define O2HB_DB_TYPE_LIVENODES 0 +struct o2hb_debug_buf { +int db_type; +int db_size; +int db_len; +void *db_data; +}; + +static struct o2hb_debug_buf *o2hb_db_livenodes; +#define O2HB_DEBUG_DIR "o2hb"#define O2HB_DEBUG_LIVENODES "livenodes" +static struct dentry *o2hb_debug_dir;static struct dentry *o2hb_debug_livenodes;
@@ -969,21 +980,35 @@ static int o2hb_thread(void *data)#ifdef CONFIG_DEBUG_FS static int o2hb_debug_open (struct inode *inode,struct file *file) { +struct o2hb_debug_buf *db = inode->i_private;unsigned long map[BITS_TO_LONGS (O2NM_MAX_NODES )];char *buf =NULL ;int i =-1 ;int out =0 ; +/* max_nodes should be the largest bitmap we pass here */ +BUG_ON (sizeof (map) < db->db_size); + buf =kmalloc (PAGE_SIZE ,GFP_KERNEL );if (!buf) goto bail; -o2hb_fill_node_map (map,sizeof (map)); +switch (db->db_type) { +case O2HB_DB_TYPE_LIVENODES : +spin_lock (&o2hb_live_lock); +memcpy (map, db->db_data, db->db_size); +spin_unlock (&o2hb_live_lock); +break ; + +default : + goto done; + } -while ((i =find_next_bit (map,O2NM_MAX_NODES , i +1 )) <O2NM_MAX_NODES ) +while ((i =find_next_bit (map, db->db_len, i +1 )) < db->db_len) out +=snprintf (buf + out,PAGE_SIZE - out,"%d " , i); out +=snprintf (buf + out,PAGE_SIZE - out,"\n" ); +done:i_size_write (inode, out); file->private_data = buf;
@@ -1030,10 +1055,56 @@ static const struct file_operations o2hb_debug_fops = {void o2hb_exit (void ) { -if (o2hb_debug_livenodes) -debugfs_remove (o2hb_debug_livenodes); -if (o2hb_debug_dir) -debugfs_remove (o2hb_debug_dir); +kfree (o2hb_db_livenodes); +debugfs_remove (o2hb_debug_livenodes); +debugfs_remove (o2hb_debug_dir); +} + +static struct dentry *o2hb_debug_create (const char *name,struct dentry *dir, +struct o2hb_debug_buf **db,int db_len, +int type,int size,int len,void *data) +{ + *db =kmalloc (db_len,GFP_KERNEL ); +if (!*db) +return NULL ; + + (*db)->db_type = type; + (*db)->db_size = size; + (*db)->db_len = len; + (*db)->db_data = data; + +return debugfs_create_file (name,S_IFREG |S_IRUSR , dir, *db, + &o2hb_debug_fops); +} + +static int o2hb_debug_init (void ) +{ +int ret = -ENOMEM ; + + o2hb_debug_dir =debugfs_create_dir (O2HB_DEBUG_DIR ,NULL ); +if (!o2hb_debug_dir) { +mlog_errno (ret); + goto bail; + } + + o2hb_debug_livenodes =o2hb_debug_create (O2HB_DEBUG_LIVENODES , + o2hb_debug_dir, + &o2hb_db_livenodes, +sizeof (*o2hb_db_livenodes), +O2HB_DB_TYPE_LIVENODES , +sizeof (o2hb_live_node_bitmap), +O2NM_MAX_NODES , + o2hb_live_node_bitmap); +if (!o2hb_debug_livenodes) { +mlog_errno (ret); + goto bail; + } + ret =0 ; +bail: +if (ret) +o2hb_exit (); + +return ret; }int o2hb_init (void )
@@ -1050,23 +1121,7 @@ int o2hb_init(void)memset (o2hb_live_node_bitmap,0 ,sizeof (o2hb_live_node_bitmap)); - o2hb_debug_dir =debugfs_create_dir (O2HB_DEBUG_DIR ,NULL ); -if (!o2hb_debug_dir) { -mlog_errno (-ENOMEM ); -return -ENOMEM ; - } - - o2hb_debug_livenodes =debugfs_create_file (O2HB_DEBUG_LIVENODES , -S_IFREG |S_IRUSR , - o2hb_debug_dir,NULL , - &o2hb_debug_fops); -if (!o2hb_debug_livenodes) { -mlog_errno (-ENOMEM ); -debugfs_remove (o2hb_debug_dir); -return -ENOMEM ; - } - -return 0 ; +return o2hb_debug_init (); }/* if we're already in a callback then we're already serialized by the sem */
Signature · 1 line
1.7.0.4
PATCH 13/22 ocfs2/cluster: Maintain live node bitmap per heartbeat region
In reply to Sunil MushranCurrently we track a global livenode bitmap that keeps track of all nodes that are heartbeating in all regions.
This patch adds the ability to track the livenode bitmap on a per region basis. We will use this facility in a later patch to allow us to withstand the loss of a minority number of regions.
--- fs/ocfs2/cluster/heartbeat.c | 7 +++++++ 1 files changed, 7 insertions(+), 0 deletions(-)
fs/ocfs2/cluster/heartbeat.c +7 -0
@@ -174,6 +174,9 @@ struct o2hb_region {struct block_device *hr_bdev;struct o2hb_disk_slot *hr_slots; +/* live node map of this region */ +unsigned long hr_live_node_bitmap[BITS_TO_LONGS (O2NM_MAX_NODES )]; +/* let the person setting up hb wait for it to return until it * has reached a 'steady' state. This will be fixed when we have * a more complete api that doesn't lead to this sort of fragility. */
@@ -688,6 +691,8 @@ fire_callbacks:mlog (ML_HEARTBEAT ,"Node %d (id 0x%llx) joined my region\n" , slot->ds_node_num, (long long )slot->ds_last_generation); +set_bit (slot->ds_node_num, reg->hr_live_node_bitmap); +/* first on the list generates a callback */ if (list_empty (&o2hb_live_slots[slot->ds_node_num])) {set_bit (slot->ds_node_num, o2hb_live_node_bitmap);
@@ -733,6 +738,8 @@ fire_callbacks:mlog (ML_HEARTBEAT ,"Node %d left my region\n" , slot->ds_node_num); +clear_bit (slot->ds_node_num, reg->hr_live_node_bitmap); +/* last off the live_slot generates a callback */ list_del_init (&slot->ds_live_item);if (list_empty (&o2hb_live_slots[slot->ds_node_num])) {
Signature · 1 line
1.7.0.4
PATCH 14/22 ocfs2/cluster: Track number of global heartbeat regions
In reply to Sunil MushranIn global heartbeat mode, we have a upper limit for the number of active regions. This patch adds the facility to track the number of active global heartbeat regions and fails to start heartbeat if the number exceeds the maximum.
--- fs/ocfs2/cluster/heartbeat.c | 24 ++++++++++++++++++++++-- 1 files changed, 22 insertions(+), 2 deletions(-)
fs/ocfs2/cluster/heartbeat.c +22 -2
@@ -62,6 +62,12 @@ static unsigned long o2hb_live_node_bitmap[BITS_TO_LONGS(O2NM_MAX_NODES)];static LIST_HEAD (o2hb_node_events);static DECLARE_WAIT_QUEUE_HEAD (o2hb_steady_queue); +/* +* In global heartbeat, we maintain a series of region bitmaps. +* - o2hb_region_bitmap allows us to limit the region number to max region. +*/ +static unsigned long o2hb_region_bitmap[BITS_TO_LONGS (O2NM_MAX_REGIONS )]; +#define O2HB_DB_TYPE_LIVENODES 0struct o2hb_debug_buf {int db_type;
@@ -176,6 +182,7 @@ struct o2hb_region {/* live node map of this region */ unsigned long hr_live_node_bitmap[BITS_TO_LONGS (O2NM_MAX_NODES )]; +unsigned int hr_region_num;/* let the person setting up hb wait for it to return until it * has reached a 'steady' state. This will be fixed when we have
@@ -1127,6 +1134,7 @@ int o2hb_init(void)INIT_LIST_HEAD (&o2hb_node_events);memset (o2hb_live_node_bitmap,0 ,sizeof (o2hb_live_node_bitmap)); +memset (o2hb_region_bitmap,0 ,sizeof (o2hb_region_bitmap));return o2hb_debug_init (); }
@@ -1716,12 +1724,22 @@ static struct config_item *o2hb_heartbeat_group_make_item(struct config_group *gif (strlen (name) >O2HB_MAX_REGION_NAME_LEN )return ERR_PTR (-ENAMETOOLONG ); -config_item_init_type_name (®->hr_item, name, &o2hb_region_type); -spin_lock (&o2hb_live_lock); + reg->hr_region_num =0 ; +if (o2hb_global_heartbeat_active ()) { + reg->hr_region_num =find_first_zero_bit (o2hb_region_bitmap, +O2NM_MAX_REGIONS ); +if (reg->hr_region_num >=O2NM_MAX_REGIONS ) { +spin_unlock (&o2hb_live_lock); +return ERR_PTR (-EFBIG ); + } +set_bit (reg->hr_region_num, o2hb_region_bitmap); + }list_add_tail (®->hr_all_item, &o2hb_all_regions);spin_unlock (&o2hb_live_lock); +config_item_init_type_name (®->hr_item, name, &o2hb_region_type); +return ®->hr_item; }
@@ -1733,6 +1751,8 @@ static void o2hb_heartbeat_group_drop_item(struct config_group *group,/* stop the thread when the user removes the region dir */ spin_lock (&o2hb_live_lock); +if (o2hb_global_heartbeat_active ()) +clear_bit (reg->hr_region_num, o2hb_region_bitmap); hb_task = reg->hr_task; reg->hr_task =NULL ;spin_unlock (&o2hb_live_lock);
Signature · 1 line
1.7.0.4
PATCH 15/22 ocfs2/cluster: Track bitmap of live heartbeat regions
In reply to Sunil MushranA heartbeat region becomes live (or active) after a fixed number of (steady) iterations.
--- fs/ocfs2/cluster/heartbeat.c | 9 ++++++++- 1 files changed, 8 insertions(+), 1 deletions(-)
fs/ocfs2/cluster/heartbeat.c +8 -1
@@ -65,8 +65,10 @@ static DECLARE_WAIT_QUEUE_HEAD(o2hb_steady_queue);/* * In global heartbeat, we maintain a series of region bitmaps. * - o2hb_region_bitmap allows us to limit the region number to max region. +* - o2hb_live_region_bitmap tracks live regions (seen steady iterations). */ static unsigned long o2hb_region_bitmap[BITS_TO_LONGS (O2NM_MAX_REGIONS )]; +static unsigned long o2hb_live_region_bitmap[BITS_TO_LONGS (O2NM_MAX_REGIONS )];#define O2HB_DB_TYPE_LIVENODES 0struct o2hb_debug_buf {
@@ -1135,6 +1137,7 @@ int o2hb_init(void)memset (o2hb_live_node_bitmap,0 ,sizeof (o2hb_live_node_bitmap));memset (o2hb_region_bitmap,0 ,sizeof (o2hb_region_bitmap)); +memset (o2hb_live_region_bitmap,0 ,sizeof (o2hb_live_region_bitmap));return o2hb_debug_init (); }
@@ -1563,6 +1566,8 @@ static ssize_t o2hb_region_dev_write(struct o2hb_region *reg,/* Ok, we were woken. Make sure it wasn't by drop_item() */ spin_lock (&o2hb_live_lock); hb_task = reg->hr_task; +if (o2hb_global_heartbeat_active ()) +set_bit (reg->hr_region_num, o2hb_live_region_bitmap);spin_unlock (&o2hb_live_lock);if (hb_task)
@@ -1751,8 +1756,10 @@ static void o2hb_heartbeat_group_drop_item(struct config_group *group,/* stop the thread when the user removes the region dir */ spin_lock (&o2hb_live_lock); -if (o2hb_global_heartbeat_active ()) +if (o2hb_global_heartbeat_active ()) {clear_bit (reg->hr_region_num, o2hb_region_bitmap); +clear_bit (reg->hr_region_num, o2hb_live_region_bitmap); + } hb_task = reg->hr_task; reg->hr_task =NULL ;spin_unlock (&o2hb_live_lock);
Signature · 1 line
1.7.0.4
PATCH 16/22 ocfs2/cluster: Maintain bitmap of quorum regions
In reply to Sunil Mushrano2hb allows online adding of regions. However, a newly added region is not used in quorum calculations unless it has been added on all nodes. This patch tracks a bitmap of such quorum regions.
--- fs/ocfs2/cluster/heartbeat.c | 35 +++++++++++++++++++++++++++++++++++ 1 files changed, 35 insertions(+), 0 deletions(-)
fs/ocfs2/cluster/heartbeat.c +35 -0
@@ -66,9 +66,12 @@ static DECLARE_WAIT_QUEUE_HEAD(o2hb_steady_queue);* In global heartbeat, we maintain a series of region bitmaps. * - o2hb_region_bitmap allows us to limit the region number to max region. * - o2hb_live_region_bitmap tracks live regions (seen steady iterations). +* - o2hb_quorum_region_bitmap tracks live regions that have seen all nodes +* heartbeat on it. */ static unsigned long o2hb_region_bitmap[BITS_TO_LONGS (O2NM_MAX_REGIONS )];static unsigned long o2hb_live_region_bitmap[BITS_TO_LONGS (O2NM_MAX_REGIONS )]; +static unsigned long o2hb_quorum_region_bitmap[BITS_TO_LONGS (O2NM_MAX_REGIONS )];#define O2HB_DB_TYPE_LIVENODES 0struct o2hb_debug_buf {
@@ -607,6 +610,35 @@ static void o2hb_shutdown_slot(struct o2hb_disk_slot *slot)o2nm_node_put (node); } +static void o2hb_set_quorum_device (struct o2hb_region *reg, +struct o2hb_disk_slot *slot) +{ +assert_spin_locked (&o2hb_live_lock); + +if (!o2hb_global_heartbeat_active ()) +return ; + +if (test_bit (reg->hr_region_num, o2hb_quorum_region_bitmap)) +return ; + +/* +* A region can be added to the quorum only when it sees all +* live nodes heartbeat on it. In other words, the region has been +* added to all nodes. +*/ +if (memcmp (reg->hr_live_node_bitmap, o2hb_live_node_bitmap, +sizeof (o2hb_live_node_bitmap))) +return ; + +if (slot->ds_changed_samples <O2HB_LIVE_THRESHOLD ) +return ; + +printk (KERN_NOTICE "o2hb: Region %s is now a quorum device\n" , +config_item_name (®->hr_item)); + +set_bit (reg->hr_region_num, o2hb_quorum_region_bitmap); +} +static int o2hb_check_slot (struct o2hb_region *reg,struct o2hb_disk_slot *slot) {
@@ -772,6 +804,8 @@ fire_callbacks: slot->ds_equal_samples =0 ; } out: +o2hb_set_quorum_device (reg, slot); +spin_unlock (&o2hb_live_lock);o2hb_run_event_list (&event);
@@ -1138,6 +1172,7 @@ int o2hb_init(void)memset (o2hb_live_node_bitmap,0 ,sizeof (o2hb_live_node_bitmap));memset (o2hb_region_bitmap,0 ,sizeof (o2hb_region_bitmap));memset (o2hb_live_region_bitmap,0 ,sizeof (o2hb_live_region_bitmap)); +memset (o2hb_quorum_region_bitmap,0 ,sizeof (o2hb_quorum_region_bitmap));return o2hb_debug_init (); }
Signature · 1 line
1.7.0.4
PATCH 17/22 ocfs2/cluster: Maintain bitmap of failed regions
In reply to Sunil MushranIn global heartbeat mode, we track the bitmap of regions that have seen heartbeat timeouts. We fence if the number of such regions is greater than or equal to half the number of quorum regions.
--- fs/ocfs2/cluster/heartbeat.c | 41 +++++++++++++++++++++++++++++++++++++++++ 1 files changed, 41 insertions(+), 0 deletions(-)
fs/ocfs2/cluster/heartbeat.c +41 -0
@@ -68,10 +68,12 @@ static DECLARE_WAIT_QUEUE_HEAD(o2hb_steady_queue);* - o2hb_live_region_bitmap tracks live regions (seen steady iterations). * - o2hb_quorum_region_bitmap tracks live regions that have seen all nodes * heartbeat on it. +* - o2hb_failed_region_bitmap tracks the regions that have seen io timeouts. */ static unsigned long o2hb_region_bitmap[BITS_TO_LONGS (O2NM_MAX_REGIONS )];static unsigned long o2hb_live_region_bitmap[BITS_TO_LONGS (O2NM_MAX_REGIONS )];static unsigned long o2hb_quorum_region_bitmap[BITS_TO_LONGS (O2NM_MAX_REGIONS )]; +static unsigned long o2hb_failed_region_bitmap[BITS_TO_LONGS (O2NM_MAX_REGIONS )];#define O2HB_DB_TYPE_LIVENODES 0struct o2hb_debug_buf {
@@ -217,8 +219,19 @@ struct o2hb_bio_wait_ctxt {int wc_error; }; +static int o2hb_pop_count (void *map,int count) +{ +int i =-1 , pop =0 ; + +while ((i =find_next_bit (map, count, i +1 )) < count) + pop++; +return pop; +} +static void o2hb_write_timeout (struct work_struct *work) { +int failed, quorum; +unsigned long flags;struct o2hb_region *reg = container_of(work, struct o2hb_region, hr_write_timeout_work.work);
@@ -226,6 +239,28 @@ static void o2hb_write_timeout(struct work_struct *work)mlog (ML_ERROR ,"Heartbeat write timeout to device %s after %u " "milliseconds\n" , reg->hr_dev_name,jiffies_to_msecs (jiffies - reg->hr_last_timeout_start)); + +if (o2hb_global_heartbeat_active ()) { +spin_lock_irqsave (&o2hb_live_lock, flags); +if (test_bit (reg->hr_region_num, o2hb_quorum_region_bitmap)) +set_bit (reg->hr_region_num, o2hb_failed_region_bitmap); + failed =o2hb_pop_count (&o2hb_failed_region_bitmap, +O2NM_MAX_REGIONS ); + quorum =o2hb_pop_count (&o2hb_quorum_region_bitmap, +O2NM_MAX_REGIONS ); +spin_unlock_irqrestore (&o2hb_live_lock, flags); + +mlog (ML_HEARTBEAT ,"Number of regions %d, failed regions %d\n" , + quorum, failed); + +/* +* Fence if the number of failed regions >= half the number +* of quorum regions +*/ +if ((failed <<1 ) < quorum) +return ; + } +o2quo_disk_timeout (); }
@@ -234,6 +269,11 @@ static void o2hb_arm_write_timeout(struct o2hb_region *reg)mlog (ML_HEARTBEAT ,"Queue write timeout for %u ms\n" ,O2HB_MAX_WRITE_TIMEOUT_MS ); +if (o2hb_global_heartbeat_active ()) { +spin_lock (&o2hb_live_lock); +clear_bit (reg->hr_region_num, o2hb_failed_region_bitmap); +spin_unlock (&o2hb_live_lock); + }cancel_delayed_work (®->hr_write_timeout_work); reg->hr_last_timeout_start = jiffies; schedule_delayed_work(®->hr_write_timeout_work,
@@ -1173,6 +1213,7 @@ int o2hb_init(void)memset (o2hb_region_bitmap,0 ,sizeof (o2hb_region_bitmap));memset (o2hb_live_region_bitmap,0 ,sizeof (o2hb_live_region_bitmap));memset (o2hb_quorum_region_bitmap,0 ,sizeof (o2hb_quorum_region_bitmap)); +memset (o2hb_failed_region_bitmap,0 ,sizeof (o2hb_failed_region_bitmap));return o2hb_debug_init (); }
Signature · 1 line
1.7.0.4
PATCH 18/22 ocfs2/cluster: Create debugfs files for live, quorum and failed region bitmaps
In reply to Sunil MushranThis patch prints the bitmaps of live, quorum and failed regions. This information will be useful in debugging cluster issues.
--- fs/ocfs2/cluster/heartbeat.c | 63 ++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 63 insertions(+), 0 deletions(-)
fs/ocfs2/cluster/heartbeat.c +63 -0
@@ -76,6 +76,9 @@ static unsigned long o2hb_quorum_region_bitmap[BITS_TO_LONGS(O2NM_MAX_REGIONS)];static unsigned long o2hb_failed_region_bitmap[BITS_TO_LONGS (O2NM_MAX_REGIONS )];#define O2HB_DB_TYPE_LIVENODES 0 +#define O2HB_DB_TYPE_LIVEREGIONS 1 +#define O2HB_DB_TYPE_QUORUMREGIONS 2 +#define O2HB_DB_TYPE_FAILEDREGIONS 3struct o2hb_debug_buf {int db_type;int db_size;
@@ -84,12 +87,21 @@ struct o2hb_debug_buf { };static struct o2hb_debug_buf *o2hb_db_livenodes; +static struct o2hb_debug_buf *o2hb_db_liveregions; +static struct o2hb_debug_buf *o2hb_db_quorumregions; +static struct o2hb_debug_buf *o2hb_db_failedregions;#define O2HB_DEBUG_DIR "o2hb"#define O2HB_DEBUG_LIVENODES "livenodes" +#define O2HB_DEBUG_LIVEREGIONS "live_regions" +#define O2HB_DEBUG_QUORUMREGIONS "quorum_regions" +#define O2HB_DEBUG_FAILEDREGIONS "failed_regions"static struct dentry *o2hb_debug_dir;static struct dentry *o2hb_debug_livenodes; +static struct dentry *o2hb_debug_liveregions; +static struct dentry *o2hb_debug_quorumregions; +static struct dentry *o2hb_debug_failedregions;static LIST_HEAD (o2hb_all_regions);
@@ -1085,6 +1097,9 @@ static int o2hb_debug_open(struct inode *inode, struct file *file)switch (db->db_type) {case O2HB_DB_TYPE_LIVENODES : +case O2HB_DB_TYPE_LIVEREGIONS : +case O2HB_DB_TYPE_QUORUMREGIONS : +case O2HB_DB_TYPE_FAILEDREGIONS :spin_lock (&o2hb_live_lock);memcpy (map, db->db_data, db->db_size);spin_unlock (&o2hb_live_lock);
@@ -1146,6 +1161,12 @@ static const struct file_operations o2hb_debug_fops = {void o2hb_exit (void ) {kfree (o2hb_db_livenodes); +kfree (o2hb_db_liveregions); +kfree (o2hb_db_quorumregions); +kfree (o2hb_db_failedregions); +debugfs_remove (o2hb_debug_failedregions); +debugfs_remove (o2hb_debug_quorumregions); +debugfs_remove (o2hb_debug_liveregions);debugfs_remove (o2hb_debug_livenodes);debugfs_remove (o2hb_debug_dir); }
@@ -1189,6 +1210,48 @@ static int o2hb_debug_init(void)mlog_errno (ret); goto bail; } + + o2hb_debug_liveregions =o2hb_debug_create (O2HB_DEBUG_LIVEREGIONS , + o2hb_debug_dir, + &o2hb_db_liveregions, +sizeof (*o2hb_db_liveregions), +O2HB_DB_TYPE_LIVEREGIONS , +sizeof (o2hb_live_region_bitmap), +O2NM_MAX_REGIONS , + o2hb_live_region_bitmap); +if (!o2hb_debug_liveregions) { +mlog_errno (ret); + goto bail; + } + + o2hb_debug_quorumregions = +o2hb_debug_create (O2HB_DEBUG_QUORUMREGIONS , + o2hb_debug_dir, + &o2hb_db_quorumregions, +sizeof (*o2hb_db_quorumregions), +O2HB_DB_TYPE_QUORUMREGIONS , +sizeof (o2hb_quorum_region_bitmap), +O2NM_MAX_REGIONS , + o2hb_quorum_region_bitmap); +if (!o2hb_debug_quorumregions) { +mlog_errno (ret); + goto bail; + } + + o2hb_debug_failedregions = +o2hb_debug_create (O2HB_DEBUG_FAILEDREGIONS , + o2hb_debug_dir, + &o2hb_db_failedregions, +sizeof (*o2hb_db_failedregions), +O2HB_DB_TYPE_FAILEDREGIONS , +sizeof (o2hb_failed_region_bitmap), +O2NM_MAX_REGIONS , + o2hb_failed_region_bitmap); +if (!o2hb_debug_failedregions) { +mlog_errno (ret); + goto bail; + } + ret =0 ; bail:if (ret)
Signature · 1 line
1.7.0.4
PATCH 19/22 ocfs2/cluster: Create debugfs dir/files for each region
In reply to Sunil MushranThis patch creates debugfs directory for each o2hb region and creates files to expose the region number and the per region live node bitmap. This information will be useful in debugging cluster issues.
--- fs/ocfs2/cluster/heartbeat.c | 77 ++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 77 insertions(+), 0 deletions(-)
fs/ocfs2/cluster/heartbeat.c +77 -0
@@ -79,6 +79,8 @@ static unsigned long o2hb_failed_region_bitmap[BITS_TO_LONGS(O2NM_MAX_REGIONS)];#define O2HB_DB_TYPE_LIVEREGIONS 1#define O2HB_DB_TYPE_QUORUMREGIONS 2#define O2HB_DB_TYPE_FAILEDREGIONS 3 +#define O2HB_DB_TYPE_REGION_LIVENODES 4 +#define O2HB_DB_TYPE_REGION_NUMBER 5struct o2hb_debug_buf {int db_type;int db_size;
@@ -96,6 +98,7 @@ static struct o2hb_debug_buf *o2hb_db_failedregions;#define O2HB_DEBUG_LIVEREGIONS "live_regions"#define O2HB_DEBUG_QUORUMREGIONS "quorum_regions"#define O2HB_DEBUG_FAILEDREGIONS "failed_regions" +#define O2HB_DEBUG_REGION_NUMBER "num"static struct dentry *o2hb_debug_dir;static struct dentry *o2hb_debug_livenodes;
@@ -203,6 +206,12 @@ struct o2hb_region {unsigned long hr_live_node_bitmap[BITS_TO_LONGS (O2NM_MAX_NODES )];unsigned int hr_region_num; +struct dentry *hr_debug_dir; +struct dentry *hr_debug_livenodes; +struct dentry *hr_debug_regnum; +struct o2hb_debug_buf *hr_db_livenodes; +struct o2hb_debug_buf *hr_db_regnum; +/* let the person setting up hb wait for it to return until it * has reached a 'steady' state. This will be fixed when we have * a more complete api that doesn't lead to this sort of fragility. */
@@ -1083,6 +1092,7 @@ static int o2hb_thread(void *data)static int o2hb_debug_open (struct inode *inode,struct file *file) {struct o2hb_debug_buf *db = inode->i_private; +struct o2hb_region *reg;unsigned long map[BITS_TO_LONGS (O2NM_MAX_NODES )];char *buf =NULL ;int i =-1 ;
@@ -1105,6 +1115,19 @@ static int o2hb_debug_open(struct inode *inode, struct file *file)spin_unlock (&o2hb_live_lock);break ; +case O2HB_DB_TYPE_REGION_LIVENODES : +spin_lock (&o2hb_live_lock); + reg = (struct o2hb_region *)db->db_data; +memcpy (map, reg->hr_live_node_bitmap, db->db_size); +spin_unlock (&o2hb_live_lock); +break ; + +case O2HB_DB_TYPE_REGION_NUMBER : + reg = (struct o2hb_region *)db->db_data; + out +=snprintf (buf + out,PAGE_SIZE - out,"%d\n" , + reg->hr_region_num); + goto done; +default : goto done; }
@@ -1342,6 +1365,12 @@ static void o2hb_region_release(struct config_item *item)if (reg->hr_slots)kfree (reg->hr_slots); +kfree (reg->hr_db_regnum); +kfree (reg->hr_db_livenodes); +debugfs_remove (reg->hr_debug_livenodes); +debugfs_remove (reg->hr_debug_regnum); +debugfs_remove (reg->hr_debug_dir); +spin_lock (&o2hb_live_lock);list_del (®->hr_all_item);spin_unlock (&o2hb_live_lock);
@@ -1856,10 +1885,52 @@ static struct o2hb_heartbeat_group *to_o2hb_heartbeat_group(struct config_group : NULL; } +static int o2hb_debug_region_init (struct o2hb_region *reg,struct dentry *dir) +{ +int ret = -ENOMEM ; + + reg->hr_debug_dir = +debugfs_create_dir (config_item_name (®->hr_item), dir); +if (!reg->hr_debug_dir) { +mlog_errno (ret); + goto bail; + } + + reg->hr_debug_livenodes = +o2hb_debug_create (O2HB_DEBUG_LIVENODES , + reg->hr_debug_dir, + &(reg->hr_db_livenodes), +sizeof (*(reg->hr_db_livenodes)), +O2HB_DB_TYPE_REGION_LIVENODES , +sizeof (reg->hr_live_node_bitmap), +O2NM_MAX_NODES , reg); +if (!reg->hr_debug_livenodes) { +mlog_errno (ret); + goto bail; + } + + reg->hr_debug_regnum = +o2hb_debug_create (O2HB_DEBUG_REGION_NUMBER , + reg->hr_debug_dir, + &(reg->hr_db_regnum), +sizeof (*(reg->hr_db_regnum)), +O2HB_DB_TYPE_REGION_NUMBER , +0 ,O2NM_MAX_NODES , reg); +if (!reg->hr_debug_regnum) { +mlog_errno (ret); + goto bail; + } + + ret =0 ; +bail: +return ret; +} +static struct config_item *o2hb_heartbeat_group_make_item (struct config_group *group,const char *name) {struct o2hb_region *reg =NULL ; +int ret; reg =kzalloc (sizeof (struct o2hb_region ),GFP_KERNEL );if (reg ==NULL )
@@ -1884,6 +1955,12 @@ static struct config_item *o2hb_heartbeat_group_make_item(struct config_group *gconfig_item_init_type_name (®->hr_item, name, &o2hb_region_type); + ret =o2hb_debug_region_init (reg, o2hb_debug_dir); +if (ret) { +config_item_put (®->hr_item); +return ERR_PTR (ret); + } +return ®->hr_item; }
Signature · 1 line
1.7.0.4
PATCH 20/22 ocfs2/cluster: Add mlogs for heartbeat up/down events
In reply to Sunil MushranThis patch adds mlogs for o2hb up and down events.
--- fs/ocfs2/cluster/heartbeat.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-)
fs/ocfs2/cluster/heartbeat.c +4 -0
@@ -797,6 +797,8 @@ fire_callbacks:/* first on the list generates a callback */ if (list_empty (&o2hb_live_slots[slot->ds_node_num])) { +mlog (ML_HEARTBEAT ,"o2hb: Add node %d to live nodes " +"bitmap\n" , slot->ds_node_num);set_bit (slot->ds_node_num, o2hb_live_node_bitmap); o2hb_queue_node_event(&event, O2HB_NODE_UP_CB, node,
@@ -845,6 +847,8 @@ fire_callbacks:/* last off the live_slot generates a callback */ list_del_init (&slot->ds_live_item);if (list_empty (&o2hb_live_slots[slot->ds_node_num])) { +mlog (ML_HEARTBEAT ,"o2hb: Remove node %d from live " +"nodes bitmap\n" , slot->ds_node_num);clear_bit (slot->ds_node_num, o2hb_live_node_bitmap);/* node can be null */
Signature · 1 line
1.7.0.4
PATCH 21/22 ocfs2/cluster: Show per region heartbeat elapsed time
In reply to Sunil MushranThis patch adds a per region debugfs file that shows the elapsed time since the time the o2hb timer was last armed.
--- fs/ocfs2/cluster/heartbeat.c | 23 +++++++++++++++++++++++ 1 files changed, 23 insertions(+), 0 deletions(-)
fs/ocfs2/cluster/heartbeat.c +23 -0
@@ -81,6 +81,7 @@ static unsigned long o2hb_failed_region_bitmap[BITS_TO_LONGS(O2NM_MAX_REGIONS)];#define O2HB_DB_TYPE_FAILEDREGIONS 3#define O2HB_DB_TYPE_REGION_LIVENODES 4#define O2HB_DB_TYPE_REGION_NUMBER 5 +#define O2HB_DB_TYPE_REGION_ELAPSED_TIME 6struct o2hb_debug_buf {int db_type;int db_size;
@@ -99,6 +100,7 @@ static struct o2hb_debug_buf *o2hb_db_failedregions;#define O2HB_DEBUG_QUORUMREGIONS "quorum_regions"#define O2HB_DEBUG_FAILEDREGIONS "failed_regions"#define O2HB_DEBUG_REGION_NUMBER "num" +#define O2HB_DEBUG_REGION_ELAPSED_TIME "elapsed_time_in_ms"static struct dentry *o2hb_debug_dir;static struct dentry *o2hb_debug_livenodes;
@@ -209,8 +211,10 @@ struct o2hb_region {struct dentry *hr_debug_dir;struct dentry *hr_debug_livenodes;struct dentry *hr_debug_regnum; +struct dentry *hr_debug_elapsed_time;struct o2hb_debug_buf *hr_db_livenodes;struct o2hb_debug_buf *hr_db_regnum; +struct o2hb_debug_buf *hr_db_elapsed_time;/* let the person setting up hb wait for it to return until it * has reached a 'steady' state. This will be fixed when we have
@@ -1132,6 +1136,13 @@ static int o2hb_debug_open(struct inode *inode, struct file *file) reg->hr_region_num); goto done; +case O2HB_DB_TYPE_REGION_ELAPSED_TIME : + reg = (struct o2hb_region *)db->db_data; + out +=snprintf (buf + out,PAGE_SIZE - out,"%u\n" , +jiffies_to_msecs (jiffies - + reg->hr_last_timeout_start)); + goto done; +default : goto done; }
@@ -1925,6 +1936,18 @@ static int o2hb_debug_region_init(struct o2hb_region *reg, struct dentry *dir) goto bail; } + reg->hr_debug_elapsed_time = +o2hb_debug_create (O2HB_DEBUG_REGION_ELAPSED_TIME , + reg->hr_debug_dir, + &(reg->hr_db_elapsed_time), +sizeof (*(reg->hr_db_elapsed_time)), +O2HB_DB_TYPE_REGION_ELAPSED_TIME , +0 ,0 , reg); +if (!reg->hr_debug_elapsed_time) { +mlog_errno (ret); + goto bail; + } + ret =0 ; bail:return ret;
Signature · 1 line
1.7.0.4
PATCH 22/22 ocfs2/dlm: Bump up dlm protocol to version 1.1
In reply to Sunil Mushrandlm protocol 1.1. activates messages DLM_QUERY_REGION and DLM_QUERY_NODEINFO that are a must for global heartbeat.
It also activates o2hb_global_heartbeat_active().
--- fs/ocfs2/cluster/heartbeat.c | 2 +- fs/ocfs2/dlm/dlmdomain.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
fs/ocfs2/cluster/heartbeat.c +1 -1
@@ -2429,6 +2429,6 @@ EXPORT_SYMBOL_GPL(o2hb_get_all_regions);int o2hb_global_heartbeat_active (void ) { -return 0 ; +return (o2hb_heartbeat_mode ==O2HB_HEARTBEAT_GLOBAL ); }EXPORT_SYMBOL (o2hb_global_heartbeat_active);
fs/ocfs2/dlm/dlmdomain.c +1 -1
@@ -135,7 +135,7 @@ static DECLARE_WAIT_QUEUE_HEAD(dlm_domain_events);*/ static const struct dlm_protocol_version dlm_protocol = { .pv_major =1 , - .pv_minor =0 , + .pv_minor =1 , };#define DLM_DOMAIN_BACKOFF_MS 200
Signature · 1 line
1.7.0.4