ocfs2: fix the application IO timeout when fstrim is running
PATCH 1/1 ocfs2: fix the application IO timeout when fstrim is running
The user reported this problem, the upper application IO was timeout when fstrim was running on this ocfs2 partition. the application monitoring resource agent considered that this application did not work, then this node was fenced by the cluster brain (e.g. pacemaker).
The root cause is that fstrim thread always holds main_bm meta-file related locks until all the cluster groups are trimmed. This patch will make fstrim thread release main_bm meta-file related locks when each cluster group is trimmed, this will let the current application IO has a chance to claim the clusters from main_bm meta-file.
--- fs/ocfs2/alloc.c | 159 +++++++++++++++++++++++++---------------- fs/ocfs2/dlmglue.c | 5 ++ fs/ocfs2/ocfs2.h | 1 + fs/ocfs2/ocfs2_trace.h | 2 + fs/ocfs2/super.c | 2 + 5 files changed, 106 insertions(+), 63 deletions(-)
fs/ocfs2/alloc.c +96 -63
@@ -7532,10 +7532,11 @@ static int ocfs2_trim_group(struct super_block *sb,return count; } -int ocfs2_trim_fs (struct super_block *sb,struct fstrim_range *range) +static +int ocfs2_trim_mainbm (struct super_block *sb,struct fstrim_range *range) {struct ocfs2_super *osb =OCFS2_SB (sb); -u64 start, len, trimmed, first_group, last_group, group; +u64 start, len, trimmed =0 , first_group, last_group =0 , group =0 ;int ret, cnt;u32 first_bit, last_bit, minlen;struct buffer_head *main_bm_bh =NULL ;
@@ -7543,7 +7544,6 @@ int ocfs2_trim_fs(struct super_block *sb, struct fstrim_range *range)struct buffer_head *gd_bh =NULL ;struct ocfs2_dinode *main_bm;struct ocfs2_group_desc *gd =NULL ; -struct ocfs2_trim_fs_info info, *pinfo =NULL ; start = range->start >> osb->s_clustersize_bits; len = range->len >> osb->s_clustersize_bits;
@@ -7552,6 +7552,9 @@ int ocfs2_trim_fs(struct super_block *sb, struct fstrim_range *range)if (minlen >= osb->bitmap_cpg || range->len < sb->s_blocksize)return -EINVAL ; +trace_ocfs2_trim_mainbm (start, len, minlen); + +next_group: main_bm_inode =ocfs2_get_system_file_inode (osb,GLOBAL_BITMAP_SYSTEM_INODE ,OCFS2_INVALID_SLOT );
@@ -7570,64 +7573,34 @@ int ocfs2_trim_fs(struct super_block *sb, struct fstrim_range *range) } main_bm = (struct ocfs2_dinode *)main_bm_bh->b_data; -if (start >=le32_to_cpu (main_bm->i_clusters)) { - ret = -EINVAL ; - goto out_unlock; - } - - len = range->len >> osb->s_clustersize_bits; -if (start + len >le32_to_cpu (main_bm->i_clusters)) - len =le32_to_cpu (main_bm->i_clusters) - start; - -trace_ocfs2_trim_fs (start, len, minlen); - -ocfs2_trim_fs_lock_res_init (osb); - ret =ocfs2_trim_fs_lock (osb,NULL ,1 ); -if (ret <0 ) { -if (ret != -EAGAIN ) { -mlog_errno (ret); -ocfs2_trim_fs_lock_res_uninit (osb); +/* +* Do some check before trim the first group. +*/ +if (!group) { +if (start >=le32_to_cpu (main_bm->i_clusters)) { + ret = -EINVAL ; goto out_unlock; } -mlog (ML_NOTICE ,"Wait for trim on device (%s) to " -"finish, which is running from another node.\n" , - osb->dev_str); - ret =ocfs2_trim_fs_lock (osb, &info,0 ); -if (ret <0 ) { -mlog_errno (ret); -ocfs2_trim_fs_lock_res_uninit (osb); - goto out_unlock; - } +if (start + len >le32_to_cpu (main_bm->i_clusters)) + len =le32_to_cpu (main_bm->i_clusters) - start; -if (info.tf_valid && info.tf_success && - info.tf_start == start && info.tf_len == len && - info.tf_minlen == minlen) { -/* Avoid sending duplicated trim to a shared device */ -mlog (ML_NOTICE ,"The same trim on device (%s) was " -"just done from node (%u), return.\n" , - osb->dev_str, info.tf_nodenum); - range->len = info.tf_trimlen; - goto out_trimunlock; - } +/* +* Determine first and last group to examine based on +* start and len +*/ + first_group =ocfs2_which_cluster_group (main_bm_inode, start); +if (first_group == osb->first_cluster_group_blkno) + first_bit = start; +else + first_bit = start -ocfs2_blocks_to_clusters (sb, + first_group); + last_group =ocfs2_which_cluster_group (main_bm_inode, + start + len -1 ); + group = first_group; } - info.tf_nodenum = osb->node_num; - info.tf_start = start; - info.tf_len = len; - info.tf_minlen = minlen; - -/* Determine first and last group to examine based on start and len */ - first_group =ocfs2_which_cluster_group (main_bm_inode, start); -if (first_group == osb->first_cluster_group_blkno) - first_bit = start; -else - first_bit = start -ocfs2_blocks_to_clusters (sb, first_group); - last_group =ocfs2_which_cluster_group (main_bm_inode, start + len -1 ); - last_bit = osb->bitmap_cpg; - - trimmed =0 ; -for (group = first_group; group <= last_group;) { + do { if (first_bit + len >= osb->bitmap_cpg) last_bit = osb->bitmap_cpg; else
@@ -7659,21 +7632,81 @@ int ocfs2_trim_fs(struct super_block *sb, struct fstrim_range *range) group =ocfs2_clusters_to_blocks (sb, osb->bitmap_cpg); else group += ocfs2_clusters_to_blocks(sb, osb->bitmap_cpg); - } - range->len = trimmed * sb->s_blocksize; + }while (0 ); - info.tf_trimlen = range->len; - info.tf_success = (ret ?0 :1 ); - pinfo = &info; -out_trimunlock: -ocfs2_trim_fs_unlock (osb, pinfo); -ocfs2_trim_fs_lock_res_uninit (osb); out_unlock:ocfs2_inode_unlock (main_bm_inode,0 );brelse (main_bm_bh); + main_bm_bh =NULL ; out_mutex:inode_unlock (main_bm_inode);iput (main_bm_inode); + +/* +* If all the groups trim are not done or failed, but we should release +* main_bm related locks for avoiding the current IO starve, then go to +* trim the next group +*/ +if (ret >=0 && group <= last_group) + goto next_group; out: + range->len = trimmed * sb->s_blocksize; +return ret; +} + +int ocfs2_trim_fs (struct super_block *sb,struct fstrim_range *range) +{ +int ret; +struct ocfs2_super *osb =OCFS2_SB (sb); +struct ocfs2_trim_fs_info info, *pinfo =NULL ; + +ocfs2_trim_fs_lock_res_init (osb); + +trace_ocfs2_trim_fs (range->start, range->len, range->minlen); + + ret =ocfs2_trim_fs_lock (osb,NULL ,1 ); +if (ret <0 ) { +if (ret != -EAGAIN ) { +mlog_errno (ret); +ocfs2_trim_fs_lock_res_uninit (osb); +return ret; + } + +mlog (ML_NOTICE ,"Wait for trim on device (%s) to " +"finish, which is running from another node.\n" , + osb->dev_str); + ret =ocfs2_trim_fs_lock (osb, &info,0 ); +if (ret <0 ) { +mlog_errno (ret); +ocfs2_trim_fs_lock_res_uninit (osb); +return ret; + } + +if (info.tf_valid && info.tf_success && + info.tf_start == range->start && + info.tf_len == range->len && + info.tf_minlen == range->minlen) { +/* Avoid sending duplicated trim to a shared device */ +mlog (ML_NOTICE ,"The same trim on device (%s) was " +"just done from node (%u), return.\n" , + osb->dev_str, info.tf_nodenum); + range->len = info.tf_trimlen; + goto out; + } + } + + info.tf_nodenum = osb->node_num; + info.tf_start = range->start; + info.tf_len = range->len; + info.tf_minlen = range->minlen; + + ret =ocfs2_trim_mainbm (sb, range); + + info.tf_trimlen = range->len; + info.tf_success = (ret <0 ?0 :1 ); + pinfo = &info; +out: +ocfs2_trim_fs_unlock (osb, pinfo); +ocfs2_trim_fs_lock_res_uninit (osb);return ret; }
fs/ocfs2/dlmglue.c +5 -0
@@ -686,6 +686,9 @@ void ocfs2_trim_fs_lock_res_init(struct ocfs2_super *osb) {struct ocfs2_lock_res *lockres = &osb->osb_trim_fs_lockres; +/* Only one trimfs thread are allowed to work at the same time. */ +mutex_lock (&osb->obs_trim_fs_mutex); +ocfs2_lock_res_init_once (lockres);ocfs2_build_lock_name (OCFS2_LOCK_TYPE_TRIM_FS ,0 ,0 , lockres->l_name); ocfs2_lock_res_init_common(osb, lockres, OCFS2_LOCK_TYPE_TRIM_FS,
@@ -698,6 +701,8 @@ void ocfs2_trim_fs_lock_res_uninit(struct ocfs2_super *osb)ocfs2_simple_drop_lockres (osb, lockres);ocfs2_lock_res_free (lockres); + +mutex_unlock (&osb->obs_trim_fs_mutex); } static void ocfs2_orphan_scan_lock_res_init(struct ocfs2_lock_res *res,
fs/ocfs2/ocfs2.h +1 -0
@@ -407,6 +407,7 @@ struct ocfs2_superstruct ocfs2_lock_res osb_rename_lockres;struct ocfs2_lock_res osb_nfs_sync_lockres;struct ocfs2_lock_res osb_trim_fs_lockres; +struct mutex obs_trim_fs_mutex;struct ocfs2_dlm_debug *osb_dlm_debug;struct dentry *osb_debug_root;
fs/ocfs2/ocfs2_trace.h +2 -0
@@ -712,6 +712,8 @@ TRACE_EVENT(ocfs2_trim_extent,DEFINE_OCFS2_ULL_UINT_UINT_UINT_EVENT (ocfs2_trim_group); +DEFINE_OCFS2_ULL_ULL_ULL_EVENT (ocfs2_trim_mainbm); +DEFINE_OCFS2_ULL_ULL_ULL_EVENT (ocfs2_trim_fs);/* End of trace events for fs/ocfs2/alloc.c. */
fs/ocfs2/super.c +2 -0
@@ -1847,6 +1847,8 @@ static int ocfs2_mount_volume(struct super_block *sb)if (ocfs2_is_hard_readonly (osb)) goto leave; +mutex_init (&osb->obs_trim_fs_mutex); + status =ocfs2_dlm_init (osb);if (status <0 ) {mlog_errno (status);
Signature · 1 line
2.19.2
Hi Gang,
Quoted 11 lines · Gang He
The user reported this problem, the upper application IO was timeout when fstrim was running on this ocfs2 partition. the application monitoring resource agent considered that this application did not work, then this node was fenced by the cluster brain (e.g. pacemaker).
The root cause is that fstrim thread always holdsmain_bmmeta-file related locks until all the cluster groups are trimmed. This patch will make fstrim thread releasemain_bmmeta-file related locks when each cluster group is trimmed, this will let the current application IO has a chance to claim the clusters frommain_bmmeta-file.
Sounds a good idea, I will look into this.
Thanks,
Changwei
Quoted 285 lines
Signed-off-by Gang He <[email protected]>--- fs/ocfs2/alloc.c | 159 +++++++++++++++++++++++++---------------- fs/ocfs2/dlmglue.c | 5 ++ fs/ocfs2/ocfs2.h | 1 + fs/ocfs2/ocfs2_trace.h | 2 + fs/ocfs2/super.c | 2 + 5 files changed, 106 insertions(+), 63 deletions(-)fs/ocfs2/alloc.c +96
-63@@ -7532,10 +7532,11 @@ static int ocfs2_trim_group(struct super_block *sb,return count; } -int ocfs2_trim_fs (struct super_block *sb,struct fstrim_range *range) +static +int ocfs2_trim_mainbm (struct super_block *sb,struct fstrim_range *range) {struct ocfs2_super *osb =OCFS2_SB (sb); -u64 start, len, trimmed, first_group, last_group, group; +u64 start, len, trimmed =0 , first_group, last_group =0 , group =0 ;int ret, cnt;u32 first_bit, last_bit, minlen;struct buffer_head *main_bm_bh =NULL ;@@ -7543,7 +7544,6 @@ int ocfs2_trim_fs(struct super_block *sb, struct fstrim_range *range)struct buffer_head *gd_bh =NULL ;struct ocfs2_dinode *main_bm;struct ocfs2_group_desc *gd =NULL ; -struct ocfs2_trim_fs_info info, *pinfo =NULL ; start = range->start >> osb->s_clustersize_bits; len = range->len >> osb->s_clustersize_bits;@@ -7552,6 +7552,9 @@ int ocfs2_trim_fs(struct super_block *sb, struct fstrim_range *range)if (minlen >= osb->bitmap_cpg || range->len < sb->s_blocksize)return -EINVAL ; +trace_ocfs2_trim_mainbm (start, len, minlen); + +next_group: main_bm_inode =ocfs2_get_system_file_inode (osb,GLOBAL_BITMAP_SYSTEM_INODE ,OCFS2_INVALID_SLOT );@@ -7570,64 +7573,34 @@ int ocfs2_trim_fs(struct super_block *sb, struct fstrim_range *range) } main_bm = (struct ocfs2_dinode *)main_bm_bh->b_data; -if (start >=le32_to_cpu (main_bm->i_clusters)) { - ret = -EINVAL ; - goto out_unlock; - } - - len = range->len >> osb->s_clustersize_bits; -if (start + len >le32_to_cpu (main_bm->i_clusters)) - len =le32_to_cpu (main_bm->i_clusters) - start; - -trace_ocfs2_trim_fs (start, len, minlen); - -ocfs2_trim_fs_lock_res_init (osb); - ret =ocfs2_trim_fs_lock (osb,NULL ,1 ); -if (ret <0 ) { -if (ret != -EAGAIN ) { -mlog_errno (ret); -ocfs2_trim_fs_lock_res_uninit (osb); +/* +* Do some check before trim the first group. +*/ +if (!group) { +if (start >=le32_to_cpu (main_bm->i_clusters)) { + ret = -EINVAL ; goto out_unlock; } -mlog (ML_NOTICE ,"Wait for trim on device (%s) to " -"finish, which is running from another node.\n" , - osb->dev_str); - ret =ocfs2_trim_fs_lock (osb, &info,0 ); -if (ret <0 ) { -mlog_errno (ret); -ocfs2_trim_fs_lock_res_uninit (osb); - goto out_unlock; - } +if (start + len >le32_to_cpu (main_bm->i_clusters)) + len =le32_to_cpu (main_bm->i_clusters) - start; -if (info.tf_valid && info.tf_success && - info.tf_start == start && info.tf_len == len && - info.tf_minlen == minlen) { -/* Avoid sending duplicated trim to a shared device */ -mlog (ML_NOTICE ,"The same trim on device (%s) was " -"just done from node (%u), return.\n" , - osb->dev_str, info.tf_nodenum); - range->len = info.tf_trimlen; - goto out_trimunlock; - } +/* +* Determine first and last group to examine based on +* start and len +*/ + first_group =ocfs2_which_cluster_group (main_bm_inode, start); +if (first_group == osb->first_cluster_group_blkno) + first_bit = start; +else + first_bit = start -ocfs2_blocks_to_clusters (sb, + first_group); + last_group =ocfs2_which_cluster_group (main_bm_inode, + start + len -1 ); + group = first_group; } - info.tf_nodenum = osb->node_num; - info.tf_start = start; - info.tf_len = len; - info.tf_minlen = minlen; - -/* Determine first and last group to examine based on start and len */ - first_group =ocfs2_which_cluster_group (main_bm_inode, start); -if (first_group == osb->first_cluster_group_blkno) - first_bit = start; -else - first_bit = start -ocfs2_blocks_to_clusters (sb, first_group); - last_group =ocfs2_which_cluster_group (main_bm_inode, start + len -1 ); - last_bit = osb->bitmap_cpg; - - trimmed =0 ; -for (group = first_group; group <= last_group;) { + do { if (first_bit + len >= osb->bitmap_cpg) last_bit = osb->bitmap_cpg; else@@ -7659,21 +7632,81 @@ int ocfs2_trim_fs(struct super_block *sb, struct fstrim_range *range) group =ocfs2_clusters_to_blocks (sb, osb->bitmap_cpg); else group += ocfs2_clusters_to_blocks(sb, osb->bitmap_cpg); - } - range->len = trimmed * sb->s_blocksize; + }while (0 ); - info.tf_trimlen = range->len; - info.tf_success = (ret ?0 :1 ); - pinfo = &info; -out_trimunlock: -ocfs2_trim_fs_unlock (osb, pinfo); -ocfs2_trim_fs_lock_res_uninit (osb); out_unlock:ocfs2_inode_unlock (main_bm_inode,0 );brelse (main_bm_bh); + main_bm_bh =NULL ; out_mutex:inode_unlock (main_bm_inode);iput (main_bm_inode); + +/* +* If all the groups trim are not done or failed, but we should release +* main_bm related locks for avoiding the current IO starve, then go to +* trim the next group +*/ +if (ret >=0 && group <= last_group) + goto next_group; out: + range->len = trimmed * sb->s_blocksize; +return ret; +} + +int ocfs2_trim_fs (struct super_block *sb,struct fstrim_range *range) +{ +int ret; +struct ocfs2_super *osb =OCFS2_SB (sb); +struct ocfs2_trim_fs_info info, *pinfo =NULL ; + +ocfs2_trim_fs_lock_res_init (osb); + +trace_ocfs2_trim_fs (range->start, range->len, range->minlen); + + ret =ocfs2_trim_fs_lock (osb,NULL ,1 ); +if (ret <0 ) { +if (ret != -EAGAIN ) { +mlog_errno (ret); +ocfs2_trim_fs_lock_res_uninit (osb); +return ret; + } + +mlog (ML_NOTICE ,"Wait for trim on device (%s) to " +"finish, which is running from another node.\n" , + osb->dev_str); + ret =ocfs2_trim_fs_lock (osb, &info,0 ); +if (ret <0 ) { +mlog_errno (ret); +ocfs2_trim_fs_lock_res_uninit (osb); +return ret; + } + +if (info.tf_valid && info.tf_success && + info.tf_start == range->start && + info.tf_len == range->len && + info.tf_minlen == range->minlen) { +/* Avoid sending duplicated trim to a shared device */ +mlog (ML_NOTICE ,"The same trim on device (%s) was " +"just done from node (%u), return.\n" , + osb->dev_str, info.tf_nodenum); + range->len = info.tf_trimlen; + goto out; + } + } + + info.tf_nodenum = osb->node_num; + info.tf_start = range->start; + info.tf_len = range->len; + info.tf_minlen = range->minlen; + + ret =ocfs2_trim_mainbm (sb, range); + + info.tf_trimlen = range->len; + info.tf_success = (ret <0 ?0 :1 ); + pinfo = &info; +out: +ocfs2_trim_fs_unlock (osb, pinfo); +ocfs2_trim_fs_lock_res_uninit (osb);return ret; }fs/ocfs2/dlmglue.c +5
-0@@ -686,6 +686,9 @@ void ocfs2_trim_fs_lock_res_init(struct ocfs2_super *osb) {struct ocfs2_lock_res *lockres = &osb->osb_trim_fs_lockres; +/* Only one trimfs thread are allowed to work at the same time. */ +mutex_lock (&osb->obs_trim_fs_mutex); +ocfs2_lock_res_init_once (lockres);ocfs2_build_lock_name (OCFS2_LOCK_TYPE_TRIM_FS ,0 ,0 , lockres->l_name); ocfs2_lock_res_init_common(osb, lockres, OCFS2_LOCK_TYPE_TRIM_FS,@@ -698,6 +701,8 @@ void ocfs2_trim_fs_lock_res_uninit(struct ocfs2_super *osb)ocfs2_simple_drop_lockres (osb, lockres);ocfs2_lock_res_free (lockres); + +mutex_unlock (&osb->obs_trim_fs_mutex); } static void ocfs2_orphan_scan_lock_res_init(struct ocfs2_lock_res *res,fs/ocfs2/ocfs2.h +1
-0@@ -407,6 +407,7 @@ struct ocfs2_superstruct ocfs2_lock_res osb_rename_lockres;struct ocfs2_lock_res osb_nfs_sync_lockres;struct ocfs2_lock_res osb_trim_fs_lockres; +struct mutex obs_trim_fs_mutex;struct ocfs2_dlm_debug *osb_dlm_debug;struct dentry *osb_debug_root;fs/ocfs2/ocfs2_trace.h +2
-0@@ -712,6 +712,8 @@ TRACE_EVENT(ocfs2_trim_extent,DEFINE_OCFS2_ULL_UINT_UINT_UINT_EVENT (ocfs2_trim_group); +DEFINE_OCFS2_ULL_ULL_ULL_EVENT (ocfs2_trim_mainbm); +DEFINE_OCFS2_ULL_ULL_ULL_EVENT (ocfs2_trim_fs);/* End of trace events for fs/ocfs2/alloc.c. */ fs/ocfs2/super.c +2
-0@@ -1847,6 +1847,8 @@ static int ocfs2_mount_volume(struct super_block *sb)if (ocfs2_is_hard_readonly (osb)) goto leave; +mutex_init (&osb->obs_trim_fs_mutex); + status =ocfs2_dlm_init (osb);if (status <0 ) {mlog_errno (status);
Hi Gang,
Most parts of this patch look sane to me, just a tiny question...
Quoted 239 lines · Gang He
The user reported this problem, the upper application IO was timeout when fstrim was running on this ocfs2 partition. the application monitoring resource agent considered that this application did not work, then this node was fenced by the cluster brain (e.g. pacemaker).
The root cause is that fstrim thread always holdsmain_bmmeta-file related locks until all the cluster groups are trimmed. This patch will make fstrim thread releasemain_bmmeta-file related locks when each cluster group is trimmed, this will let the current application IO has a chance to claim the clusters frommain_bmmeta-file.Signed-off-by Gang He <[email protected]>--- fs/ocfs2/alloc.c | 159 +++++++++++++++++++++++++---------------- fs/ocfs2/dlmglue.c | 5 ++ fs/ocfs2/ocfs2.h | 1 + fs/ocfs2/ocfs2_trace.h | 2 + fs/ocfs2/super.c | 2 + 5 files changed, 106 insertions(+), 63 deletions(-)fs/ocfs2/alloc.c +96
-63@@ -7532,10 +7532,11 @@ static int ocfs2_trim_group(struct super_block *sb,return count; } -int ocfs2_trim_fs (struct super_block *sb,struct fstrim_range *range) +static +int ocfs2_trim_mainbm (struct super_block *sb,struct fstrim_range *range) {struct ocfs2_super *osb =OCFS2_SB (sb); -u64 start, len, trimmed, first_group, last_group, group; +u64 start, len, trimmed =0 , first_group, last_group =0 , group =0 ;int ret, cnt;u32 first_bit, last_bit, minlen;struct buffer_head *main_bm_bh =NULL ;@@ -7543,7 +7544,6 @@ int ocfs2_trim_fs(struct super_block *sb, struct fstrim_range *range)struct buffer_head *gd_bh =NULL ;struct ocfs2_dinode *main_bm;struct ocfs2_group_desc *gd =NULL ; -struct ocfs2_trim_fs_info info, *pinfo =NULL ; start = range->start >> osb->s_clustersize_bits; len = range->len >> osb->s_clustersize_bits;@@ -7552,6 +7552,9 @@ int ocfs2_trim_fs(struct super_block *sb, struct fstrim_range *range)if (minlen >= osb->bitmap_cpg || range->len < sb->s_blocksize)return -EINVAL ; +trace_ocfs2_trim_mainbm (start, len, minlen); + +next_group: main_bm_inode =ocfs2_get_system_file_inode (osb,GLOBAL_BITMAP_SYSTEM_INODE ,OCFS2_INVALID_SLOT );@@ -7570,64 +7573,34 @@ int ocfs2_trim_fs(struct super_block *sb, struct fstrim_range *range) } main_bm = (struct ocfs2_dinode *)main_bm_bh->b_data; -if (start >=le32_to_cpu (main_bm->i_clusters)) { - ret = -EINVAL ; - goto out_unlock; - } - - len = range->len >> osb->s_clustersize_bits; -if (start + len >le32_to_cpu (main_bm->i_clusters)) - len =le32_to_cpu (main_bm->i_clusters) - start; - -trace_ocfs2_trim_fs (start, len, minlen); - -ocfs2_trim_fs_lock_res_init (osb); - ret =ocfs2_trim_fs_lock (osb,NULL ,1 ); -if (ret <0 ) { -if (ret != -EAGAIN ) { -mlog_errno (ret); -ocfs2_trim_fs_lock_res_uninit (osb); +/* +* Do some check before trim the first group. +*/ +if (!group) { +if (start >=le32_to_cpu (main_bm->i_clusters)) { + ret = -EINVAL ; goto out_unlock; } -mlog (ML_NOTICE ,"Wait for trim on device (%s) to " -"finish, which is running from another node.\n" , - osb->dev_str); - ret =ocfs2_trim_fs_lock (osb, &info,0 ); -if (ret <0 ) { -mlog_errno (ret); -ocfs2_trim_fs_lock_res_uninit (osb); - goto out_unlock; - } +if (start + len >le32_to_cpu (main_bm->i_clusters)) + len =le32_to_cpu (main_bm->i_clusters) - start; -if (info.tf_valid && info.tf_success && - info.tf_start == start && info.tf_len == len && - info.tf_minlen == minlen) { -/* Avoid sending duplicated trim to a shared device */ -mlog (ML_NOTICE ,"The same trim on device (%s) was " -"just done from node (%u), return.\n" , - osb->dev_str, info.tf_nodenum); - range->len = info.tf_trimlen; - goto out_trimunlock; - } +/* +* Determine first and last group to examine based on +* start and len +*/ + first_group =ocfs2_which_cluster_group (main_bm_inode, start); +if (first_group == osb->first_cluster_group_blkno) + first_bit = start; +else + first_bit = start -ocfs2_blocks_to_clusters (sb, + first_group); + last_group =ocfs2_which_cluster_group (main_bm_inode, + start + len -1 ); + group = first_group; } - info.tf_nodenum = osb->node_num; - info.tf_start = start; - info.tf_len = len; - info.tf_minlen = minlen; - -/* Determine first and last group to examine based on start and len */ - first_group =ocfs2_which_cluster_group (main_bm_inode, start); -if (first_group == osb->first_cluster_group_blkno) - first_bit = start; -else - first_bit = start -ocfs2_blocks_to_clusters (sb, first_group); - last_group =ocfs2_which_cluster_group (main_bm_inode, start + len -1 ); - last_bit = osb->bitmap_cpg; - - trimmed =0 ; -for (group = first_group; group <= last_group;) { + do { if (first_bit + len >= osb->bitmap_cpg) last_bit = osb->bitmap_cpg; else@@ -7659,21 +7632,81 @@ int ocfs2_trim_fs(struct super_block *sb, struct fstrim_range *range) group =ocfs2_clusters_to_blocks (sb, osb->bitmap_cpg); else group += ocfs2_clusters_to_blocks(sb, osb->bitmap_cpg); - } - range->len = trimmed * sb->s_blocksize; + }while (0 ); - info.tf_trimlen = range->len; - info.tf_success = (ret ?0 :1 ); - pinfo = &info; -out_trimunlock: -ocfs2_trim_fs_unlock (osb, pinfo); -ocfs2_trim_fs_lock_res_uninit (osb); out_unlock:ocfs2_inode_unlock (main_bm_inode,0 );brelse (main_bm_bh); + main_bm_bh =NULL ; out_mutex:inode_unlock (main_bm_inode);iput (main_bm_inode); + +/* +* If all the groups trim are not done or failed, but we should release +* main_bm related locks for avoiding the current IO starve, then go to +* trim the next group +*/ +if (ret >=0 && group <= last_group) + goto next_group; out: + range->len = trimmed * sb->s_blocksize; +return ret; +} + +int ocfs2_trim_fs (struct super_block *sb,struct fstrim_range *range) +{ +int ret; +struct ocfs2_super *osb =OCFS2_SB (sb); +struct ocfs2_trim_fs_info info, *pinfo =NULL ; + +ocfs2_trim_fs_lock_res_init (osb); + +trace_ocfs2_trim_fs (range->start, range->len, range->minlen); + + ret =ocfs2_trim_fs_lock (osb,NULL ,1 ); +if (ret <0 ) { +if (ret != -EAGAIN ) { +mlog_errno (ret); +ocfs2_trim_fs_lock_res_uninit (osb); +return ret; + } + +mlog (ML_NOTICE ,"Wait for trim on device (%s) to " +"finish, which is running from another node.\n" , + osb->dev_str); + ret =ocfs2_trim_fs_lock (osb, &info,0 ); +if (ret <0 ) { +mlog_errno (ret); +ocfs2_trim_fs_lock_res_uninit (osb); +return ret; + } + +if (info.tf_valid && info.tf_success && + info.tf_start == range->start && + info.tf_len == range->len && + info.tf_minlen == range->minlen) { +/* Avoid sending duplicated trim to a shared device */ +mlog (ML_NOTICE ,"The same trim on device (%s) was " +"just done from node (%u), return.\n" , + osb->dev_str, info.tf_nodenum); + range->len = info.tf_trimlen; + goto out; + } + } + + info.tf_nodenum = osb->node_num; + info.tf_start = range->start; + info.tf_len = range->len; + info.tf_minlen = range->minlen; + + ret =ocfs2_trim_mainbm (sb, range); + + info.tf_trimlen = range->len; + info.tf_success = (ret <0 ?0 :1 ); + pinfo = &info; +out: +ocfs2_trim_fs_unlock (osb, pinfo); +ocfs2_trim_fs_lock_res_uninit (osb);return ret; }fs/ocfs2/dlmglue.c +0
-0
fs/ocfs2/dlmglue.c void ocfs2_trim_fs_lock_res_init(struct ocfs2_super *osb) · line 691
Quoted 6 lines
fs/ocfs2/dlmglue.c +3
-0@@ -686,6 +686,9 @@ void ocfs2_trim_fs_lock_res_init(struct ocfs2_super *osb) {struct ocfs2_lock_res *lockres = &osb->osb_trim_fs_lockres; +/* Only one trimfs thread are allowed to work at the same time. */ +mutex_lock (&osb->obs_trim_fs_mutex); +
Cluster lock of fstrim have a trylock behavior, will it be better if we trylock here?
Thanks,
Changwei
Quoted 51 lines
fs/ocfs2/dlmglue.c +2
-0ocfs2_lock_res_init_once (lockres);ocfs2_build_lock_name (OCFS2_LOCK_TYPE_TRIM_FS ,0 ,0 , lockres->l_name); ocfs2_lock_res_init_common(osb, lockres, OCFS2_LOCK_TYPE_TRIM_FS,@@ -698,6 +701,8 @@ void ocfs2_trim_fs_lock_res_uninit(struct ocfs2_super *osb)ocfs2_simple_drop_lockres (osb, lockres);ocfs2_lock_res_free (lockres); + +mutex_unlock (&osb->obs_trim_fs_mutex); } static void ocfs2_orphan_scan_lock_res_init(struct ocfs2_lock_res *res,fs/ocfs2/ocfs2.h +1
-0@@ -407,6 +407,7 @@ struct ocfs2_superstruct ocfs2_lock_res osb_rename_lockres;struct ocfs2_lock_res osb_nfs_sync_lockres;struct ocfs2_lock_res osb_trim_fs_lockres; +struct mutex obs_trim_fs_mutex;struct ocfs2_dlm_debug *osb_dlm_debug;struct dentry *osb_debug_root;fs/ocfs2/ocfs2_trace.h +2
-0@@ -712,6 +712,8 @@ TRACE_EVENT(ocfs2_trim_extent,DEFINE_OCFS2_ULL_UINT_UINT_UINT_EVENT (ocfs2_trim_group); +DEFINE_OCFS2_ULL_ULL_ULL_EVENT (ocfs2_trim_mainbm); +DEFINE_OCFS2_ULL_ULL_ULL_EVENT (ocfs2_trim_fs);/* End of trace events for fs/ocfs2/alloc.c. */ fs/ocfs2/super.c +2
-0@@ -1847,6 +1847,8 @@ static int ocfs2_mount_volume(struct super_block *sb)if (ocfs2_is_hard_readonly (osb)) goto leave; +mutex_init (&osb->obs_trim_fs_mutex); + status =ocfs2_dlm_init (osb);if (status <0 ) {mlog_errno (status);
Hello Changewei,
Quoted 1 line
Quoted 1 line
Quoted 1 line
On 2019/1/15 at 11:50, in message
<63ADC13FD55D6546B7DECE290D39E3730127825EFD@H3CMLB12-EX.srv.huawei-3com.com>,
Quoted 251 lines · Changwei Ge
Hi Gang,
Most parts of this patch look sane to me, just a tiny question...
fs/ocfs2/alloc.c int ocfs2_trim_fs(struct super_block *sb, struct
Quoted 40 lines · Gang He
The user reported this problem, the upper application IO was timeout when fstrim was running on this ocfs2 partition. the application monitoring resource agent considered that this application did not work, then this node was fenced by the cluster brain (e.g. pacemaker).
The root cause is that fstrim thread always holdsmain_bmmeta-file related locks until all the cluster groups are trimmed. This patch will make fstrim thread releasemain_bmmeta-file related locks when each cluster group is trimmed, this will let the current application IO has a chance to claim the clusters frommain_bmmeta-file.Signed-off-by Gang He <[email protected]>--- fs/ocfs2/alloc.c | 159 +++++++++++++++++++++++++---------------- fs/ocfs2/dlmglue.c | 5 ++ fs/ocfs2/ocfs2.h | 1 + fs/ocfs2/ocfs2_trace.h | 2 + fs/ocfs2/super.c | 2 + 5 files changed, 106 insertions(+), 63 deletions(-)fs/ocfs2/alloc.c +3
-2@@ -7532,10 +7532,11 @@ static int ocfs2_trim_group(struct super_block *sb,return count; } -int ocfs2_trim_fs (struct super_block *sb,struct fstrim_range *range) +static +int ocfs2_trim_mainbm (struct super_block *sb,struct fstrim_range *range) {struct ocfs2_super *osb =OCFS2_SB (sb); -u64 start, len, trimmed, first_group, last_group, group; +u64 start, len, trimmed =0 , first_group, last_group =0 , group =0 ;int ret, cnt;u32 first_bit, last_bit, minlen;struct buffer_head *main_bm_bh =NULL ;@@ -7543,7 +7544,6 @@ int ocfs2_trim_fs(struct super_block *sb, struct
fstrim_range*range)fs/ocfs2/alloc.c int ocfs2_trim_fs(struct super_block *sb, struct
Quoted 8 lines
fs/ocfs2/alloc.c +0
-1struct buffer_head *gd_bh =NULL ;struct ocfs2_dinode *main_bm;struct ocfs2_group_desc *gd =NULL ; -struct ocfs2_trim_fs_info info, *pinfo =NULL ; start = range->start >> osb->s_clustersize_bits; len = range->len >> osb->s_clustersize_bits;@@ -7552,6 +7552,9 @@ int ocfs2_trim_fs(struct super_block *sb, struct
fstrim_range*range)fs/ocfs2/alloc.c int ocfs2_trim_fs(struct super_block *sb, struct
Quoted 10 lines
fs/ocfs2/alloc.c +3
-0if (minlen >= osb->bitmap_cpg || range->len < sb->s_blocksize)return -EINVAL ; +trace_ocfs2_trim_mainbm (start, len, minlen); + +next_group: main_bm_inode =ocfs2_get_system_file_inode (osb,GLOBAL_BITMAP_SYSTEM_INODE ,OCFS2_INVALID_SLOT );@@ -7570,64 +7573,34 @@ int ocfs2_trim_fs(struct super_block *sb, struct
fstrim_range*range)fs/ocfs2/alloc.c int ocfs2_trim_fs(struct super_block *sb, struct
Quoted 87 lines
fs/ocfs2/alloc.c +22
-52} main_bm = (struct ocfs2_dinode *)main_bm_bh->b_data; -if (start >=le32_to_cpu (main_bm->i_clusters)) { - ret = -EINVAL ; - goto out_unlock; - } - - len = range->len >> osb->s_clustersize_bits; -if (start + len >le32_to_cpu (main_bm->i_clusters)) - len =le32_to_cpu (main_bm->i_clusters) - start; - -trace_ocfs2_trim_fs (start, len, minlen); - -ocfs2_trim_fs_lock_res_init (osb); - ret =ocfs2_trim_fs_lock (osb,NULL ,1 ); -if (ret <0 ) { -if (ret != -EAGAIN ) { -mlog_errno (ret); -ocfs2_trim_fs_lock_res_uninit (osb); +/* +* Do some check before trim the first group. +*/ +if (!group) { +if (start >=le32_to_cpu (main_bm->i_clusters)) { + ret = -EINVAL ; goto out_unlock; } -mlog (ML_NOTICE ,"Wait for trim on device (%s) to " -"finish, which is running from another node.\n" , - osb->dev_str); - ret =ocfs2_trim_fs_lock (osb, &info,0 ); -if (ret <0 ) { -mlog_errno (ret); -ocfs2_trim_fs_lock_res_uninit (osb); - goto out_unlock; - } +if (start + len >le32_to_cpu (main_bm->i_clusters)) + len =le32_to_cpu (main_bm->i_clusters) - start; -if (info.tf_valid && info.tf_success && - info.tf_start == start && info.tf_len == len && - info.tf_minlen == minlen) { -/* Avoid sending duplicated trim to a shared device */ -mlog (ML_NOTICE ,"The same trim on device (%s) was " -"just done from node (%u), return.\n" , - osb->dev_str, info.tf_nodenum); - range->len = info.tf_trimlen; - goto out_trimunlock; - } +/* +* Determine first and last group to examine based on +* start and len +*/ + first_group =ocfs2_which_cluster_group (main_bm_inode, start); +if (first_group == osb->first_cluster_group_blkno) + first_bit = start; +else + first_bit = start -ocfs2_blocks_to_clusters (sb, + first_group); + last_group =ocfs2_which_cluster_group (main_bm_inode, + start + len -1 ); + group = first_group; } - info.tf_nodenum = osb->node_num; - info.tf_start = start; - info.tf_len = len; - info.tf_minlen = minlen; - -/* Determine first and last group to examine based on start and len */ - first_group =ocfs2_which_cluster_group (main_bm_inode, start); -if (first_group == osb->first_cluster_group_blkno) - first_bit = start; -else - first_bit = start -ocfs2_blocks_to_clusters (sb, first_group); - last_group =ocfs2_which_cluster_group (main_bm_inode, start + len -1 ); - last_bit = osb->bitmap_cpg; - - trimmed =0 ; -for (group = first_group; group <= last_group;) { + do { if (first_bit + len >= osb->bitmap_cpg) last_bit = osb->bitmap_cpg; else@@ -7659,21 +7632,81 @@ int ocfs2_trim_fs(struct super_block *sb, struct
fstrim_range*range)Quoted 97 lines
fs/ocfs2/alloc.c +68
-8group =ocfs2_clusters_to_blocks (sb, osb->bitmap_cpg); else group += ocfs2_clusters_to_blocks(sb, osb->bitmap_cpg); - } - range->len = trimmed * sb->s_blocksize; + }while (0 ); - info.tf_trimlen = range->len; - info.tf_success = (ret ?0 :1 ); - pinfo = &info; -out_trimunlock: -ocfs2_trim_fs_unlock (osb, pinfo); -ocfs2_trim_fs_lock_res_uninit (osb); out_unlock:ocfs2_inode_unlock (main_bm_inode,0 );brelse (main_bm_bh); + main_bm_bh =NULL ; out_mutex:inode_unlock (main_bm_inode);iput (main_bm_inode); + +/* +* If all the groups trim are not done or failed, but we should release +* main_bm related locks for avoiding the current IO starve, then go to +* trim the next group +*/ +if (ret >=0 && group <= last_group) + goto next_group; out: + range->len = trimmed * sb->s_blocksize; +return ret; +} + +int ocfs2_trim_fs (struct super_block *sb,struct fstrim_range *range) +{ +int ret; +struct ocfs2_super *osb =OCFS2_SB (sb); +struct ocfs2_trim_fs_info info, *pinfo =NULL ; + +ocfs2_trim_fs_lock_res_init (osb); + +trace_ocfs2_trim_fs (range->start, range->len, range->minlen); + + ret =ocfs2_trim_fs_lock (osb,NULL ,1 ); +if (ret <0 ) { +if (ret != -EAGAIN ) { +mlog_errno (ret); +ocfs2_trim_fs_lock_res_uninit (osb); +return ret; + } + +mlog (ML_NOTICE ,"Wait for trim on device (%s) to " +"finish, which is running from another node.\n" , + osb->dev_str); + ret =ocfs2_trim_fs_lock (osb, &info,0 ); +if (ret <0 ) { +mlog_errno (ret); +ocfs2_trim_fs_lock_res_uninit (osb); +return ret; + } + +if (info.tf_valid && info.tf_success && + info.tf_start == range->start && + info.tf_len == range->len && + info.tf_minlen == range->minlen) { +/* Avoid sending duplicated trim to a shared device */ +mlog (ML_NOTICE ,"The same trim on device (%s) was " +"just done from node (%u), return.\n" , + osb->dev_str, info.tf_nodenum); + range->len = info.tf_trimlen; + goto out; + } + } + + info.tf_nodenum = osb->node_num; + info.tf_start = range->start; + info.tf_len = range->len; + info.tf_minlen = range->minlen; + + ret =ocfs2_trim_mainbm (sb, range); + + info.tf_trimlen = range->len; + info.tf_success = (ret <0 ?0 :1 ); + pinfo = &info; +out: +ocfs2_trim_fs_unlock (osb, pinfo); +ocfs2_trim_fs_lock_res_uninit (osb);return ret; }fs/ocfs2/dlmglue.c +0
-0@@ -686,6 +686,9 @@ void ocfs2_trim_fs_lock_res_init(struct ocfs2_super *osb) {struct ocfs2_lock_res *lockres = &osb->osb_trim_fs_lockres;
Quoted 6 lines
fs/ocfs2/dlmglue.c void ocfs2_trim_fs_lock_res_init(struct ocfs2_super *osb) · line 691
Quoted 3 lines
fs/ocfs2/dlmglue.c +3
-0+/* Only one trimfs thread are allowed to work at the same time. */ +mutex_lock (&osb->obs_trim_fs_mutex); +Cluster lock of fstrim have a trylock behavior, will it be better if we trylock here?
Here, I prefer to just serialize fstrim threads on the local node to simplify the code logic, maybe the user want to do like that, although this behavior is not recommended. You know, on one node, ideally, the user should call fstrim command once regularly. If he calls fstrim command more times in a very short time,
the code will not make each fstrim command return to failure, just do the fstrim task one by one.
Thanks
Gang
Quoted 56 lines
Thanks,
Changweifs/ocfs2/dlmglue.c void ocfs2_trim_fs_lock_res_uninit(struct ocfs2_super
Quoted 4 lines
fs/ocfs2/dlmglue.c +0
-0ocfs2_lock_res_init_once (lockres);ocfs2_build_lock_name (OCFS2_LOCK_TYPE_TRIM_FS ,0 ,0 , lockres->l_name); ocfs2_lock_res_init_common(osb, lockres, OCFS2_LOCK_TYPE_TRIM_FS,@@ -698,6 +701,8 @@ void ocfs2_trim_fs_lock_res_uninit(struct ocfs2_super*osb)
Quoted 47 lines
fs/ocfs2/dlmglue.c +2
-0ocfs2_simple_drop_lockres (osb, lockres);ocfs2_lock_res_free (lockres); + +mutex_unlock (&osb->obs_trim_fs_mutex); } static void ocfs2_orphan_scan_lock_res_init(struct ocfs2_lock_res *res,fs/ocfs2/ocfs2.h +1
-0@@ -407,6 +407,7 @@ struct ocfs2_superstruct ocfs2_lock_res osb_rename_lockres;struct ocfs2_lock_res osb_nfs_sync_lockres;struct ocfs2_lock_res osb_trim_fs_lockres; +struct mutex obs_trim_fs_mutex;struct ocfs2_dlm_debug *osb_dlm_debug;struct dentry *osb_debug_root;fs/ocfs2/ocfs2_trace.h +2
-0@@ -712,6 +712,8 @@ TRACE_EVENT(ocfs2_trim_extent,DEFINE_OCFS2_ULL_UINT_UINT_UINT_EVENT (ocfs2_trim_group); +DEFINE_OCFS2_ULL_ULL_ULL_EVENT (ocfs2_trim_mainbm); +DEFINE_OCFS2_ULL_ULL_ULL_EVENT (ocfs2_trim_fs);/* End of trace events for fs/ocfs2/alloc.c. */ fs/ocfs2/super.c +2
-0@@ -1847,6 +1847,8 @@ static int ocfs2_mount_volume(struct super_block *sb)if (ocfs2_is_hard_readonly (osb)) goto leave; +mutex_init (&osb->obs_trim_fs_mutex); + status =ocfs2_dlm_init (osb);if (status <0 ) {mlog_errno (status);
Quoted 262 lines · Gang He
Hello Changewei,
Quoted 1 line
Quoted 1 line
Quoted 1 line
On 2019/1/15 at 11:50, in message
<63ADC13FD55D6546B7DECE290D39E3730127825EFD@H3CMLB12-EX.srv.huawei-3com.com>,
Quoted 251 lines · Changwei Ge
Hi Gang,
Most parts of this patch look sane to me, just a tiny question...
fs/ocfs2/alloc.c int ocfs2_trim_fs(struct super_block *sb, struct
Quoted 40 lines · Gang He
The user reported this problem, the upper application IO was timeout when fstrim was running on this ocfs2 partition. the application monitoring resource agent considered that this application did not work, then this node was fenced by the cluster brain (e.g. pacemaker).
The root cause is that fstrim thread always holdsmain_bmmeta-file related locks until all the cluster groups are trimmed. This patch will make fstrim thread releasemain_bmmeta-file related locks when each cluster group is trimmed, this will let the current application IO has a chance to claim the clusters frommain_bmmeta-file.Signed-off-by Gang He <[email protected]>--- fs/ocfs2/alloc.c | 159 +++++++++++++++++++++++++---------------- fs/ocfs2/dlmglue.c | 5 ++ fs/ocfs2/ocfs2.h | 1 + fs/ocfs2/ocfs2_trace.h | 2 + fs/ocfs2/super.c | 2 + 5 files changed, 106 insertions(+), 63 deletions(-)fs/ocfs2/alloc.c +3
-2@@ -7532,10 +7532,11 @@ static int ocfs2_trim_group(struct super_block *sb,return count; } -int ocfs2_trim_fs (struct super_block *sb,struct fstrim_range *range) +static +int ocfs2_trim_mainbm (struct super_block *sb,struct fstrim_range *range) {struct ocfs2_super *osb =OCFS2_SB (sb); -u64 start, len, trimmed, first_group, last_group, group; +u64 start, len, trimmed =0 , first_group, last_group =0 , group =0 ;int ret, cnt;u32 first_bit, last_bit, minlen;struct buffer_head *main_bm_bh =NULL ;@@ -7543,7 +7544,6 @@ int ocfs2_trim_fs(struct super_block *sb, struct
fstrim_range*range)fs/ocfs2/alloc.c int ocfs2_trim_fs(struct super_block *sb, struct
Quoted 8 lines
fs/ocfs2/alloc.c +0
-1struct buffer_head *gd_bh =NULL ;struct ocfs2_dinode *main_bm;struct ocfs2_group_desc *gd =NULL ; -struct ocfs2_trim_fs_info info, *pinfo =NULL ; start = range->start >> osb->s_clustersize_bits; len = range->len >> osb->s_clustersize_bits;@@ -7552,6 +7552,9 @@ int ocfs2_trim_fs(struct super_block *sb, struct
fstrim_range*range)fs/ocfs2/alloc.c int ocfs2_trim_fs(struct super_block *sb, struct
Quoted 10 lines
fs/ocfs2/alloc.c +3
-0if (minlen >= osb->bitmap_cpg || range->len < sb->s_blocksize)return -EINVAL ; +trace_ocfs2_trim_mainbm (start, len, minlen); + +next_group: main_bm_inode =ocfs2_get_system_file_inode (osb,GLOBAL_BITMAP_SYSTEM_INODE ,OCFS2_INVALID_SLOT );@@ -7570,64 +7573,34 @@ int ocfs2_trim_fs(struct super_block *sb, struct
fstrim_range*range)fs/ocfs2/alloc.c int ocfs2_trim_fs(struct super_block *sb, struct
Quoted 87 lines
fs/ocfs2/alloc.c +22
-52} main_bm = (struct ocfs2_dinode *)main_bm_bh->b_data; -if (start >=le32_to_cpu (main_bm->i_clusters)) { - ret = -EINVAL ; - goto out_unlock; - } - - len = range->len >> osb->s_clustersize_bits; -if (start + len >le32_to_cpu (main_bm->i_clusters)) - len =le32_to_cpu (main_bm->i_clusters) - start; - -trace_ocfs2_trim_fs (start, len, minlen); - -ocfs2_trim_fs_lock_res_init (osb); - ret =ocfs2_trim_fs_lock (osb,NULL ,1 ); -if (ret <0 ) { -if (ret != -EAGAIN ) { -mlog_errno (ret); -ocfs2_trim_fs_lock_res_uninit (osb); +/* +* Do some check before trim the first group. +*/ +if (!group) { +if (start >=le32_to_cpu (main_bm->i_clusters)) { + ret = -EINVAL ; goto out_unlock; } -mlog (ML_NOTICE ,"Wait for trim on device (%s) to " -"finish, which is running from another node.\n" , - osb->dev_str); - ret =ocfs2_trim_fs_lock (osb, &info,0 ); -if (ret <0 ) { -mlog_errno (ret); -ocfs2_trim_fs_lock_res_uninit (osb); - goto out_unlock; - } +if (start + len >le32_to_cpu (main_bm->i_clusters)) + len =le32_to_cpu (main_bm->i_clusters) - start; -if (info.tf_valid && info.tf_success && - info.tf_start == start && info.tf_len == len && - info.tf_minlen == minlen) { -/* Avoid sending duplicated trim to a shared device */ -mlog (ML_NOTICE ,"The same trim on device (%s) was " -"just done from node (%u), return.\n" , - osb->dev_str, info.tf_nodenum); - range->len = info.tf_trimlen; - goto out_trimunlock; - } +/* +* Determine first and last group to examine based on +* start and len +*/ + first_group =ocfs2_which_cluster_group (main_bm_inode, start); +if (first_group == osb->first_cluster_group_blkno) + first_bit = start; +else + first_bit = start -ocfs2_blocks_to_clusters (sb, + first_group); + last_group =ocfs2_which_cluster_group (main_bm_inode, + start + len -1 ); + group = first_group; } - info.tf_nodenum = osb->node_num; - info.tf_start = start; - info.tf_len = len; - info.tf_minlen = minlen; - -/* Determine first and last group to examine based on start and len */ - first_group =ocfs2_which_cluster_group (main_bm_inode, start); -if (first_group == osb->first_cluster_group_blkno) - first_bit = start; -else - first_bit = start -ocfs2_blocks_to_clusters (sb, first_group); - last_group =ocfs2_which_cluster_group (main_bm_inode, start + len -1 ); - last_bit = osb->bitmap_cpg; - - trimmed =0 ; -for (group = first_group; group <= last_group;) { + do { if (first_bit + len >= osb->bitmap_cpg) last_bit = osb->bitmap_cpg; else@@ -7659,21 +7632,81 @@ int ocfs2_trim_fs(struct super_block *sb, struct
fstrim_range*range)Quoted 97 lines
fs/ocfs2/alloc.c +68
-8group =ocfs2_clusters_to_blocks (sb, osb->bitmap_cpg); else group += ocfs2_clusters_to_blocks(sb, osb->bitmap_cpg); - } - range->len = trimmed * sb->s_blocksize; + }while (0 ); - info.tf_trimlen = range->len; - info.tf_success = (ret ?0 :1 ); - pinfo = &info; -out_trimunlock: -ocfs2_trim_fs_unlock (osb, pinfo); -ocfs2_trim_fs_lock_res_uninit (osb); out_unlock:ocfs2_inode_unlock (main_bm_inode,0 );brelse (main_bm_bh); + main_bm_bh =NULL ; out_mutex:inode_unlock (main_bm_inode);iput (main_bm_inode); + +/* +* If all the groups trim are not done or failed, but we should release +* main_bm related locks for avoiding the current IO starve, then go to +* trim the next group +*/ +if (ret >=0 && group <= last_group) + goto next_group; out: + range->len = trimmed * sb->s_blocksize; +return ret; +} + +int ocfs2_trim_fs (struct super_block *sb,struct fstrim_range *range) +{ +int ret; +struct ocfs2_super *osb =OCFS2_SB (sb); +struct ocfs2_trim_fs_info info, *pinfo =NULL ; + +ocfs2_trim_fs_lock_res_init (osb); + +trace_ocfs2_trim_fs (range->start, range->len, range->minlen); + + ret =ocfs2_trim_fs_lock (osb,NULL ,1 ); +if (ret <0 ) { +if (ret != -EAGAIN ) { +mlog_errno (ret); +ocfs2_trim_fs_lock_res_uninit (osb); +return ret; + } + +mlog (ML_NOTICE ,"Wait for trim on device (%s) to " +"finish, which is running from another node.\n" , + osb->dev_str); + ret =ocfs2_trim_fs_lock (osb, &info,0 ); +if (ret <0 ) { +mlog_errno (ret); +ocfs2_trim_fs_lock_res_uninit (osb); +return ret; + } + +if (info.tf_valid && info.tf_success && + info.tf_start == range->start && + info.tf_len == range->len && + info.tf_minlen == range->minlen) { +/* Avoid sending duplicated trim to a shared device */ +mlog (ML_NOTICE ,"The same trim on device (%s) was " +"just done from node (%u), return.\n" , + osb->dev_str, info.tf_nodenum); + range->len = info.tf_trimlen; + goto out; + } + } + + info.tf_nodenum = osb->node_num; + info.tf_start = range->start; + info.tf_len = range->len; + info.tf_minlen = range->minlen; + + ret =ocfs2_trim_mainbm (sb, range); + + info.tf_trimlen = range->len; + info.tf_success = (ret <0 ?0 :1 ); + pinfo = &info; +out: +ocfs2_trim_fs_unlock (osb, pinfo); +ocfs2_trim_fs_lock_res_uninit (osb);return ret; }fs/ocfs2/dlmglue.c +0
-0@@ -686,6 +686,9 @@ void ocfs2_trim_fs_lock_res_init(struct ocfs2_super *osb) {struct ocfs2_lock_res *lockres = &osb->osb_trim_fs_lockres;Quoted 6 lines
fs/ocfs2/dlmglue.c void ocfs2_trim_fs_lock_res_init(struct ocfs2_super *osb) · line 691
Quoted 3 lines
fs/ocfs2/dlmglue.c +3
-0+/* Only one trimfs thread are allowed to work at the same time. */ +mutex_lock (&osb->obs_trim_fs_mutex); +Cluster lock of fstrim have a trylock behavior, will it be better if we trylock here?
Quoted 5 lines
Here, I prefer to just serialize fstrim threads on the local node to simplify the code logic, maybe the user want to do like that, although this behavior is not recommended. You know, on one node, ideally, the user should call fstrim command once regularly. If he calls fstrim command more times in a very short time,
the code will not make each fstrim command return to failure, just do the fstrim task one by one.
I have a thought having nothing to do with your patch.
Do you think it's possible for us to implement *discard on unlink file* like ext4 does. So the application doesn't have to invoke fstrim periodically.
Thanks,
Changwei
Quoted 61 lines
Thanks
GangQuoted 56 lines
Thanks,
Changweifs/ocfs2/dlmglue.c void ocfs2_trim_fs_lock_res_uninit(struct ocfs2_super
Quoted 4 lines
fs/ocfs2/dlmglue.c +0
-0ocfs2_lock_res_init_once (lockres);ocfs2_build_lock_name (OCFS2_LOCK_TYPE_TRIM_FS ,0 ,0 , lockres->l_name); ocfs2_lock_res_init_common(osb, lockres, OCFS2_LOCK_TYPE_TRIM_FS,@@ -698,6 +701,8 @@ void ocfs2_trim_fs_lock_res_uninit(struct ocfs2_super*osb)
Quoted 47 lines
fs/ocfs2/dlmglue.c +2
-0ocfs2_simple_drop_lockres (osb, lockres);ocfs2_lock_res_free (lockres); + +mutex_unlock (&osb->obs_trim_fs_mutex); } static void ocfs2_orphan_scan_lock_res_init(struct ocfs2_lock_res *res,fs/ocfs2/ocfs2.h +1
-0@@ -407,6 +407,7 @@ struct ocfs2_superstruct ocfs2_lock_res osb_rename_lockres;struct ocfs2_lock_res osb_nfs_sync_lockres;struct ocfs2_lock_res osb_trim_fs_lockres; +struct mutex obs_trim_fs_mutex;struct ocfs2_dlm_debug *osb_dlm_debug;struct dentry *osb_debug_root;fs/ocfs2/ocfs2_trace.h +2
-0@@ -712,6 +712,8 @@ TRACE_EVENT(ocfs2_trim_extent,DEFINE_OCFS2_ULL_UINT_UINT_UINT_EVENT (ocfs2_trim_group); +DEFINE_OCFS2_ULL_ULL_ULL_EVENT (ocfs2_trim_mainbm); +DEFINE_OCFS2_ULL_ULL_ULL_EVENT (ocfs2_trim_fs);/* End of trace events for fs/ocfs2/alloc.c. */ fs/ocfs2/super.c +2
-0@@ -1847,6 +1847,8 @@ static int ocfs2_mount_volume(struct super_block *sb)if (ocfs2_is_hard_readonly (osb)) goto leave; +mutex_init (&osb->obs_trim_fs_mutex); + status =ocfs2_dlm_init (osb);if (status <0 ) {mlog_errno (status);
Hello ChangWei,
Quoted 1 line
Quoted 1 line
Quoted 1 line
On 2019/1/15 at 16:00, in message
<63ADC13FD55D6546B7DECE290D39E37301278265F3@H3CMLB12-EX.srv.huawei-3com.com>,
Quoted 269 lines · Changwei Ge
Quoted 256 lines · Gang He
Hello Changewei,
Quoted 1 line
Quoted 1 line
Quoted 1 line
On 2019/1/15 at 11:50, in message
<63ADC13FD55D6546B7DECE290D39E3730127825EFD@H3CMLB12-EX.srv.huawei-3com.com>,
Quoted 251 lines · Changwei Ge
Hi Gang,
Most parts of this patch look sane to me, just a tiny question...
fs/ocfs2/alloc.c int ocfs2_trim_fs(struct super_block *sb, struct
Quoted 40 lines · Gang He
The user reported this problem, the upper application IO was timeout when fstrim was running on this ocfs2 partition. the application monitoring resource agent considered that this application did not work, then this node was fenced by the cluster brain (e.g. pacemaker).
The root cause is that fstrim thread always holdsmain_bmmeta-file related locks until all the cluster groups are trimmed. This patch will make fstrim thread releasemain_bmmeta-file related locks when each cluster group is trimmed, this will let the current application IO has a chance to claim the clusters frommain_bmmeta-file.Signed-off-by Gang He <[email protected]>--- fs/ocfs2/alloc.c | 159 +++++++++++++++++++++++++---------------- fs/ocfs2/dlmglue.c | 5 ++ fs/ocfs2/ocfs2.h | 1 + fs/ocfs2/ocfs2_trace.h | 2 + fs/ocfs2/super.c | 2 + 5 files changed, 106 insertions(+), 63 deletions(-)fs/ocfs2/alloc.c +3
-2@@ -7532,10 +7532,11 @@ static int ocfs2_trim_group(struct super_block *sb,return count; } -int ocfs2_trim_fs (struct super_block *sb,struct fstrim_range *range) +static +int ocfs2_trim_mainbm (struct super_block *sb,struct fstrim_range *range) {struct ocfs2_super *osb =OCFS2_SB (sb); -u64 start, len, trimmed, first_group, last_group, group; +u64 start, len, trimmed =0 , first_group, last_group =0 , group =0 ;int ret, cnt;u32 first_bit, last_bit, minlen;struct buffer_head *main_bm_bh =NULL ;@@ -7543,7 +7544,6 @@ int ocfs2_trim_fs(struct super_block *sb, struct
fstrim_range*range)fs/ocfs2/alloc.c int ocfs2_trim_fs(struct super_block *sb, struct
Quoted 8 lines
fs/ocfs2/alloc.c +0
-1struct buffer_head *gd_bh =NULL ;struct ocfs2_dinode *main_bm;struct ocfs2_group_desc *gd =NULL ; -struct ocfs2_trim_fs_info info, *pinfo =NULL ; start = range->start >> osb->s_clustersize_bits; len = range->len >> osb->s_clustersize_bits;@@ -7552,6 +7552,9 @@ int ocfs2_trim_fs(struct super_block *sb, struct
fstrim_range*range)fs/ocfs2/alloc.c int ocfs2_trim_fs(struct super_block *sb, struct
Quoted 10 lines
fs/ocfs2/alloc.c +3
-0if (minlen >= osb->bitmap_cpg || range->len < sb->s_blocksize)return -EINVAL ; +trace_ocfs2_trim_mainbm (start, len, minlen); + +next_group: main_bm_inode =ocfs2_get_system_file_inode (osb,GLOBAL_BITMAP_SYSTEM_INODE ,OCFS2_INVALID_SLOT );@@ -7570,64 +7573,34 @@ int ocfs2_trim_fs(struct super_block *sb, struct
fstrim_range*range)fs/ocfs2/alloc.c int ocfs2_trim_fs(struct super_block *sb, struct
Quoted 87 lines
fs/ocfs2/alloc.c +22
-52} main_bm = (struct ocfs2_dinode *)main_bm_bh->b_data; -if (start >=le32_to_cpu (main_bm->i_clusters)) { - ret = -EINVAL ; - goto out_unlock; - } - - len = range->len >> osb->s_clustersize_bits; -if (start + len >le32_to_cpu (main_bm->i_clusters)) - len =le32_to_cpu (main_bm->i_clusters) - start; - -trace_ocfs2_trim_fs (start, len, minlen); - -ocfs2_trim_fs_lock_res_init (osb); - ret =ocfs2_trim_fs_lock (osb,NULL ,1 ); -if (ret <0 ) { -if (ret != -EAGAIN ) { -mlog_errno (ret); -ocfs2_trim_fs_lock_res_uninit (osb); +/* +* Do some check before trim the first group. +*/ +if (!group) { +if (start >=le32_to_cpu (main_bm->i_clusters)) { + ret = -EINVAL ; goto out_unlock; } -mlog (ML_NOTICE ,"Wait for trim on device (%s) to " -"finish, which is running from another node.\n" , - osb->dev_str); - ret =ocfs2_trim_fs_lock (osb, &info,0 ); -if (ret <0 ) { -mlog_errno (ret); -ocfs2_trim_fs_lock_res_uninit (osb); - goto out_unlock; - } +if (start + len >le32_to_cpu (main_bm->i_clusters)) + len =le32_to_cpu (main_bm->i_clusters) - start; -if (info.tf_valid && info.tf_success && - info.tf_start == start && info.tf_len == len && - info.tf_minlen == minlen) { -/* Avoid sending duplicated trim to a shared device */ -mlog (ML_NOTICE ,"The same trim on device (%s) was " -"just done from node (%u), return.\n" , - osb->dev_str, info.tf_nodenum); - range->len = info.tf_trimlen; - goto out_trimunlock; - } +/* +* Determine first and last group to examine based on +* start and len +*/ + first_group =ocfs2_which_cluster_group (main_bm_inode, start); +if (first_group == osb->first_cluster_group_blkno) + first_bit = start; +else + first_bit = start -ocfs2_blocks_to_clusters (sb, + first_group); + last_group =ocfs2_which_cluster_group (main_bm_inode, + start + len -1 ); + group = first_group; } - info.tf_nodenum = osb->node_num; - info.tf_start = start; - info.tf_len = len; - info.tf_minlen = minlen; - -/* Determine first and last group to examine based on start and len */ - first_group =ocfs2_which_cluster_group (main_bm_inode, start); -if (first_group == osb->first_cluster_group_blkno) - first_bit = start; -else - first_bit = start -ocfs2_blocks_to_clusters (sb, first_group); - last_group =ocfs2_which_cluster_group (main_bm_inode, start + len -1 ); - last_bit = osb->bitmap_cpg; - - trimmed =0 ; -for (group = first_group; group <= last_group;) { + do { if (first_bit + len >= osb->bitmap_cpg) last_bit = osb->bitmap_cpg; else@@ -7659,21 +7632,81 @@ int ocfs2_trim_fs(struct super_block *sb, struct
fstrim_range*range)Quoted 97 lines
fs/ocfs2/alloc.c +68
-8group =ocfs2_clusters_to_blocks (sb, osb->bitmap_cpg); else group += ocfs2_clusters_to_blocks(sb, osb->bitmap_cpg); - } - range->len = trimmed * sb->s_blocksize; + }while (0 ); - info.tf_trimlen = range->len; - info.tf_success = (ret ?0 :1 ); - pinfo = &info; -out_trimunlock: -ocfs2_trim_fs_unlock (osb, pinfo); -ocfs2_trim_fs_lock_res_uninit (osb); out_unlock:ocfs2_inode_unlock (main_bm_inode,0 );brelse (main_bm_bh); + main_bm_bh =NULL ; out_mutex:inode_unlock (main_bm_inode);iput (main_bm_inode); + +/* +* If all the groups trim are not done or failed, but we should release +* main_bm related locks for avoiding the current IO starve, then go to +* trim the next group +*/ +if (ret >=0 && group <= last_group) + goto next_group; out: + range->len = trimmed * sb->s_blocksize; +return ret; +} + +int ocfs2_trim_fs (struct super_block *sb,struct fstrim_range *range) +{ +int ret; +struct ocfs2_super *osb =OCFS2_SB (sb); +struct ocfs2_trim_fs_info info, *pinfo =NULL ; + +ocfs2_trim_fs_lock_res_init (osb); + +trace_ocfs2_trim_fs (range->start, range->len, range->minlen); + + ret =ocfs2_trim_fs_lock (osb,NULL ,1 ); +if (ret <0 ) { +if (ret != -EAGAIN ) { +mlog_errno (ret); +ocfs2_trim_fs_lock_res_uninit (osb); +return ret; + } + +mlog (ML_NOTICE ,"Wait for trim on device (%s) to " +"finish, which is running from another node.\n" , + osb->dev_str); + ret =ocfs2_trim_fs_lock (osb, &info,0 ); +if (ret <0 ) { +mlog_errno (ret); +ocfs2_trim_fs_lock_res_uninit (osb); +return ret; + } + +if (info.tf_valid && info.tf_success && + info.tf_start == range->start && + info.tf_len == range->len && + info.tf_minlen == range->minlen) { +/* Avoid sending duplicated trim to a shared device */ +mlog (ML_NOTICE ,"The same trim on device (%s) was " +"just done from node (%u), return.\n" , + osb->dev_str, info.tf_nodenum); + range->len = info.tf_trimlen; + goto out; + } + } + + info.tf_nodenum = osb->node_num; + info.tf_start = range->start; + info.tf_len = range->len; + info.tf_minlen = range->minlen; + + ret =ocfs2_trim_mainbm (sb, range); + + info.tf_trimlen = range->len; + info.tf_success = (ret <0 ?0 :1 ); + pinfo = &info; +out: +ocfs2_trim_fs_unlock (osb, pinfo); +ocfs2_trim_fs_lock_res_uninit (osb);return ret; }fs/ocfs2/dlmglue.c +0
-0@@ -686,6 +686,9 @@ void ocfs2_trim_fs_lock_res_init(struct ocfs2_super *osb) {struct ocfs2_lock_res *lockres = &osb->osb_trim_fs_lockres;Quoted 7 lines
Quoted 6 lines
fs/ocfs2/dlmglue.c void ocfs2_trim_fs_lock_res_init(struct ocfs2_super *osb) · line 691
Quoted 3 lines
fs/ocfs2/dlmglue.c +3
-0+/* Only one trimfs thread are allowed to work at the same time. */ +mutex_lock (&osb->obs_trim_fs_mutex); +Cluster lock of fstrim have a trylock behavior, will it be better if we trylock here?
Here, I prefer to just serialize fstrim threads on the local node to
simplify the code logic,
Quoted 1 line
maybe the user want to do like that, although this behavior is not
recommended.
Quoted 1 line
You know, on one node, ideally, the user should call fstrim command once
regularly.
Quoted 8 lines
Quoted 2 lines
If he calls fstrim command more times in a very short time, the code will not make each fstrim command return to failure, just do the
fstrim task one by one.
I have a thought having nothing to do with your patch.
Do you think it's possible for us to implement *discard on unlink file* like ext4 does.
So the application doesn't have to invoke fstrim periodically.
Yes, we can do some investigation for adding a mount option "discard" to support discard a file when it is deleted. This can be considered as another feature, since its discard occasion and granularity is different with fstrim. The scheduled fstrim command can considered as a traditional file system level trim.
If the file system supports the on-demand discard when some blocks were released, that's better. Of course. these two features can coexist.
Thanks
Gang
Quoted 65 lines
Thanks,
ChangweiQuoted 61 lines
Thanks
GangQuoted 56 lines
Thanks,
Changweifs/ocfs2/dlmglue.c void ocfs2_trim_fs_lock_res_uninit(struct ocfs2_super
Quoted 4 lines
fs/ocfs2/dlmglue.c +0
-0ocfs2_lock_res_init_once (lockres);ocfs2_build_lock_name (OCFS2_LOCK_TYPE_TRIM_FS ,0 ,0 , lockres->l_name); ocfs2_lock_res_init_common(osb, lockres, OCFS2_LOCK_TYPE_TRIM_FS,@@ -698,6 +701,8 @@ void ocfs2_trim_fs_lock_res_uninit(struct ocfs2_super*osb)
Quoted 47 lines
fs/ocfs2/dlmglue.c +2
-0ocfs2_simple_drop_lockres (osb, lockres);ocfs2_lock_res_free (lockres); + +mutex_unlock (&osb->obs_trim_fs_mutex); } static void ocfs2_orphan_scan_lock_res_init(struct ocfs2_lock_res *res,fs/ocfs2/ocfs2.h +1
-0@@ -407,6 +407,7 @@ struct ocfs2_superstruct ocfs2_lock_res osb_rename_lockres;struct ocfs2_lock_res osb_nfs_sync_lockres;struct ocfs2_lock_res osb_trim_fs_lockres; +struct mutex obs_trim_fs_mutex;struct ocfs2_dlm_debug *osb_dlm_debug;struct dentry *osb_debug_root;fs/ocfs2/ocfs2_trace.h +2
-0@@ -712,6 +712,8 @@ TRACE_EVENT(ocfs2_trim_extent,DEFINE_OCFS2_ULL_UINT_UINT_UINT_EVENT (ocfs2_trim_group); +DEFINE_OCFS2_ULL_ULL_ULL_EVENT (ocfs2_trim_mainbm); +DEFINE_OCFS2_ULL_ULL_ULL_EVENT (ocfs2_trim_fs);/* End of trace events for fs/ocfs2/alloc.c. */ fs/ocfs2/super.c +2
-0@@ -1847,6 +1847,8 @@ static int ocfs2_mount_volume(struct super_block *sb)if (ocfs2_is_hard_readonly (osb)) goto leave; +mutex_init (&osb->obs_trim_fs_mutex); + status =ocfs2_dlm_init (osb);if (status <0 ) {mlog_errno (status);
Changwei Ge
Reviewed-byHi Gang,
It looks good to me.
Quoted 29 lines · Gang He
Hello ChangWei,
Quoted 1 line
Quoted 1 line
Quoted 1 line
On 2019/1/15 at 16:00, in message
<63ADC13FD55D6546B7DECE290D39E37301278265F3@H3CMLB12-EX.srv.huawei-3com.com>,
Quoted 24 lines · Changwei Ge
Quoted 23 lines · Gang He
Hello Changewei,
Quoted 1 line
Quoted 1 line
Quoted 1 line
On 2019/1/15 at 11:50, in message
<63ADC13FD55D6546B7DECE290D39E3730127825EFD@H3CMLB12-EX.srv.huawei-3com.com>,
Quoted 18 lines · Changwei Ge
Hi Gang,
Most parts of this patch look sane to me, just a tiny question...
Quoted 13 lines · Gang He
The user reported this problem, the upper application IO was timeout when fstrim was running on this ocfs2 partition. the application monitoring resource agent considered that this application did not work, then this node was fenced by the cluster brain (e.g. pacemaker).
The root cause is that fstrim thread always holdsmain_bmmeta-file related locks until all the cluster groups are trimmed. This patch will make fstrim thread releasemain_bmmeta-file related locks when each cluster group is trimmed, this will let the current application IO has a chance to claim the clusters frommain_bmmeta-file.Signed-off-by Gang He <[email protected]>
Quoted 253 lines
Quoted 245 lines
Quoted 233 lines
Quoted 233 lines
fs/ocfs2/alloc.c int ocfs2_trim_fs(struct super_block *sb, struct
Quoted 27 lines
--- fs/ocfs2/alloc.c | 159 +++++++++++++++++++++++++---------------- fs/ocfs2/dlmglue.c | 5 ++ fs/ocfs2/ocfs2.h | 1 + fs/ocfs2/ocfs2_trace.h | 2 + fs/ocfs2/super.c | 2 + 5 files changed, 106 insertions(+), 63 deletions(-)fs/ocfs2/alloc.c +3
-2@@ -7532,10 +7532,11 @@ static int ocfs2_trim_group(struct super_block *sb,return count; } -int ocfs2_trim_fs (struct super_block *sb,struct fstrim_range *range) +static +int ocfs2_trim_mainbm (struct super_block *sb,struct fstrim_range *range) {struct ocfs2_super *osb =OCFS2_SB (sb); -u64 start, len, trimmed, first_group, last_group, group; +u64 start, len, trimmed =0 , first_group, last_group =0 , group =0 ;int ret, cnt;u32 first_bit, last_bit, minlen;struct buffer_head *main_bm_bh =NULL ;@@ -7543,7 +7544,6 @@ int ocfs2_trim_fs(struct super_block *sb, struct
fstrim_range*range)fs/ocfs2/alloc.c int ocfs2_trim_fs(struct super_block *sb, struct
Quoted 8 lines
fs/ocfs2/alloc.c +0
-1struct buffer_head *gd_bh =NULL ;struct ocfs2_dinode *main_bm;struct ocfs2_group_desc *gd =NULL ; -struct ocfs2_trim_fs_info info, *pinfo =NULL ; start = range->start >> osb->s_clustersize_bits; len = range->len >> osb->s_clustersize_bits;@@ -7552,6 +7552,9 @@ int ocfs2_trim_fs(struct super_block *sb, struct
fstrim_range*range)fs/ocfs2/alloc.c int ocfs2_trim_fs(struct super_block *sb, struct
Quoted 10 lines
fs/ocfs2/alloc.c +3
-0if (minlen >= osb->bitmap_cpg || range->len < sb->s_blocksize)return -EINVAL ; +trace_ocfs2_trim_mainbm (start, len, minlen); + +next_group: main_bm_inode =ocfs2_get_system_file_inode (osb,GLOBAL_BITMAP_SYSTEM_INODE ,OCFS2_INVALID_SLOT );@@ -7570,64 +7573,34 @@ int ocfs2_trim_fs(struct super_block *sb, struct
fstrim_range*range)fs/ocfs2/alloc.c int ocfs2_trim_fs(struct super_block *sb, struct
Quoted 87 lines
fs/ocfs2/alloc.c +22
-52} main_bm = (struct ocfs2_dinode *)main_bm_bh->b_data; -if (start >=le32_to_cpu (main_bm->i_clusters)) { - ret = -EINVAL ; - goto out_unlock; - } - - len = range->len >> osb->s_clustersize_bits; -if (start + len >le32_to_cpu (main_bm->i_clusters)) - len =le32_to_cpu (main_bm->i_clusters) - start; - -trace_ocfs2_trim_fs (start, len, minlen); - -ocfs2_trim_fs_lock_res_init (osb); - ret =ocfs2_trim_fs_lock (osb,NULL ,1 ); -if (ret <0 ) { -if (ret != -EAGAIN ) { -mlog_errno (ret); -ocfs2_trim_fs_lock_res_uninit (osb); +/* +* Do some check before trim the first group. +*/ +if (!group) { +if (start >=le32_to_cpu (main_bm->i_clusters)) { + ret = -EINVAL ; goto out_unlock; } -mlog (ML_NOTICE ,"Wait for trim on device (%s) to " -"finish, which is running from another node.\n" , - osb->dev_str); - ret =ocfs2_trim_fs_lock (osb, &info,0 ); -if (ret <0 ) { -mlog_errno (ret); -ocfs2_trim_fs_lock_res_uninit (osb); - goto out_unlock; - } +if (start + len >le32_to_cpu (main_bm->i_clusters)) + len =le32_to_cpu (main_bm->i_clusters) - start; -if (info.tf_valid && info.tf_success && - info.tf_start == start && info.tf_len == len && - info.tf_minlen == minlen) { -/* Avoid sending duplicated trim to a shared device */ -mlog (ML_NOTICE ,"The same trim on device (%s) was " -"just done from node (%u), return.\n" , - osb->dev_str, info.tf_nodenum); - range->len = info.tf_trimlen; - goto out_trimunlock; - } +/* +* Determine first and last group to examine based on +* start and len +*/ + first_group =ocfs2_which_cluster_group (main_bm_inode, start); +if (first_group == osb->first_cluster_group_blkno) + first_bit = start; +else + first_bit = start -ocfs2_blocks_to_clusters (sb, + first_group); + last_group =ocfs2_which_cluster_group (main_bm_inode, + start + len -1 ); + group = first_group; } - info.tf_nodenum = osb->node_num; - info.tf_start = start; - info.tf_len = len; - info.tf_minlen = minlen; - -/* Determine first and last group to examine based on start and len */ - first_group =ocfs2_which_cluster_group (main_bm_inode, start); -if (first_group == osb->first_cluster_group_blkno) - first_bit = start; -else - first_bit = start -ocfs2_blocks_to_clusters (sb, first_group); - last_group =ocfs2_which_cluster_group (main_bm_inode, start + len -1 ); - last_bit = osb->bitmap_cpg; - - trimmed =0 ; -for (group = first_group; group <= last_group;) { + do { if (first_bit + len >= osb->bitmap_cpg) last_bit = osb->bitmap_cpg; else@@ -7659,21 +7632,81 @@ int ocfs2_trim_fs(struct super_block *sb, struct
fstrim_range*range)Quoted 97 lines
fs/ocfs2/alloc.c +68
-8group =ocfs2_clusters_to_blocks (sb, osb->bitmap_cpg); else group += ocfs2_clusters_to_blocks(sb, osb->bitmap_cpg); - } - range->len = trimmed * sb->s_blocksize; + }while (0 ); - info.tf_trimlen = range->len; - info.tf_success = (ret ?0 :1 ); - pinfo = &info; -out_trimunlock: -ocfs2_trim_fs_unlock (osb, pinfo); -ocfs2_trim_fs_lock_res_uninit (osb); out_unlock:ocfs2_inode_unlock (main_bm_inode,0 );brelse (main_bm_bh); + main_bm_bh =NULL ; out_mutex:inode_unlock (main_bm_inode);iput (main_bm_inode); + +/* +* If all the groups trim are not done or failed, but we should release +* main_bm related locks for avoiding the current IO starve, then go to +* trim the next group +*/ +if (ret >=0 && group <= last_group) + goto next_group; out: + range->len = trimmed * sb->s_blocksize; +return ret; +} + +int ocfs2_trim_fs (struct super_block *sb,struct fstrim_range *range) +{ +int ret; +struct ocfs2_super *osb =OCFS2_SB (sb); +struct ocfs2_trim_fs_info info, *pinfo =NULL ; + +ocfs2_trim_fs_lock_res_init (osb); + +trace_ocfs2_trim_fs (range->start, range->len, range->minlen); + + ret =ocfs2_trim_fs_lock (osb,NULL ,1 ); +if (ret <0 ) { +if (ret != -EAGAIN ) { +mlog_errno (ret); +ocfs2_trim_fs_lock_res_uninit (osb); +return ret; + } + +mlog (ML_NOTICE ,"Wait for trim on device (%s) to " +"finish, which is running from another node.\n" , + osb->dev_str); + ret =ocfs2_trim_fs_lock (osb, &info,0 ); +if (ret <0 ) { +mlog_errno (ret); +ocfs2_trim_fs_lock_res_uninit (osb); +return ret; + } + +if (info.tf_valid && info.tf_success && + info.tf_start == range->start && + info.tf_len == range->len && + info.tf_minlen == range->minlen) { +/* Avoid sending duplicated trim to a shared device */ +mlog (ML_NOTICE ,"The same trim on device (%s) was " +"just done from node (%u), return.\n" , + osb->dev_str, info.tf_nodenum); + range->len = info.tf_trimlen; + goto out; + } + } + + info.tf_nodenum = osb->node_num; + info.tf_start = range->start; + info.tf_len = range->len; + info.tf_minlen = range->minlen; + + ret =ocfs2_trim_mainbm (sb, range); + + info.tf_trimlen = range->len; + info.tf_success = (ret <0 ?0 :1 ); + pinfo = &info; +out: +ocfs2_trim_fs_unlock (osb, pinfo); +ocfs2_trim_fs_lock_res_uninit (osb);return ret; }fs/ocfs2/dlmglue.c +0
-0@@ -686,6 +686,9 @@ void ocfs2_trim_fs_lock_res_init(struct ocfs2_super *osb) {struct ocfs2_lock_res *lockres = &osb->osb_trim_fs_lockres;Quoted 7 lines
Quoted 6 lines
fs/ocfs2/dlmglue.c void ocfs2_trim_fs_lock_res_init(struct ocfs2_super *osb) · line 691
Quoted 3 lines
fs/ocfs2/dlmglue.c +3
-0+/* Only one trimfs thread are allowed to work at the same time. */ +mutex_lock (&osb->obs_trim_fs_mutex); +Cluster lock of fstrim have a trylock behavior, will it be better if we trylock here?
Here, I prefer to just serialize fstrim threads on the local node to
simplify the code logic,
Quoted 1 line
maybe the user want to do like that, although this behavior is not
recommended.
Quoted 1 line
You know, on one node, ideally, the user should call fstrim command once
regularly.
Quoted 8 lines
Quoted 2 lines
If he calls fstrim command more times in a very short time, the code will not make each fstrim command return to failure, just do the
fstrim task one by one.
I have a thought having nothing to do with your patch.
Do you think it's possible for us to implement *discard on unlink file* like ext4 does.
So the application doesn't have to invoke fstrim periodically.
Quoted 5 lines
Yes, we can do some investigation for adding a mount option "discard" to support discard a file when it is deleted. This can be considered as another feature, since its discard occasion and granularity is different with fstrim. The scheduled fstrim command can considered as a traditional file system level trim.
If the file system supports the on-demand discard when some blocks were released, that's better. Of course. these two features can coexist.
Good to hear this. :-)
Quoted 70 lines
Thanks
GangQuoted 65 lines
Thanks,
ChangweiQuoted 61 lines
Thanks
GangQuoted 56 lines
Thanks,
Changweifs/ocfs2/dlmglue.c void ocfs2_trim_fs_lock_res_uninit(struct ocfs2_super
Quoted 4 lines
fs/ocfs2/dlmglue.c +0
-0ocfs2_lock_res_init_once (lockres);ocfs2_build_lock_name (OCFS2_LOCK_TYPE_TRIM_FS ,0 ,0 , lockres->l_name); ocfs2_lock_res_init_common(osb, lockres, OCFS2_LOCK_TYPE_TRIM_FS,@@ -698,6 +701,8 @@ void ocfs2_trim_fs_lock_res_uninit(struct ocfs2_super*osb)
Quoted 47 lines
fs/ocfs2/dlmglue.c +2
-0ocfs2_simple_drop_lockres (osb, lockres);ocfs2_lock_res_free (lockres); + +mutex_unlock (&osb->obs_trim_fs_mutex); } static void ocfs2_orphan_scan_lock_res_init(struct ocfs2_lock_res *res,fs/ocfs2/ocfs2.h +1
-0@@ -407,6 +407,7 @@ struct ocfs2_superstruct ocfs2_lock_res osb_rename_lockres;struct ocfs2_lock_res osb_nfs_sync_lockres;struct ocfs2_lock_res osb_trim_fs_lockres; +struct mutex obs_trim_fs_mutex;struct ocfs2_dlm_debug *osb_dlm_debug;struct dentry *osb_debug_root;fs/ocfs2/ocfs2_trace.h +2
-0@@ -712,6 +712,8 @@ TRACE_EVENT(ocfs2_trim_extent,DEFINE_OCFS2_ULL_UINT_UINT_UINT_EVENT (ocfs2_trim_group); +DEFINE_OCFS2_ULL_ULL_ULL_EVENT (ocfs2_trim_mainbm); +DEFINE_OCFS2_ULL_ULL_ULL_EVENT (ocfs2_trim_fs);/* End of trace events for fs/ocfs2/alloc.c. */ fs/ocfs2/super.c +2
-0@@ -1847,6 +1847,8 @@ static int ocfs2_mount_volume(struct super_block *sb)if (ocfs2_is_hard_readonly (osb)) goto leave; +mutex_init (&osb->obs_trim_fs_mutex); + status =ocfs2_dlm_init (osb);if (status <0 ) {mlog_errno (status);