Backport that adds delayed orphan scan timer to 1.4
Next two patches are backport of orphan scan timer patches to ocfs2-1.4
PATCH 1/2 ocfs2: timer to queue scan of all orphan slots
When a dentry is unlinked, the unlinking node takes an EX on the dentry lock before moving the dentry to the orphan directory. The other nodes, that all had a PR on the same dentry lock, flag the corresponding inode as MAYBE_ORPHANED during the downconvert. The inode is finally deleted when the last node to iput the inode notices the MAYBE_ORPHANED flag.
A problem arises if a node is forced to free dentry locks because of memory pressure. If this happens, the node will no longer get downconvert notifications for the dentries that have been unlinked on another node. If it also happens that node is actively using the corresponding inode and happens to be the one performing the last iput on that inode, it will fail to delete the inode as it will not have the MAYBE_ORPHANED flag set.
This patch fixes this shortcoming by introducing a periodic scan of the orphan directories to delete such inodes. Care has been taken to distribute the workload across the cluster so that no one node has to perform the task all the time.
--- fs/ocfs2/dlmglue.c | 51 ++++++++++++++++++++++ fs/ocfs2/dlmglue.h | 10 ++++ fs/ocfs2/journal.c | 106 +++++++++++++++++++++++++++++++++++++++++++++++ fs/ocfs2/journal.h | 4 ++ fs/ocfs2/ocfs2.h | 9 ++++ fs/ocfs2/ocfs2_lockid.h | 5 ++ fs/ocfs2/super.c | 9 ++++ 7 files changed, 194 insertions(+), 0 deletions(-)
fs/ocfs2/dlmglue.c +51 -0
@@ -246,6 +246,10 @@ static struct ocfs2_lock_res_ops ocfs2_rename_lops = { .flags = 0, }; +static struct ocfs2_lock_res_ops ocfs2_orphan_scan_lops = { + .flags =LOCK_TYPE_REQUIRES_REFRESH |LOCK_TYPE_USES_LVB , +}; +static struct ocfs2_lock_res_ops ocfs2_dentry_lops = { .get_osb = ocfs2_get_dentry_osb, .post_unlock = ocfs2_dentry_post_unlock,
@@ -632,6 +636,19 @@ static void ocfs2_rename_lock_res_init(struct ocfs2_lock_res *res, &ocfs2_rename_lops, osb); } +static void ocfs2_orphan_scan_lock_res_init (struct ocfs2_lock_res *res, +struct ocfs2_super *osb) +{ +struct ocfs2_orphan_scan_lvb *lvb; + +ocfs2_lock_res_init_once (res); +ocfs2_build_lock_name (OCFS2_LOCK_TYPE_ORPHAN_SCAN ,0 ,0 , res->l_name); +ocfs2_lock_res_init_common (osb, res,OCFS2_LOCK_TYPE_ORPHAN_SCAN , + &ocfs2_orphan_scan_lops, osb); + lvb = (struct ocfs2_orphan_scan_lvb *)res->l_lksb.lvb; + lvb->lvb_version =OCFS2_ORPHAN_LVB_VERSION ; +} +void ocfs2_file_lock_res_init (struct ocfs2_lock_res *lockres,struct ocfs2_file_private *fp) {
@@ -2212,6 +2229,37 @@ void ocfs2_inode_unlock(struct inode *inode,mlog_exit_void (); } +int ocfs2_orphan_scan_lock (struct ocfs2_super *osb,u32 *seqno,int ex) +{ +struct ocfs2_lock_res *lockres; +struct ocfs2_orphan_scan_lvb *lvb; +int level = ex ?LKM_EXMODE :LKM_PRMODE ; +int status =0 ; + + lockres = &osb->osb_orphan_scan.os_lockres; + status =ocfs2_cluster_lock (osb, lockres, level,0 ,0 ); +if (status <0 ) +return status; + + lvb = (struct ocfs2_orphan_scan_lvb *)lockres->l_lksb.lvb; +if (lvb->lvb_version ==OCFS2_ORPHAN_LVB_VERSION ) + *seqno =be32_to_cpu (lvb->lvb_os_seqno); +return status; +} + +void ocfs2_orphan_scan_unlock (struct ocfs2_super *osb,u32 seqno,int ex) +{ +struct ocfs2_lock_res *lockres; +struct ocfs2_orphan_scan_lvb *lvb; +int level = ex ?LKM_EXMODE :LKM_PRMODE ; + + lockres = &osb->osb_orphan_scan.os_lockres; + lvb = (struct ocfs2_orphan_scan_lvb *)lockres->l_lksb.lvb; + lvb->lvb_version =OCFS2_ORPHAN_LVB_VERSION ; + lvb->lvb_os_seqno =cpu_to_be32 (seqno); +ocfs2_cluster_unlock (osb, lockres, level); +} +int ocfs2_super_lock (struct ocfs2_super *osb,int ex) {
@@ -2676,6 +2724,7 @@ int ocfs2_dlm_init(struct ocfs2_super *osb) local:ocfs2_super_lock_res_init (&osb->osb_super_lockres, osb);ocfs2_rename_lock_res_init (&osb->osb_rename_lockres, osb); +ocfs2_orphan_scan_lock_res_init (&osb->osb_orphan_scan.os_lockres, osb); osb->dlm = dlm;
@@ -2706,6 +2755,7 @@ void ocfs2_dlm_shutdown(struct ocfs2_super *osb)ocfs2_lock_res_free (&osb->osb_super_lockres);ocfs2_lock_res_free (&osb->osb_rename_lockres); +ocfs2_lock_res_free (&osb->osb_orphan_scan.os_lockres);dlm_unregister_domain (osb->dlm); osb->dlm =NULL ;
@@ -2904,6 +2954,7 @@ static void ocfs2_drop_osb_locks(struct ocfs2_super *osb) {ocfs2_simple_drop_lockres (osb, &osb->osb_super_lockres);ocfs2_simple_drop_lockres (osb, &osb->osb_rename_lockres); +ocfs2_simple_drop_lockres (osb, &osb->osb_orphan_scan.os_lockres); }int ocfs2_drop_inode_locks (struct inode *inode)
fs/ocfs2/dlmglue.h +10 -0
@@ -49,6 +49,14 @@ struct ocfs2_meta_lvb {__be32 lvb_reserved2; }; +#define OCFS2_ORPHAN_LVB_VERSION 1 + +struct ocfs2_orphan_scan_lvb { +__u8 lvb_version; +__u8 lvb_reserved[3 ]; +__be32 lvb_os_seqno; +}; +/* ocfs2_inode_lock_full() 'arg_flags' flags */ /* don't wait on recovery. */ #define OCFS2_META_LOCK_RECOVERY (0x01)
@@ -97,6 +105,8 @@ int ocfs2_super_lock(struct ocfs2_super *osb, int ex);void ocfs2_super_unlock (struct ocfs2_super *osb,int ex); +int ocfs2_orphan_scan_lock (struct ocfs2_super *osb,u32 *seqno,int ex); +void ocfs2_orphan_scan_unlock (struct ocfs2_super *osb,u32 seqno,int ex);int ocfs2_rename_lock (struct ocfs2_super *osb);void ocfs2_rename_unlock (struct ocfs2_super *osb);int ocfs2_dentry_lock (struct dentry *dentry,int ex);
fs/ocfs2/journal.c +106 -0
124 lines not on this page · raw message
fs/ocfs2/journal.h +4 -0
@@ -133,6 +133,10 @@ static inline void ocfs2_inode_set_new(struct ocfs2_super *osb, }/* Exported only for the journal struct init code in super.c. Do not call. */ +int ocfs2_orphan_scan_init (struct ocfs2_super *osb); +void ocfs2_orphan_scan_stop (struct ocfs2_super *osb); +void ocfs2_orphan_scan_exit (struct ocfs2_super *osb); +void ocfs2_complete_recovery (kapi_work_struct_t *work);int ocfs2_compute_replay_slots (struct ocfs2_super *osb);
fs/ocfs2/ocfs2.h +9 -0
@@ -145,6 +145,14 @@ struct ocfs2_lock_res {#endif }; +struct ocfs2_orphan_scan { +struct mutex os_lock; +struct ocfs2_super *os_osb; +struct ocfs2_lock_res os_lockres;/* lock to synchronize scans */ +struct delayed_work os_orphan_scan_work; +u32 os_seqno;/* incremented on every scan */ +}; +struct ocfs2_dlm_debug {struct kref d_refcnt;struct dentry *d_locking_state;
@@ -319,6 +327,7 @@ struct ocfs2_superstruct ocfs2_node_map osb_recovering_orphan_dirs;unsigned int *osb_orphan_wipes;wait_queue_head_t osb_wipe_event; +struct ocfs2_orphan_scan osb_orphan_scan;/* the group we used to allocate inodes. */ u64 osb_inode_alloc_group;
fs/ocfs2/ocfs2_lockid.h +5 -0
@@ -46,6 +46,7 @@ enum ocfs2_lock_type {OCFS2_LOCK_TYPE_DENTRY ,OCFS2_LOCK_TYPE_OPEN ,OCFS2_LOCK_TYPE_FLOCK , +OCFS2_LOCK_TYPE_ORPHAN_SCAN , OCFS2_NUM_LOCK_TYPES };
@@ -77,6 +78,9 @@ static inline char ocfs2_lock_type_char(enum ocfs2_lock_type type)case OCFS2_LOCK_TYPE_FLOCK : c ='F' ;break ; +case OCFS2_LOCK_TYPE_ORPHAN_SCAN : + c ='P' ; +break ;default : c ='\0' ; }
@@ -95,6 +99,7 @@ static char *ocfs2_lock_type_strings[] = { [OCFS2_LOCK_TYPE_DENTRY] = "Dentry", [OCFS2_LOCK_TYPE_OPEN] = "Open", [OCFS2_LOCK_TYPE_FLOCK] = "Flock", + [OCFS2_LOCK_TYPE_ORPHAN_SCAN] = "OrphanScan", };static inline const char *ocfs2_lock_type_string (enum ocfs2_lock_type type)
fs/ocfs2/super.c +9 -0
@@ -1495,6 +1495,8 @@ static void ocfs2_dismount_volume(struct super_block *sb, int mnt_err)ocfs2_truncate_log_shutdown (osb); +ocfs2_orphan_scan_stop (osb); +/* disable any new recovery threads and wait for any currently * running ones to exit. Do this before setting the vol_state. */ mutex_lock (&osb->recovery_lock);
@@ -1640,6 +1642,13 @@ static int ocfs2_initialize_super(struct super_block *sb, osb->disable_recovery =0 ; osb->recovery_thread_task =NULL ; + status =ocfs2_orphan_scan_init (osb); +if (status) { +mlog (ML_ERROR ,"Unable to initialize delayed orphan scan\n" ); +mlog_errno (status); + goto bail; + } +init_waitqueue_head (&osb->checkpoint_event);atomic_set (&osb->needs_checkpoint,0 );
Signature · 1 line
1.5.6.5
PATCH 1/2 ocfs2: timer to queue scan of all orphan slots
Srini,
I was re-reviewing these patches as part of 1.4 merge and something caught my eye. Specifically that this may slowdown umount unnecessarily.
Consider the case if scan_work() is fired a tick before scan_stop(). Currently, scan_stop() will cancel the newly queued scan_work() but do nothing to the tasks queued by the earlier fire. umount thread will have to wait for all those tasks to complete.
One solution is to add a flag, atomic_t os_stop_scan, in struct ocfs2_orphan_scan. Call atomic_set() at the top of ocfs2_orphan_scan_stop(). In ocfs2_queue_orphan_scan(), check if the flag is set before and after ocfs2_orphan_scan_lock(). If set, exit without queuing the tasks.
Secondly, call scan_stop() earlier in umount. Definitely before truncatelog shutdown. Actually make it even before localalloc shutdown.
Make the patch atop what is in Joel's git tree already.
Thanks
Sunil
Quoted 132 lines · Srinivas Eeda
fs/ocfs2/journal.c +76
-0+void ocfs2_queue_orphan_scan (struct ocfs2_super *osb) +{ +struct ocfs2_orphan_scan *os; +int status, i; +u32 seqno =0 ; + + os = &osb->osb_orphan_scan; + + status =ocfs2_orphan_scan_lock (osb, &seqno,LKM_EXMODE ); +if (status <0 ) { +if (status != -EAGAIN ) +mlog_errno (status); + goto out; + } + +if (os->os_seqno != seqno) { + os->os_seqno = seqno; + goto unlock; + } + +for (i =0 ; i < osb->max_slots; i++) +ocfs2_queue_recovery_completion (osb->journal, i,NULL ,NULL ); + +/* +* We queued a recovery on orphan slots, increment the sequence +* number and update LVB so other node will skip the scan for a while +*/ + seqno++; +unlock: +ocfs2_orphan_scan_unlock (osb, seqno,LKM_EXMODE ); +out: +return ; +} + +/* Worker task that gets fired every ORPHAN_SCAN_SCHEDULE_TIMEOUT millsec */ +void ocfs2_orphan_scan_work (kapi_work_struct_t *work) +{ +struct ocfs2_orphan_scan *os; +struct ocfs2_super *osb; + + os = work_to_object(work, struct ocfs2_orphan_scan, + os_orphan_scan_work.work); + osb = os->os_osb; + +mutex_lock (&os->os_lock); +ocfs2_queue_orphan_scan (osb); +schedule_delayed_work (&os->os_orphan_scan_work, +ocfs2_orphan_scan_timeout ()); +mutex_unlock (&os->os_lock); +} + +void ocfs2_orphan_scan_stop (struct ocfs2_super *osb) +{ +struct ocfs2_orphan_scan *os; + + os = &osb->osb_orphan_scan; +mutex_lock (&os->os_lock); +cancel_delayed_work (&os->os_orphan_scan_work); +mutex_unlock (&os->os_lock); +} + +int ocfs2_orphan_scan_init (struct ocfs2_super *osb) +{ +struct ocfs2_orphan_scan *os; + + os = &osb->osb_orphan_scan; + os->os_osb = osb; +mutex_init (&os->os_lock); + +KAPI_INIT_DELAYED_WORK (&os->os_orphan_scan_work, + ocfs2_orphan_scan_work, os); +schedule_delayed_work (&os->os_orphan_scan_work, +ocfs2_orphan_scan_timeout ()); +return 0 ; +} +fs/ocfs2/ocfs2.h +8
-0@@ -145,6 +145,14 @@ struct ocfs2_lock_res {#endif }; +struct ocfs2_orphan_scan { +struct mutex os_lock; +struct ocfs2_super *os_osb; +struct ocfs2_lock_res os_lockres;/* lock to synchronize scans */ +struct delayed_work os_orphan_scan_work; +u32 os_seqno;/* incremented on every scan */ +}; +fs/ocfs2/super.c +9
-0@@ -1495,6 +1495,8 @@ static void ocfs2_dismount_volume(struct super_block *sb, int mnt_err)ocfs2_truncate_log_shutdown (osb); +ocfs2_orphan_scan_stop (osb); +/* disable any new recovery threads and wait for any currently * running ones to exit. Do this before setting the vol_state. */ mutex_lock (&osb->recovery_lock);@@ -1640,6 +1642,13 @@ static int ocfs2_initialize_super(struct super_block *sb, osb->disable_recovery =0 ; osb->recovery_thread_task =NULL ; + status =ocfs2_orphan_scan_init (osb); +if (status) { +mlog (ML_ERROR ,"Unable to initialize delayed orphan scan\n" ); +mlog_errno (status); + goto bail; + } +init_waitqueue_head (&osb->checkpoint_event);atomic_set (&osb->needs_checkpoint,0 );
PATCH 2/2 ocfs2 patch to track delayed orphan scan timer statistics
In reply to Srinivas EedaPatch to track delayed orphan scan timer statistics.
Modifies ocfs2_osb_dump to print the following:
Orphan Scan=> Local: 10 Global: 21 Last Scan: 67 seconds ago
--- fs/ocfs2/journal.c | 4 ++++ fs/ocfs2/ocfs2.h | 4 +++- fs/ocfs2/super.c | 7 +++++++ 3 files changed, 14 insertions(+), 1 deletions(-)
fs/ocfs2/journal.c +4 -0
@@ -1498,6 +1498,8 @@ void ocfs2_queue_orphan_scan(struct ocfs2_super *osb)* number and update LVB so other node will skip the scan for a while */ seqno++; + os->os_count++; + os->os_scantime =CURRENT_TIME ; unlock:ocfs2_orphan_scan_unlock (osb, seqno,LKM_EXMODE ); out:
@@ -1537,6 +1539,8 @@ int ocfs2_orphan_scan_init(struct ocfs2_super *osb) os = &osb->osb_orphan_scan; os->os_osb = osb; + os->os_count =0 ; + os->os_scantime =CURRENT_TIME ;mutex_init (&os->os_lock); KAPI_INIT_DELAYED_WORK(&os->os_orphan_scan_work,
fs/ocfs2/ocfs2.h +3 -1
@@ -150,7 +150,9 @@ struct ocfs2_orphan_scan {struct ocfs2_super *os_osb;struct ocfs2_lock_res os_lockres;/* lock to synchronize scans */ struct delayed_work os_orphan_scan_work; -u32 os_seqno;/* incremented on every scan */ +struct timespec os_scantime;/* time this node ran the scan */ +u32 os_count;/* tracks node specific scans */ +u32 os_seqno;/* tracks cluster wide scans */ };struct ocfs2_dlm_debug {
fs/ocfs2/super.c +7 -0
@@ -198,6 +198,7 @@ static int ocfs2_osb_dump(struct ocfs2_super *osb, char *buf, int len) {int out =0 ;int i; +struct ocfs2_orphan_scan *os; out += snprintf(buf + out, len - out, "%10s => Id: %-s Uuid: %-s Gen: 0x%X Label: %-s\n",
@@ -290,6 +291,12 @@ static int ocfs2_osb_dump(struct ocfs2_super *osb, char *buf, int len) i, osb->slot_recovery_generations[i]); } + os = &osb->osb_orphan_scan; + out +=snprintf (buf + out, len - out,"Orphan Scan=> " ); + out +=snprintf (buf + out, len - out,"Local: %u Global: %u " , + os->os_count, os->os_seqno); + out +=snprintf (buf + out, len - out," Last Scan: %lu seconds ago\n" , + (get_seconds () - os->os_scantime.tv_sec));return out; }
Signature · 1 line
1.5.6.5