mm: swap: async free swap slot cache entries
Subsystems (2)
Chris Li
maintainerPATCH v3 1/1 mm: swap: async free swap slot cache entries
We discovered that 1% swap page fault is 100us+ while 50% of the swap fault is under 20us.
Further investigation show that a large portion of the time spent in the free_swap_slots() function for the long tail case.
The percpu cache of swap slots is freed in a batch of 64 entries inside free_swap_slots(). These cache entries are accumulated from previous page faults, which may not be related to the current process.
Doing the batch free in the page fault handler causes longer tail latencies and penalizes the current process.
Add /sys/kernel/mm/swap/swap_slot_async_free to control the async free behavior. When enabled, using work queue to async free the swap slot when the swap slot cache is full.
Testing:
Chun-Tse did some benchmark in chromebook, showing that zram_wait_metrics improve about 15% with 80% and 95% confidence.
I recently ran some experiments on about 1000 Google production machines. It shows swapin latency drops in the long tail 100us - 500us bucket dramatically.
platform (100-500us) (0-100us)
A 1.12% -> 0.36% 98.47% -> 99.22%
B 0.65% -> 0.15% 98.96% -> 99.46%
C 0.61% -> 0.23% 98.96% -> 99.38%
---
Changes in v3:
- Address feedback from Tim Chen, direct free path will free all swap slots.
- Add /sys/kernel/mm/swap/swap_slot_async_fee to enable async free. Default is off.
- Link to v2: https://lore.kernel.org/r/[email protected]
Changes in v2:
- Add description of the impact of time changing suggest by Ying.
- Remove create_workqueue() and use schedule_work()
- Link to v1: https://lore.kernel.org/r/[email protected]
--- include/linux/swap_slots.h | 2 ++ mm/swap_slots.c | 20 ++++++++++++++++++++ mm/swap_state.c | 23 +++++++++++++++++++++++ 3 files changed, 45 insertions(+)
include/linux/swap_slots.h +2 -0
@@ -19,6 +19,7 @@ struct swap_slots_cache {spinlock_t free_lock;/* protects slots_ret, n_ret */ swp_entry_t *slots_ret;int n_ret; +struct work_struct async_free; };void disable_swap_slots_cache_lock (void );
@@ -27,5 +28,6 @@ void enable_swap_slots_cache(void);void free_swap_slot (swp_entry_t entry);extern bool swap_slot_cache_enabled; +extern uint8_t slot_cache_async_free__read_mostly ;#endif /* _LINUX_SWAP_SLOTS_H */
mm/swap_slots.c +20 -0
@@ -38,12 +38,15 @@ static DEFINE_PER_CPU(struct swap_slots_cache, swp_slots);static bool swap_slot_cache_active;bool swap_slot_cache_enabled; +uint8_t slot_cache_async_free; +static bool swap_slot_cache_initialized;static DEFINE_MUTEX (swap_slots_cache_mutex);/* Serialize swap slots cache enable/disable operations */ static DEFINE_MUTEX (swap_slots_cache_enable_mutex);static void __drain_swap_slots_cache (unsigned int type); +static void swapcache_async_free_entries (struct work_struct *data);#define use_swap_slot_cache (swap_slot_cache_active && swap_slot_cache_enabled)#define SLOTS_CACHE 0x1
@@ -149,6 +152,7 @@ static int alloc_swap_slot_cache(unsigned int cpu)spin_lock_init (&cache->free_lock); cache->lock_initialized = true; } +INIT_WORK (&cache->async_free, swapcache_async_free_entries); cache->nr =0 ; cache->cur =0 ; cache->n_ret =0 ;
@@ -269,6 +273,20 @@ static int refill_swap_slots_cache(struct swap_slots_cache *cache)return cache->nr; } +static void swapcache_async_free_entries (struct work_struct *data) +{ +struct swap_slots_cache *cache; + + cache = container_of(data, struct swap_slots_cache, async_free); +spin_lock_irq (&cache->free_lock); +/* Swap slots cache may be deactivated before acquiring lock */ +if (cache->slots_ret && cache->n_ret) { +swapcache_free_entries (cache->slots_ret, cache->n_ret); + cache->n_ret =0 ; + } +spin_unlock_irq (&cache->free_lock); +} +void free_swap_slot (swp_entry_t entry) {struct swap_slots_cache *cache;
@@ -293,6 +311,8 @@ void free_swap_slot(swp_entry_t entry) } cache->slots_ret[cache->n_ret++] = entry;spin_unlock_irq (&cache->free_lock); +if (slot_cache_async_free && cache->n_ret >=SWAP_SLOTS_CACHE_SIZE ) +schedule_work (&cache->async_free); }else { direct_free:swapcache_free_entries (&entry,1 );
mm/swap_state.c +23 -0
@@ -912,8 +912,31 @@ static ssize_t vma_ra_enabled_store(struct kobject *kobj, }static struct kobj_attribute vma_ra_enabled_attr =__ATTR_RW (vma_ra_enabled); +static ssize_t swap_slot_async_free_show (struct kobject *kobj, +struct kobj_attribute *attr,char *buf) +{ +return sprintf (buf,"%d\n" ,READ_ONCE (slot_cache_async_free)); +} +static ssize_t swap_slot_async_free_store (struct kobject *kobj, +struct kobj_attribute *attr, +const char *buf,size_t count) +{ +ssize_t ret; +int val; + + ret =kstrtoint (buf,0 , &val); +if (ret) +return ret; +WRITE_ONCE (slot_cache_async_free, !!val); +return count; +} +static struct kobj_attribute swap_slot_async_free_attr = +__ATTR (swap_slot_async_free,0644 , swap_slot_async_free_show, + swap_slot_async_free_store); +static struct attribute *swap_attrs[] = { &vma_ra_enabled_attr.attr, + &swap_slot_async_free_attr.attr,NULL , };
---
base-commit: eacce8189e28717da6f44ee492b7404c636ae0de
change-id: 20231216-async-free-bef392015432
Best regards,
Signature · 1 line
Chris Li <[email protected]>
Quoted 62 lines · Chris Li
We discovered that 1% swap page fault is 100us+ while 50% of the swap fault is under 20us.
Further investigation show that a large portion of the time spent in the
free_swap_slots()function for the long tail case.The percpu cache of swap slots is freed in a batch of 64 entries inside
free_swap_slots(). These cache entries are accumulated from previous page faults, which may not be related to the current process.Doing the batch free in the page fault handler causes longer tail latencies and penalizes the current process.
Add
/sys/kernel/mm/swap/swap_slot_async_freeto control the async free behavior. When enabled, using work queue to async free the swap slot when the swap slot cache is full.Testing:
Chun-Tse did some benchmark in chromebook, showing that
zram_wait_metricsimprove about 15% with 80% and 95% confidence.I recently ran some experiments on about 1000 Google production machines. It shows swapin latency drops in the long tail 100us - 500us bucket dramatically.
platform (100-500us) (0-100us)
A 1.12% -> 0.36% 98.47% -> 99.22%
B 0.65% -> 0.15% 98.96% -> 99.46%
C 0.61% -> 0.23% 98.96% -> 99.38%Signed-off-by Chris Li <[email protected]>---
Changes in v3:
- Address feedback from Tim Chen, direct free path will free all swap slots.
- Add/sys/kernel/mm/swap/swap_slot_async_feeto enable async free. Default is off.
- Link to v2: https://lore.kernel.org/r/[email protected]Changes in v2:
- Add description of the impact of time changing suggest by Ying.
- Removecreate_workqueue()and useschedule_work()
- Link to v1: https://lore.kernel.org/r/[email protected]--- include/linux/swap_slots.h | 2 ++ mm/swap_slots.c | 20 ++++++++++++++++++++ mm/swap_state.c | 23 +++++++++++++++++++++++ 3 files changed, 45 insertions(+)include/linux/swap_slots.h +1
-0@@ -19,6 +19,7 @@ struct swap_slots_cache {spinlock_t free_lock;/* protects slots_ret, n_ret */ swp_entry_t *slots_ret;int n_ret; +struct work_struct async_free; };void disable_swap_slots_cache_lock (void );
include/linux/swap_slots.h void enable_swap_slots_cache(void); · line 31
Quoted 4 lines
include/linux/swap_slots.h +1
-0@@ -27,5 +28,6 @@ void enable_swap_slots_cache(void);void free_swap_slot (swp_entry_t entry);extern bool swap_slot_cache_enabled; +extern uint8_t slot_cache_async_free__read_mostly ;
Why wouldn't you enable the async_free always?
Otherwise the patch looks fine to me.
Tim
Quoted 1 line
Quoted 66 lines · Tim Chen
Quoted 64 lines · Chris Li
We discovered that 1% swap page fault is 100us+ while 50% of the swap fault is under 20us.
Further investigation show that a large portion of the time spent in the
free_swap_slots()function for the long tail case.The percpu cache of swap slots is freed in a batch of 64 entries inside
free_swap_slots(). These cache entries are accumulated from previous page faults, which may not be related to the current process.Doing the batch free in the page fault handler causes longer tail latencies and penalizes the current process.
Add
/sys/kernel/mm/swap/swap_slot_async_freeto control the async free behavior. When enabled, using work queue to async free the swap slot when the swap slot cache is full.Testing:
Chun-Tse did some benchmark in chromebook, showing that
zram_wait_metricsimprove about 15% with 80% and 95% confidence.I recently ran some experiments on about 1000 Google production machines. It shows swapin latency drops in the long tail 100us - 500us bucket dramatically.
platform (100-500us) (0-100us) A 1.12% -> 0.36% 98.47% -> 99.22% B 0.65% -> 0.15% 98.96% -> 99.46% C 0.61% -> 0.23% 98.96% -> 99.38%Signed-off-by Chris Li <[email protected]>---
Changes in v3:
- Address feedback from Tim Chen, direct free path will free all swap slots.
- Add/sys/kernel/mm/swap/swap_slot_async_feeto enable async free. Default is off.
- Link to v2: https://lore.kernel.org/r/[email protected]Changes in v2:
- Add description of the impact of time changing suggest by Ying.
- Removecreate_workqueue()and useschedule_work()
- Link to v1: https://lore.kernel.org/r/[email protected]--- include/linux/swap_slots.h | 2 ++ mm/swap_slots.c | 20 ++++++++++++++++++++ mm/swap_state.c | 23 +++++++++++++++++++++++ 3 files changed, 45 insertions(+)include/linux/swap_slots.h +1
-0@@ -19,6 +19,7 @@ struct swap_slots_cache {spinlock_t free_lock;/* protects slots_ret, n_ret */ swp_entry_t *slots_ret;int n_ret; +struct work_struct async_free; };void disable_swap_slots_cache_lock (void );@@ -27,5 +28,6 @@ void enable_swap_slots_cache(void);void free_swap_slot (swp_entry_t entry);
Quoted 5 lines
include/linux/swap_slots.h void enable_swap_slots_cache(void); · line 31
Quoted 2 lines
include/linux/swap_slots.h +1
-0extern bool swap_slot_cache_enabled; +extern uint8_t slot_cache_async_free__read_mostly ;Why wouldn't you enable the
async_freealways?
Otherwise the patch looks fine to me.
Thanks for the feedback.
Just in case someone doesn't care about this optimization and wants to opt out this behavior?
Anyway, I am happy to update the patch without the sysfs control file as well.
Chris
Quoted 7 lines · Chris Li
Quoted 6 lines · Tim Chen
Quoted 5 lines
extern bool
swap_slot_cache_enabled;
+externuint8_tslot_cache_async_free__read_mostly;
Quoted 9 lines
Quoted 2 lines
Why wouldn't you enable the
async_freealways?
Otherwise the patch looks fine to me.Thanks for the feedback.
Just in case someone doesn't care about this optimization and wants to opt out this behavior?
Anyway, I am happy to update the patch without the sysfs control file as well.
At least I couldn't see a downside to enable it always in the latest patch. I think adding an extra sysfs is unnecessary.
Tim
Chris Li
maintainerQuoted 14 lines · Tim Chen
Quoted 12 lines · Chris Li
Quoted 8 lines · Tim Chen
Quoted 5 lines
extern bool
swap_slot_cache_enabled;
+externuint8_tslot_cache_async_free__read_mostly;Why wouldn't you enable the
async_freealways?
Otherwise the patch looks fine to me.Thanks for the feedback.
Quoted 7 lines
Quoted 4 lines
Just in case someone doesn't care about this optimization and wants to opt out this behavior?
Anyway, I am happy to update the patch without the sysfs control file as well.At least I couldn't see a downside to enable it always in the latest patch. I think adding an extra sysfs is unnecessary.
Thanks, I just sent out V4 to remove the sysfs control file.
Chris