mm/swap: Add cache for swap slots allocation
Subsystems (2)
PATCH 7/8 mm/swap: Add cache for swap slots allocation
We add per cpu caches for swap slots that can be allocated and freed quickly without the need to touch the swap info lock.
Two separate caches are maintained for swap slots allocated and swap slots returned. This is to allow the swap slots to be returned to the global pool in a batch so they will have a chance to be coaelesced with other slots in a cluster. We do not reuse the slots that are returned right away, as it may increase fragmentation of the slots.
The swap allocation cache is protected by a mutex as we may sleep when searching for empty slots in cache. The swap free cache is protected by a spin lock as we cannot sleep in the free path.
We refill the swap slots cache when we run out of slots, and we disable the swap slots cache and drain the slots if the global number of slots fall below a low watermark threshold. We re-enable the cache agian when the slots available are above a high watermark.
--- include/linux/swap.h | 3 + include/linux/swap_slots.h | 37 ++++++ mm/Makefile | 2 +- mm/swap_slots.c | 305 +++++++++++++++++++++++++++++++++++++++++++++ mm/swap_state.c | 1 + mm/swapfile.c | 15 +-- 6 files changed, 351 insertions(+), 12 deletions(-) create mode 100644 include/linux/swap_slots.h create mode 100644 mm/swap_slots.c
include/linux/swap.h +3 -0
@@ -434,6 +434,9 @@ struct backing_dev_info;extern int init_swap_address_space (unsigned int type,unsigned long nr_pages);extern void exit_swap_address_space (unsigned int type); +extern int get_swap_slots (int n,swp_entry_t *slots); +extern void swapcache_free_batch (swp_entry_t *entries,int n); +#else /* CONFIG_SWAP */ #define swap_address_space (entry) (NULL)
include/linux/swap_slots.h new file +37 -0
@@ -0,0 +1,37 @@ +#ifndef _LINUX_SWAP_SLOTS_H +#define _LINUX_SWAP_SLOTS_H + +#include <linux/swap.h> +#include <linux/spinlock.h> +#include <linux/mutex.h> + +#define SWAP_SLOTS_CACHE_SIZE SWAP_BATCH +#define THRESHOLD_ACTIVATE_SWAP_SLOTS_CACHE (5*SWAP_SLOTS_CACHE_SIZE) +#define THRESHOLD_DEACTIVATE_SWAP_SLOTS_CACHE (2*SWAP_SLOTS_CACHE_SIZE) + +struct swap_slots_cache { +struct mutex alloc_lock; +swp_entry_t *slots; +int nr; +int cur; +spinlock_t free_lock; +swp_entry_t *slots_ret; +int n_ret; +}; + +DECLARE_PER_CPU(struct swap_slots_cache, swp_slots); +extern bool swap_slot_cache_enabled; + +void drain_swap_slots_cache (unsigned int type); +void deactivate_swap_slots_cache (void ); +void disable_swap_slots_cache (void ); +void reenable_swap_slots_cache (void ); +void reactivate_swap_slots_cache (void ); + +int free_swap_slot_to_cache (swp_entry_t entry); +swp_entry_t get_swap_slot (void ); +int get_swap_slots (int n,swp_entry_t *entries); +void free_swap_slot_cache (int cpu); +int init_swap_slot_caches (void ); + +#endif /* _LINUX_SWAP_SLOTS_H */
mm/Makefile +1 -1
@@ -38,7 +38,7 @@ obj-y := filemap.o mempool.o oom_kill.o \ readahead.o swap.o truncate.o vmscan.o shmem.o \ util.o mmzone.o vmstat.o backing-dev.o \ mm_init.o mmu_context.o percpu.o slab_common.o \ - compaction.o vmacache.o \ + compaction.o vmacache.o swap_slots.o \ interval_tree.o list_lru.o workingset.o \ debug.o $(mmu-y)
mm/swap_slots.c new file +305 -0
305 lines not on this page · raw message
mm/swap_state.c +1 -0
@@ -18,6 +18,7 @@#include <linux/pagevec.h> #include <linux/migrate.h> #include <linux/vmalloc.h> +#include <linux/swap_slots.h> #include <asm/pgtable.h>
mm/swapfile.c +4 -11
@@ -34,6 +34,7 @@#include <linux/frontswap.h> #include <linux/swapfile.h> #include <linux/export.h> +#include <linux/swap_slots.h> #include <asm/pgtable.h> #include <asm/tlbflush.h>
@@ -880,14 +881,6 @@ noswap:return n_ret; } -swp_entry_t get_swap_page (void ) -{ -swp_entry_t entry; - -get_swap_pages (1 , &entry); -return entry; -} -/* The only caller of this function is now suspend routine */ swp_entry_t get_swap_page_of_type (int type) {
@@ -1078,7 +1071,7 @@ void swap_free(swp_entry_t entry) p =_swap_info_get (entry);if (p) {if (!__swap_entry_free (p, entry,1 )) -swapcache_free_entries (&entry,1 ); +free_swap_slot_to_cache (entry); } }
@@ -1092,7 +1085,7 @@ void swapcache_free(swp_entry_t entry) p =_swap_info_get (entry);if (p) {if (!__swap_entry_free (p, entry,SWAP_HAS_CACHE )) -swapcache_free_entries (&entry,1 ); +free_swap_slot_to_cache (entry); } }
@@ -1300,7 +1293,7 @@ int free_swap_and_cache(swp_entry_t entry) page =NULL ; } }else if (!count) -swapcache_free_entries (&entry,1 ); +free_swap_slot_to_cache (entry); }if (page) {/*
Signature · 7 lines
2.5.5 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
On Wednesday, September 28, 2016 1:19 AM Tim Chen wrote [...]
Quoted 25 lines
mm/swap_slots.c new file +25
-0+ +static int alloc_swap_slot_cache (int cpu) +{ +struct swap_slots_cache *cache; + + cache = &per_cpu (swp_slots, cpu); +mutex_init (&cache->alloc_lock); +spin_lock_init (&cache->free_lock); + cache->nr =0 ; + cache->cur =0 ; + cache->n_ret =0 ; + cache->slots =vzalloc (sizeof (swp_entry_t) *SWAP_SLOTS_CACHE_SIZE ); +if (!cache->slots) { + swap_slot_cache_enabled = false; +return -ENOMEM ; + } + cache->slots_ret =vzalloc (sizeof (swp_entry_t) *SWAP_SLOTS_CACHE_SIZE ); +if (!cache->slots_ret) { +vfree (cache->slots); + swap_slot_cache_enabled = false; +return -ENOMEM ; + } +return 0 ; +} +
[...]
mm/swap_slots.c line 178
Quoted 12 lines
mm/swap_slots.c new file +12
-0+ +static void free_slot_cache (int cpu) +{ +struct swap_slots_cache *cache; + +mutex_lock (&swap_slots_cache_mutex); +drain_slots_cache_cpu (cpu,SLOTS_CACHE |SLOTS_CACHE_RET ); + cache = &per_cpu (swp_slots, cpu); + cache->nr =0 ; + cache->cur =0 ; + cache->n_ret =0 ; +vfree (cache->slots);
Also free cache->slots_ret?
Or fold the relevant two allocations into one?
Review
Quoted 3 lines
(quoted diff) +2
-0+ mutex_unlock(&swap_slots_cache_mutex); +}
thanks
Hillf
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 30 lines · Hillf Danton
On Wednesday, September 28, 2016 1:19 AM Tim Chen wrote [...]
Quoted 26 lines
mm/swap_slots.c new file +25
-0+ +static int alloc_swap_slot_cache (int cpu) +{ +struct swap_slots_cache *cache; + + cache = &per_cpu (swp_slots, cpu); +mutex_init (&cache->alloc_lock); +spin_lock_init (&cache->free_lock); + cache->nr =0 ; + cache->cur =0 ; + cache->n_ret =0 ; + cache->slots =vzalloc (sizeof (swp_entry_t) *SWAP_SLOTS_CACHE_SIZE ); +if (!cache->slots) { + swap_slot_cache_enabled = false; +return -ENOMEM ; + } + cache->slots_ret =vzalloc (sizeof (swp_entry_t) *SWAP_SLOTS_CACHE_SIZE ); +if (!cache->slots_ret) { +vfree (cache->slots); + swap_slot_cache_enabled = false; +return -ENOMEM ; + } +return 0 ; +} +[...]
Quoted 1 line
Quoted 13 lines
mm/swap_slots.c line 178
Quoted 12 lines
mm/swap_slots.c new file +12
-0+ +static void free_slot_cache (int cpu) +{ +struct swap_slots_cache *cache; + +mutex_lock (&swap_slots_cache_mutex); +drain_slots_cache_cpu (cpu,SLOTS_CACHE |SLOTS_CACHE_RET ); + cache = &per_cpu (swp_slots, cpu); + cache->nr =0 ; + cache->cur =0 ; + cache->n_ret =0 ; +vfree (cache->slots);Also free
cache->slots_ret?
Good point. A Should free cache->slots_ret here.
Tim
Quoted 2 lines
A
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>