zbud: allow up to PAGE_SIZE allocations
Subsystems (2)
PATCH v2 1/1 zbud: allow up to PAGE_SIZE allocations
Currently zbud is only capable of allocating not more than PAGE_SIZE - ZHDR_SIZE_ALIGNED - CHUNK_SIZE. This is okay as long as only zswap is using it, but other users of zbud may (and likely will) want to allocate up to PAGE_SIZE. This patch addresses that by skipping the creation of zbud internal structure in the beginning of an allocated page (such pages are then called 'headless').
As a zbud page is no longer guaranteed to contain zbud header, the following changes had to be applied throughout the code:
* page->lru to be used for zbud page lists
* page->private to hold 'under_reclaim' flag
page->private will also be used to indicate if this page contains a zbud header in the beginning or not ('headless' flag).
--- mm/zbud.c | 194 +++++++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 128 insertions(+), 66 deletions(-)
mm/zbud.c +128 -66
@@ -105,18 +105,25 @@ struct zbud_pool {/* * struct zbud_header - zbud page metadata occupying the first chunk of each -* zbud page. +* zbud page, except for HEADLESS pages * @buddy: links the zbud page into the unbuddied/buddied lists in the pool -* @lru: links the zbud page into the lru list in the pool * @first_chunks: the size of the first buddy in chunks, 0 if free * @last_chunks: the size of the last buddy in chunks, 0 if free */ struct zbud_header {struct list_head buddy; -struct list_head lru;unsigned int first_chunks;unsigned int last_chunks; -bool under_reclaim; +}; + +/* +* struct zbud_page_priv - zbud flags to be stored in page->private +* @under_reclaim: if a zbud page is under reclaim +* @headless: indicates a page where zbud header didn't fit +*/ +struct zbud_page_priv { +bool under_reclaim:1 ; +bool headless:1 ; };/*****************
@@ -221,6 +228,7 @@ MODULE_ALIAS("zpool-zbud");*****************/ /* Just to make the code easier to read */ enum buddy { +HEADLESS ,FIRST ,LAST };
@@ -237,12 +245,15 @@ static int size_to_chunks(size_t size)/* Initializes the zbud header of a newly allocated zbud page */ static struct zbud_header *init_zbud_page (struct page *page) { +struct zbud_page_priv *ppriv = (struct zbud_page_priv *)page->private;struct zbud_header *zhdr =page_address (page); + +INIT_LIST_HEAD (&page->lru); + ppriv->under_reclaim =0 ; + zhdr->first_chunks =0 ; zhdr->last_chunks =0 ;INIT_LIST_HEAD (&zhdr->buddy); -INIT_LIST_HEAD (&zhdr->lru); - zhdr->under_reclaim =0 ;return zhdr; }
@@ -267,11 +278,22 @@ static unsigned long encode_handle(struct zbud_header *zhdr, enum buddy bud)* over the zbud header in the first chunk. */ handle = (unsigned long )zhdr; -if (bud ==FIRST ) +switch (bud) { +case FIRST :/* skip over zbud header */ handle +=ZHDR_SIZE_ALIGNED ; -else /* bud == LAST */ +break ; +case LAST : handle +=PAGE_SIZE - (zhdr->last_chunks <<CHUNK_SHIFT ); +break ; +case HEADLESS : +break ; +default : +/* this should never happen */ +pr_err ("zbud: invalid buddy value %d\n" , bud); + handle =0 ; +break ; + }return handle; }
@@ -287,6 +309,7 @@ static int num_free_chunks(struct zbud_header *zhdr)/* * Rather than branch for different situations, just use the fact that * free buddies have a length of zero to simplify everything. +* NB: can't be used with HEADLESS pages. */ return NCHUNKS - zhdr->first_chunks - zhdr->last_chunks; }
@@ -353,31 +376,40 @@ void zbud_destroy_pool(struct zbud_pool *pool)int zbud_alloc (struct zbud_pool *pool,size_t size,gfp_t gfp,unsigned long *handle) { -int chunks, i, freechunks; +int chunks =0 , i, freechunks;struct zbud_header *zhdr =NULL ; +struct zbud_page_priv *ppriv;enum buddy bud;struct page *page;if (!size || (gfp & __GFP_HIGHMEM))return -EINVAL ; -if (size >PAGE_SIZE -ZHDR_SIZE_ALIGNED -CHUNK_SIZE ) + +if (size >PAGE_SIZE )return -ENOSPC ; - chunks =size_to_chunks (size); -spin_lock (&pool->lock); -/* First, try to find an unbuddied zbud page. */ - zhdr =NULL ; -for_each_unbuddied_list (i, chunks) { -if (!list_empty (&pool->unbuddied[i])) { - zhdr =list_first_entry (&pool->unbuddied[i], - struct zbud_header, buddy); -list_del (&zhdr->buddy); -if (zhdr->first_chunks ==0 ) - bud =FIRST ; -else - bud =LAST ; - goto found; +if (size >PAGE_SIZE -ZHDR_SIZE_ALIGNED -CHUNK_SIZE ) + bud =HEADLESS ; +else { + chunks =size_to_chunks (size); +spin_lock (&pool->lock); + +/* First, try to find an unbuddied zbud page. */ + zhdr =NULL ; +for_each_unbuddied_list (i, chunks) { +if (!list_empty (&pool->unbuddied[i])) { + zhdr =list_first_entry (&pool->unbuddied[i], + struct zbud_header, buddy); +list_del (&zhdr->buddy); + page =virt_to_page (zhdr); +if (zhdr->first_chunks ==0 ) + bud =FIRST ; +else + bud =LAST ; + goto found; + } } + bud =FIRST ; }/* Couldn't find unbuddied zbud page, create new one */
@@ -388,27 +420,31 @@ int zbud_alloc(struct zbud_pool *pool, size_t size, gfp_t gfp,spin_lock (&pool->lock); pool->pages_nr++; zhdr =init_zbud_page (page); - bud =FIRST ; found: -if (bud ==FIRST ) - zhdr->first_chunks = chunks; -else - zhdr->last_chunks = chunks; + ppriv = (struct zbud_page_priv *)page->private; +if (bud !=HEADLESS ) { +if (bud ==FIRST ) + zhdr->first_chunks = chunks; +else + zhdr->last_chunks = chunks; + +if (zhdr->first_chunks ==0 || zhdr->last_chunks ==0 ) { +/* Add to unbuddied list */ + freechunks =num_free_chunks (zhdr); +list_add (&zhdr->buddy, &pool->unbuddied[freechunks]); + }else { +/* Add to buddied list */ +list_add (&zhdr->buddy, &pool->buddied); + } -if (zhdr->first_chunks ==0 || zhdr->last_chunks ==0 ) { -/* Add to unbuddied list */ - freechunks =num_free_chunks (zhdr); -list_add (&zhdr->buddy, &pool->unbuddied[freechunks]); - }else { -/* Add to buddied list */ -list_add (&zhdr->buddy, &pool->buddied); - } +/* Add/move zbud page to beginning of LRU */ +if (!list_empty (&page->lru)) +list_del (&page->lru); + }else + ppriv->headless = true; -/* Add/move zbud page to beginning of LRU */ -if (!list_empty (&zhdr->lru)) -list_del (&zhdr->lru); -list_add (&zhdr->lru, &pool->lru); +list_add (&page->lru, &pool->lru); *handle =encode_handle (zhdr, bud);spin_unlock (&pool->lock);
@@ -430,28 +466,41 @@ void zbud_free(struct zbud_pool *pool, unsigned long handle) {struct zbud_header *zhdr;int freechunks; +struct page *page; +enum buddy bud; +struct zbud_page_priv *ppriv;spin_lock (&pool->lock); zhdr =handle_to_zbud_header (handle); + page =virt_to_page (zhdr); + ppriv = (struct zbud_page_priv *)page->private; -/* If first buddy, handle will be page aligned */ -if ((handle -ZHDR_SIZE_ALIGNED ) & ~PAGE_MASK ) +if (!(handle & ~PAGE_MASK ))/* HEADLESS page stored */ + bud =HEADLESS ; +else if ((handle -ZHDR_SIZE_ALIGNED ) & ~PAGE_MASK ) { + bud =LAST ; zhdr->last_chunks =0 ; -else + }else { +/* If first buddy, handle will be page aligned */ + bud =FIRST ; zhdr->first_chunks =0 ; + } -if (zhdr->under_reclaim) { +if (ppriv->under_reclaim) {/* zbud page is under reclaim, reclaim will free */ spin_unlock (&pool->lock);return ; } -/* Remove from existing buddy list */ -list_del (&zhdr->buddy); +if (bud !=HEADLESS ) { +/* Remove from existing buddy list */ +list_del (&zhdr->buddy); + } -if (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 ) { +if (bud ==HEADLESS || + (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 )) {/* zbud page is empty, free */ -list_del (&zhdr->lru); +list_del (&page->lru);free_zbud_page (zhdr); pool->pages_nr--; }else {
@@ -505,6 +554,8 @@ int zbud_reclaim_page(struct zbud_pool *pool, unsigned int retries) {int i, ret, freechunks;struct zbud_header *zhdr; +struct page *page; +struct zbud_page_priv *ppriv;unsigned long first_handle =0 , last_handle =0 ;spin_lock (&pool->lock);
@@ -514,21 +565,31 @@ int zbud_reclaim_page(struct zbud_pool *pool, unsigned int retries)return -EINVAL ; }for (i =0 ; i < retries; i++) { - zhdr = list_tail_entry(&pool->lru, struct zbud_header, lru); -list_del (&zhdr->lru); -list_del (&zhdr->buddy); + page = list_tail_entry(&pool->lru, struct page, lru); + ppriv = (struct zbud_page_priv *)page->private; +list_del (&page->lru); +/* Protect zbud page against free */ - zhdr->under_reclaim = true; -/* -* We need encode the handles before unlocking, since we can -* race with free that will set (first|last)_chunks to 0 -*/ - first_handle =0 ; - last_handle =0 ; -if (zhdr->first_chunks) - first_handle =encode_handle (zhdr,FIRST ); -if (zhdr->last_chunks) - last_handle =encode_handle (zhdr,LAST ); + ppriv->under_reclaim = true; + zhdr =page_address (page); +if (!ppriv->headless) { +list_del (&zhdr->buddy); +/* +* We need encode the handles before unlocking, since +* we can race with free that will set +* (first|last)_chunks to 0 +*/ + first_handle =0 ; + last_handle =0 ; +if (zhdr->first_chunks) + first_handle =encode_handle (zhdr,FIRST ); +if (zhdr->last_chunks) + last_handle =encode_handle (zhdr,LAST ); + }else { + first_handle =encode_handle (zhdr,HEADLESS ); + last_handle =0 ; + } +spin_unlock (&pool->lock);/* Issue the eviction callback(s) */
@@ -544,8 +605,9 @@ int zbud_reclaim_page(struct zbud_pool *pool, unsigned int retries) } next:spin_lock (&pool->lock); - zhdr->under_reclaim = false; -if (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 ) { + ppriv->under_reclaim = false; +if (ppriv->headless || + (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 )) {/* * Both buddies are now free, free the zbud page and * return success.
@@ -565,7 +627,7 @@ next: }/* add to beginning of LRU */ -list_add (&zhdr->lru, &pool->lru); +list_add (&page->lru, &pool->lru); }spin_unlock (&pool->lock);return -EAGAIN ;
Signature · 7 lines
2.4.2 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 26 lines · Vitaly Wool
Currently zbud is only capable of allocating not more than
PAGE_SIZE-ZHDR_SIZE_ALIGNED-CHUNK_SIZE. This is okay as long as only zswap is using it, but other users of zbud may (and likely will) want to allocate up toPAGE_SIZE. This patch addresses that by skipping the creation of zbud internal structure in the beginning of an allocated page (such pages are then called 'headless').As a zbud page is no longer guaranteed to contain zbud header, the following changes had to be applied throughout the code:
*page->lruto be used for zbud page lists
*page->privateto hold 'under_reclaim' flag
page->privatewill also be used to indicate if this page contains a zbud header in the beginning or not ('headless' flag).Signed-off-by Vitaly Wool <[email protected]>--- mm/zbud.c | 194 +++++++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 128 insertions(+), 66 deletions(-)mm/zbud.c +0
-0
mm/zbud.c struct zbud_pool { · line 108
Quoted 5 lines
mm/zbud.c +1
-1@@ -105,18 +105,25 @@ struct zbud_pool {/* * struct zbud_header - zbud page metadata occupying the first chunk of each -* zbud page. +* zbud page, except for HEADLESS pages
hmm, personally I like "FULL" better than HEADLESS...it's only headless because it's a full page.
Quoted 12 lines
mm/zbud.c +1
-3* @buddy: links the zbud page into the unbuddied/buddied lists in the pool -* @lru: links the zbud page into the lru list in the pool * @first_chunks: the size of the first buddy in chunks, 0 if free * @last_chunks: the size of the last buddy in chunks, 0 if free */ struct zbud_header {struct list_head buddy; -struct list_head lru;unsigned int first_chunks;unsigned int last_chunks; -bool under_reclaim; +};
mm/zbud.c struct zbud_pool { · line 127
Quoted 10 lines
mm/zbud.c +9
-0+ +/* +* struct zbud_page_priv - zbud flags to be stored in page->private +* @under_reclaim: if a zbud page is under reclaim +* @headless: indicates a page where zbud header didn't fit +*/ +struct zbud_page_priv { +bool under_reclaim:1 ; +bool headless:1 ; };
Hmm, this is just my personal opinion, but I'm not a fan of casting ->private as a struct, if we're only using it as a bitmap. I'd suggest just defining bits as an enum (like page flags), e.g.
enum zbud_flags {
ZBUD_UNDER_RECLAIM,
ZBUD_FULL_PAGE,
};
or some names similar to that. then it can be checked with a simple test_bit() call, and set_bit()/clear_bit().
alternately, there are the already-existing PG_private and PG_private_2 bits in the page flags...but unless we need ->private for something else, it probably makes more sense to just use it instead of the PG_private flags.
Quoted 11 lines
mm/zbud.c +1
-0/***************** @@ -221,6 +228,7 @@ MODULE_ALIAS("zpool-zbud");*****************/ /* Just to make the code easier to read */ enum buddy { +HEADLESS ,FIRST ,LAST };
mm/zbud.c static int size_to_chunks(size_t size) · line 252
Quoted 8 lines
mm/zbud.c +4
-0@@ -237,12 +245,15 @@ static int size_to_chunks(size_t size)/* Initializes the zbud header of a newly allocated zbud page */ static struct zbud_header *init_zbud_page (struct page *page) { +struct zbud_page_priv *ppriv = (struct zbud_page_priv *)page->private;struct zbud_header *zhdr =page_address (page); + +INIT_LIST_HEAD (&page->lru); + ppriv->under_reclaim =0 ;
don't forget to initialize the headless/full bit too. (I'll mention that the page allocation code does clear ->private before handing it to us, so it should already be 0. but let's not rely on that)
Quoted 102 lines
mm/zbud.c +40
-20+ zhdr->first_chunks =0 ; zhdr->last_chunks =0 ;INIT_LIST_HEAD (&zhdr->buddy); -INIT_LIST_HEAD (&zhdr->lru); - zhdr->under_reclaim =0 ;return zhdr; }@@ -267,11 +278,22 @@ static unsigned long encode_handle(struct zbud_header *zhdr, enum buddy bud)* over the zbud header in the first chunk. */ handle = (unsigned long )zhdr; -if (bud ==FIRST ) +switch (bud) { +case FIRST :/* skip over zbud header */ handle +=ZHDR_SIZE_ALIGNED ; -else /* bud == LAST */ +break ; +case LAST : handle +=PAGE_SIZE - (zhdr->last_chunks <<CHUNK_SHIFT ); +break ; +case HEADLESS : +break ; +default : +/* this should never happen */ +pr_err ("zbud: invalid buddy value %d\n" , bud); + handle =0 ; +break ; + }return handle; }@@ -287,6 +309,7 @@ static int num_free_chunks(struct zbud_header *zhdr)/* * Rather than branch for different situations, just use the fact that * free buddies have a length of zero to simplify everything. +* NB: can't be used with HEADLESS pages. */ return NCHUNKS - zhdr->first_chunks - zhdr->last_chunks; }@@ -353,31 +376,40 @@ void zbud_destroy_pool(struct zbud_pool *pool)int zbud_alloc (struct zbud_pool *pool,size_t size,gfp_t gfp,unsigned long *handle) { -int chunks, i, freechunks; +int chunks =0 , i, freechunks;struct zbud_header *zhdr =NULL ; +struct zbud_page_priv *ppriv;enum buddy bud;struct page *page;if (!size || (gfp & __GFP_HIGHMEM))return -EINVAL ; -if (size >PAGE_SIZE -ZHDR_SIZE_ALIGNED -CHUNK_SIZE ) + +if (size >PAGE_SIZE )return -ENOSPC ; - chunks =size_to_chunks (size); -spin_lock (&pool->lock); -/* First, try to find an unbuddied zbud page. */ - zhdr =NULL ; -for_each_unbuddied_list (i, chunks) { -if (!list_empty (&pool->unbuddied[i])) { - zhdr =list_first_entry (&pool->unbuddied[i], - struct zbud_header, buddy); -list_del (&zhdr->buddy); -if (zhdr->first_chunks ==0 ) - bud =FIRST ; -else - bud =LAST ; - goto found; +if (size >PAGE_SIZE -ZHDR_SIZE_ALIGNED -CHUNK_SIZE ) + bud =HEADLESS ; +else { + chunks =size_to_chunks (size); +spin_lock (&pool->lock); + +/* First, try to find an unbuddied zbud page. */ + zhdr =NULL ; +for_each_unbuddied_list (i, chunks) { +if (!list_empty (&pool->unbuddied[i])) { + zhdr =list_first_entry (&pool->unbuddied[i], + struct zbud_header, buddy); +list_del (&zhdr->buddy); + page =virt_to_page (zhdr); +if (zhdr->first_chunks ==0 ) + bud =FIRST ; +else + bud =LAST ; + goto found; + } } + bud =FIRST ; }/* Couldn't find unbuddied zbud page, create new one */ @@ -388,27 +420,31 @@ int zbud_alloc(struct zbud_pool *pool, size_t size, gfp_t gfp,spin_lock (&pool->lock); pool->pages_nr++;
mm/zbud.c int zbud_alloc(struct zbud_pool *pool, size_t size, gfp_t gfp, · line 426
Quoted 10 lines
mm/zbud.c +2
-5zhdr =init_zbud_page (page); - bud =FIRST ; found: -if (bud ==FIRST ) - zhdr->first_chunks = chunks; -else - zhdr->last_chunks = chunks; + ppriv = (struct zbud_page_priv *)page->private; +if (bud !=HEADLESS ) {
another personal opinion...it might look simpler if you just do:
if (bud == FULL) {
set_bit(ZBUD_FULL_PAGE, &page->private);
goto full;
}
Quoted 18 lines
mm/zbud.c +13
-4+if (bud ==FIRST ) + zhdr->first_chunks = chunks; +else + zhdr->last_chunks = chunks; + +if (zhdr->first_chunks ==0 || zhdr->last_chunks ==0 ) { +/* Add to unbuddied list */ + freechunks =num_free_chunks (zhdr); +list_add (&zhdr->buddy, &pool->unbuddied[freechunks]); + }else { +/* Add to buddied list */ +list_add (&zhdr->buddy, &pool->buddied); + } -if (zhdr->first_chunks ==0 || zhdr->last_chunks ==0 ) { -/* Add to unbuddied list */ - freechunks =num_free_chunks (zhdr); -list_add (&zhdr->buddy, &pool->unbuddied[freechunks]);
mm/zbud.c int zbud_alloc(struct zbud_pool *pool, size_t size, gfp_t gfp, · line 446
Quoted 10 lines
mm/zbud.c +5
-4- }else { -/* Add to buddied list */ -list_add (&zhdr->buddy, &pool->buddied); - } +/* Add/move zbud page to beginning of LRU */ +if (!list_empty (&page->lru)) +list_del (&page->lru); + }else + ppriv->headless = true;
insert label here...
full:
mm/zbud.c int zbud_alloc(struct zbud_pool *pool, size_t size, gfp_t gfp, · line 410
Quoted 3 lines
mm/zbud.c +0
-3-/* Add/move zbud page to beginning of LRU */ -if (!list_empty (&zhdr->lru)) -list_del (&zhdr->lru);
no matter the case, this is safe; leave it here for simplicity. a full page will have a empty list head.
Quoted 111 lines
mm/zbud.c +48
-23-list_add (&zhdr->lru, &pool->lru); +list_add (&page->lru, &pool->lru); *handle =encode_handle (zhdr, bud);spin_unlock (&pool->lock);@@ -430,28 +466,41 @@ void zbud_free(struct zbud_pool *pool, unsigned long handle) {struct zbud_header *zhdr;int freechunks; +struct page *page; +enum buddy bud; +struct zbud_page_priv *ppriv;spin_lock (&pool->lock); zhdr =handle_to_zbud_header (handle); + page =virt_to_page (zhdr); + ppriv = (struct zbud_page_priv *)page->private; -/* If first buddy, handle will be page aligned */ -if ((handle -ZHDR_SIZE_ALIGNED ) & ~PAGE_MASK ) +if (!(handle & ~PAGE_MASK ))/* HEADLESS page stored */ + bud =HEADLESS ; +else if ((handle -ZHDR_SIZE_ALIGNED ) & ~PAGE_MASK ) { + bud =LAST ; zhdr->last_chunks =0 ; -else + }else { +/* If first buddy, handle will be page aligned */ + bud =FIRST ; zhdr->first_chunks =0 ; + } -if (zhdr->under_reclaim) { +if (ppriv->under_reclaim) {/* zbud page is under reclaim, reclaim will free */ spin_unlock (&pool->lock);return ; } -/* Remove from existing buddy list */ -list_del (&zhdr->buddy); +if (bud !=HEADLESS ) { +/* Remove from existing buddy list */ +list_del (&zhdr->buddy); + } -if (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 ) { +if (bud ==HEADLESS || + (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 )) {/* zbud page is empty, free */ -list_del (&zhdr->lru); +list_del (&page->lru);free_zbud_page (zhdr); pool->pages_nr--; }else {@@ -505,6 +554,8 @@ int zbud_reclaim_page(struct zbud_pool *pool, unsigned int retries) {int i, ret, freechunks;struct zbud_header *zhdr; +struct page *page; +struct zbud_page_priv *ppriv;unsigned long first_handle =0 , last_handle =0 ;spin_lock (&pool->lock);@@ -514,21 +565,31 @@ int zbud_reclaim_page(struct zbud_pool *pool, unsigned int retries)return -EINVAL ; }for (i =0 ; i < retries; i++) { - zhdr = list_tail_entry(&pool->lru, struct zbud_header, lru); -list_del (&zhdr->lru); -list_del (&zhdr->buddy); + page = list_tail_entry(&pool->lru, struct page, lru); + ppriv = (struct zbud_page_priv *)page->private; +list_del (&page->lru); +/* Protect zbud page against free */ - zhdr->under_reclaim = true; -/* -* We need encode the handles before unlocking, since we can -* race with free that will set (first|last)_chunks to 0 -*/ - first_handle =0 ; - last_handle =0 ; -if (zhdr->first_chunks) - first_handle =encode_handle (zhdr,FIRST ); -if (zhdr->last_chunks) - last_handle =encode_handle (zhdr,LAST ); + ppriv->under_reclaim = true; + zhdr =page_address (page); +if (!ppriv->headless) { +list_del (&zhdr->buddy); +/* +* We need encode the handles before unlocking, since +* we can race with free that will set +* (first|last)_chunks to 0 +*/ + first_handle =0 ; + last_handle =0 ; +if (zhdr->first_chunks) + first_handle =encode_handle (zhdr,FIRST ); +if (zhdr->last_chunks) + last_handle =encode_handle (zhdr,LAST ); + }else { + first_handle =encode_handle (zhdr,HEADLESS ); + last_handle =0 ; + } +spin_unlock (&pool->lock);/* Issue the eviction callback(s) */
mm/zbud.c int zbud_reclaim_page(struct zbud_pool *pool, unsigned int retries) · line 610
Quoted 8 lines
mm/zbud.c +3
-2@@ -544,8 +605,9 @@ int zbud_reclaim_page(struct zbud_pool *pool, unsigned int retries) } next:spin_lock (&pool->lock); - zhdr->under_reclaim = false; -if (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 ) { + ppriv->under_reclaim = false; +if (ppriv->headless || + (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 )) {
oops, this isn't right. you're assuming ->headless means the eviction succeeded. However if pool->ops->evict() failed, that isn't true.
Quoted 13 lines
mm/zbud.c +1
-2/* * Both buddies are now free, free the zbud page and * return success. @@ -565,7 +627,7 @@ next: }/* add to beginning of LRU */ -list_add (&zhdr->lru, &pool->lru); +list_add (&page->lru, &pool->lru); }spin_unlock (&pool->lock);return -EAGAIN ; --
Quoted 2 lines
2.4.2
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 28 lines · Dan Streetman
Quoted 27 lines · Vitaly Wool
Currently zbud is only capable of allocating not more than
PAGE_SIZE-ZHDR_SIZE_ALIGNED-CHUNK_SIZE. This is okay as long as only zswap is using it, but other users of zbud may (and likely will) want to allocate up toPAGE_SIZE. This patch addresses that by skipping the creation of zbud internal structure in the beginning of an allocated page (such pages are then called 'headless').As a zbud page is no longer guaranteed to contain zbud header, the following changes had to be applied throughout the code:
*page->lruto be used for zbud page lists
*page->privateto hold 'under_reclaim' flag
page->privatewill also be used to indicate if this page contains a zbud header in the beginning or not ('headless' flag).Signed-off-by Vitaly Wool <[email protected]>--- mm/zbud.c | 194 +++++++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 128 insertions(+), 66 deletions(-)mm/zbud.c +0
-0@@ -105,18 +105,25 @@ struct zbud_pool {
Quoted 7 lines
mm/zbud.c struct zbud_pool { · line 108
Quoted 4 lines
mm/zbud.c +1
-1/* * struct zbud_header - zbud page metadata occupying the first chunk of each -* zbud page. +* zbud page, except for HEADLESS pages hmm, personally I like "FULL" better than HEADLESS...it's only headless because it's a full page.
Yep, but OTOH, a page with two buddies is also full, so I just wanted to avoid ambiguity here. I have no personal naming preference TBO. :)
Quoted 27 lines
Quoted 12 lines
mm/zbud.c +1
-3* @buddy: links the zbud page into the unbuddied/buddied lists in the pool -* @lru: links the zbud page into the lru list in the pool * @first_chunks: the size of the first buddy in chunks, 0 if free * @last_chunks: the size of the last buddy in chunks, 0 if free */ struct zbud_header {struct list_head buddy; -struct list_head lru;unsigned int first_chunks;unsigned int last_chunks; -bool under_reclaim; +};mm/zbud.c struct zbud_pool { · line 127
Quoted 10 lines
mm/zbud.c +9
-0+ +/* +* struct zbud_page_priv - zbud flags to be stored in page->private +* @under_reclaim: if a zbud page is under reclaim +* @headless: indicates a page where zbud header didn't fit +*/ +struct zbud_page_priv { +bool under_reclaim:1 ; +bool headless:1 ; };Hmm, this is just my personal opinion, but I'm not a fan of casting ->private as a struct, if we're only using it as a bitmap. I'd suggest just defining bits as an enum (like page flags), e.g.
Quoted 4 lines
enum
zbud_flags{
ZBUD_UNDER_RECLAIM,
ZBUD_FULL_PAGE,
};
Agreed, will rework that.
Quoted 7 lines
or some names similar to that. then it can be checked with a simple
test_bit()call, and set_bit()/clear_bit().alternately, there are the already-existing
PG_privateandPG_private_2bits in the page flags...but unless we need ->private for something else, it probably makes more sense to just use it instead of thePG_privateflags.
IIRC I used PG_private in my first patch but the thing also is, it's safer to have under_reclaim flag in struct page, too, so page->private seems to be a better fit.
Quoted 139 lines
Quoted 11 lines
mm/zbud.c +1
-0/***************** @@ -221,6 +228,7 @@ MODULE_ALIAS("zpool-zbud");*****************/ /* Just to make the code easier to read */ enum buddy { +HEADLESS ,FIRST ,LAST };mm/zbud.c static int size_to_chunks(size_t size) · line 252
Quoted 8 lines
mm/zbud.c +4
-0@@ -237,12 +245,15 @@ static int size_to_chunks(size_t size)/* Initializes the zbud header of a newly allocated zbud page */ static struct zbud_header *init_zbud_page (struct page *page) { +struct zbud_page_priv *ppriv = (struct zbud_page_priv *)page->private;struct zbud_header *zhdr =page_address (page); + +INIT_LIST_HEAD (&page->lru); + ppriv->under_reclaim =0 ;don't forget to initialize the headless/full bit too. (I'll mention that the page allocation code does clear ->private before handing it to us, so it should already be 0. but let's not rely on that)
Quoted 102 lines
mm/zbud.c +40
-20+ zhdr->first_chunks =0 ; zhdr->last_chunks =0 ;INIT_LIST_HEAD (&zhdr->buddy); -INIT_LIST_HEAD (&zhdr->lru); - zhdr->under_reclaim =0 ;return zhdr; }@@ -267,11 +278,22 @@ static unsigned long encode_handle(struct zbud_header *zhdr, enum buddy bud)* over the zbud header in the first chunk. */ handle = (unsigned long )zhdr; -if (bud ==FIRST ) +switch (bud) { +case FIRST :/* skip over zbud header */ handle +=ZHDR_SIZE_ALIGNED ; -else /* bud == LAST */ +break ; +case LAST : handle +=PAGE_SIZE - (zhdr->last_chunks <<CHUNK_SHIFT ); +break ; +case HEADLESS : +break ; +default : +/* this should never happen */ +pr_err ("zbud: invalid buddy value %d\n" , bud); + handle =0 ; +break ; + }return handle; }@@ -287,6 +309,7 @@ static int num_free_chunks(struct zbud_header *zhdr)/* * Rather than branch for different situations, just use the fact that * free buddies have a length of zero to simplify everything. +* NB: can't be used with HEADLESS pages. */ return NCHUNKS - zhdr->first_chunks - zhdr->last_chunks; }@@ -353,31 +376,40 @@ void zbud_destroy_pool(struct zbud_pool *pool)int zbud_alloc (struct zbud_pool *pool,size_t size,gfp_t gfp,unsigned long *handle) { -int chunks, i, freechunks; +int chunks =0 , i, freechunks;struct zbud_header *zhdr =NULL ; +struct zbud_page_priv *ppriv;enum buddy bud;struct page *page;if (!size || (gfp & __GFP_HIGHMEM))return -EINVAL ; -if (size >PAGE_SIZE -ZHDR_SIZE_ALIGNED -CHUNK_SIZE ) + +if (size >PAGE_SIZE )return -ENOSPC ; - chunks =size_to_chunks (size); -spin_lock (&pool->lock); -/* First, try to find an unbuddied zbud page. */ - zhdr =NULL ; -for_each_unbuddied_list (i, chunks) { -if (!list_empty (&pool->unbuddied[i])) { - zhdr =list_first_entry (&pool->unbuddied[i], - struct zbud_header, buddy); -list_del (&zhdr->buddy); -if (zhdr->first_chunks ==0 ) - bud =FIRST ; -else - bud =LAST ; - goto found; +if (size >PAGE_SIZE -ZHDR_SIZE_ALIGNED -CHUNK_SIZE ) + bud =HEADLESS ; +else { + chunks =size_to_chunks (size); +spin_lock (&pool->lock); + +/* First, try to find an unbuddied zbud page. */ + zhdr =NULL ; +for_each_unbuddied_list (i, chunks) { +if (!list_empty (&pool->unbuddied[i])) { + zhdr =list_first_entry (&pool->unbuddied[i], + struct zbud_header, buddy); +list_del (&zhdr->buddy); + page =virt_to_page (zhdr); +if (zhdr->first_chunks ==0 ) + bud =FIRST ; +else + bud =LAST ; + goto found; + } } + bud =FIRST ; }/* Couldn't find unbuddied zbud page, create new one */ @@ -388,27 +420,31 @@ int zbud_alloc(struct zbud_pool *pool, size_t size, gfp_t gfp,spin_lock (&pool->lock); pool->pages_nr++;mm/zbud.c int zbud_alloc(struct zbud_pool *pool, size_t size, gfp_t gfp, · line 426
Quoted 10 lines
mm/zbud.c +2
-5zhdr =init_zbud_page (page); - bud =FIRST ; found: -if (bud ==FIRST ) - zhdr->first_chunks = chunks; -else - zhdr->last_chunks = chunks; + ppriv = (struct zbud_page_priv *)page->private; +if (bud !=HEADLESS ) {another personal opinion...it might look simpler if you just do:
Quoted 4 lines
if (bud == FULL) {
set_bit(ZBUD_FULL_PAGE, &page->private);
goto full;
}
Agreed here too.
Quoted 149 lines
Quoted 18 lines
mm/zbud.c +13
-4+if (bud ==FIRST ) + zhdr->first_chunks = chunks; +else + zhdr->last_chunks = chunks; + +if (zhdr->first_chunks ==0 || zhdr->last_chunks ==0 ) { +/* Add to unbuddied list */ + freechunks =num_free_chunks (zhdr); +list_add (&zhdr->buddy, &pool->unbuddied[freechunks]); + }else { +/* Add to buddied list */ +list_add (&zhdr->buddy, &pool->buddied); + } -if (zhdr->first_chunks ==0 || zhdr->last_chunks ==0 ) { -/* Add to unbuddied list */ - freechunks =num_free_chunks (zhdr); -list_add (&zhdr->buddy, &pool->unbuddied[freechunks]);mm/zbud.c int zbud_alloc(struct zbud_pool *pool, size_t size, gfp_t gfp, · line 446
Quoted 10 lines
mm/zbud.c +5
-4- }else { -/* Add to buddied list */ -list_add (&zhdr->buddy, &pool->buddied); - } +/* Add/move zbud page to beginning of LRU */ +if (!list_empty (&page->lru)) +list_del (&page->lru); + }else + ppriv->headless = true;insert label here...
full:
mm/zbud.c int zbud_alloc(struct zbud_pool *pool, size_t size, gfp_t gfp, · line 410
Quoted 3 lines
mm/zbud.c +0
-3-/* Add/move zbud page to beginning of LRU */ -if (!list_empty (&zhdr->lru)) -list_del (&zhdr->lru);no matter the case, this is safe; leave it here for simplicity. a full page will have a empty list head.
Quoted 109 lines
mm/zbud.c +48
-23-list_add (&zhdr->lru, &pool->lru); +list_add (&page->lru, &pool->lru); *handle =encode_handle (zhdr, bud);spin_unlock (&pool->lock);@@ -430,28 +466,41 @@ void zbud_free(struct zbud_pool *pool, unsigned long handle) {struct zbud_header *zhdr;int freechunks; +struct page *page; +enum buddy bud; +struct zbud_page_priv *ppriv;spin_lock (&pool->lock); zhdr =handle_to_zbud_header (handle); + page =virt_to_page (zhdr); + ppriv = (struct zbud_page_priv *)page->private; -/* If first buddy, handle will be page aligned */ -if ((handle -ZHDR_SIZE_ALIGNED ) & ~PAGE_MASK ) +if (!(handle & ~PAGE_MASK ))/* HEADLESS page stored */ + bud =HEADLESS ; +else if ((handle -ZHDR_SIZE_ALIGNED ) & ~PAGE_MASK ) { + bud =LAST ; zhdr->last_chunks =0 ; -else + }else { +/* If first buddy, handle will be page aligned */ + bud =FIRST ; zhdr->first_chunks =0 ; + } -if (zhdr->under_reclaim) { +if (ppriv->under_reclaim) {/* zbud page is under reclaim, reclaim will free */ spin_unlock (&pool->lock);return ; } -/* Remove from existing buddy list */ -list_del (&zhdr->buddy); +if (bud !=HEADLESS ) { +/* Remove from existing buddy list */ +list_del (&zhdr->buddy); + } -if (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 ) { +if (bud ==HEADLESS || + (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 )) {/* zbud page is empty, free */ -list_del (&zhdr->lru); +list_del (&page->lru);free_zbud_page (zhdr); pool->pages_nr--; }else {@@ -505,6 +554,8 @@ int zbud_reclaim_page(struct zbud_pool *pool, unsigned int retries) {int i, ret, freechunks;struct zbud_header *zhdr; +struct page *page; +struct zbud_page_priv *ppriv;unsigned long first_handle =0 , last_handle =0 ;spin_lock (&pool->lock);@@ -514,21 +565,31 @@ int zbud_reclaim_page(struct zbud_pool *pool, unsigned int retries)return -EINVAL ; }for (i =0 ; i < retries; i++) { - zhdr = list_tail_entry(&pool->lru, struct zbud_header, lru); -list_del (&zhdr->lru); -list_del (&zhdr->buddy); + page = list_tail_entry(&pool->lru, struct page, lru); + ppriv = (struct zbud_page_priv *)page->private; +list_del (&page->lru); +/* Protect zbud page against free */ - zhdr->under_reclaim = true; -/* -* We need encode the handles before unlocking, since we can -* race with free that will set (first|last)_chunks to 0 -*/ - first_handle =0 ; - last_handle =0 ; -if (zhdr->first_chunks) - first_handle =encode_handle (zhdr,FIRST ); -if (zhdr->last_chunks) - last_handle =encode_handle (zhdr,LAST ); + ppriv->under_reclaim = true; + zhdr =page_address (page); +if (!ppriv->headless) { +list_del (&zhdr->buddy); +/* +* We need encode the handles before unlocking, since +* we can race with free that will set +* (first|last)_chunks to 0 +*/ + first_handle =0 ; + last_handle =0 ; +if (zhdr->first_chunks) + first_handle =encode_handle (zhdr,FIRST ); +if (zhdr->last_chunks) + last_handle =encode_handle (zhdr,LAST ); + }else { + first_handle =encode_handle (zhdr,HEADLESS ); + last_handle =0 ; + } +spin_unlock (&pool->lock);
Quoted 13 lines
mm/zbud.c int zbud_reclaim_page(struct zbud_pool *pool, unsigned int retries) · line 610
Quoted 10 lines
mm/zbud.c +3
-2/* Issue the eviction callback(s) */ @@ -544,8 +605,9 @@ int zbud_reclaim_page(struct zbud_pool *pool, unsigned int retries) } next:spin_lock (&pool->lock); - zhdr->under_reclaim = false; -if (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 ) { + ppriv->under_reclaim = false; +if (ppriv->headless || + (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 )) {oops, this isn't right. you're assuming ->headless means the eviction succeeded. However if
pool->ops->evict()failed, that isn't true.
Aha, good catch, thanks. Then it looks like we need separate handling for the 'headless' case here, e. g.
if (ppriv->headless) {
if (ret == 0) {
free_zbud_page(zhdr);
pool->pages_nr--;
spin_unlock(&pool->lock);
}
} else if (zhdr->first_chunks == 0 && zhdr->last_chunks == 0) {...
The block under 'if (ret == 0)' may also become a static inline to avoid code duplication.
~vitaly
Quoted 389 lines · Dan Streetman
Quoted 26 lines · Vitaly Wool
Currently zbud is only capable of allocating not more than
PAGE_SIZE-ZHDR_SIZE_ALIGNED-CHUNK_SIZE. This is okay as long as only zswap is using it, but other users of zbud may (and likely will) want to allocate up toPAGE_SIZE. This patch addresses that by skipping the creation of zbud internal structure in the beginning of an allocated page (such pages are then called 'headless').As a zbud page is no longer guaranteed to contain zbud header, the following changes had to be applied throughout the code:
*page->lruto be used for zbud page lists
*page->privateto hold 'under_reclaim' flag
page->privatewill also be used to indicate if this page contains a zbud header in the beginning or not ('headless' flag).Signed-off-by Vitaly Wool <[email protected]>--- mm/zbud.c | 194 +++++++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 128 insertions(+), 66 deletions(-)mm/zbud.c +0
-0mm/zbud.c struct zbud_pool { · line 108
Quoted 5 lines
mm/zbud.c +1
-1@@ -105,18 +105,25 @@ struct zbud_pool {/* * struct zbud_header - zbud page metadata occupying the first chunk of each -* zbud page. +* zbud page, except for HEADLESS pages hmm, personally I like "FULL" better than HEADLESS...it's only headless because it's a full page.
Quoted 12 lines
mm/zbud.c +1
-3* @buddy: links the zbud page into the unbuddied/buddied lists in the pool -* @lru: links the zbud page into the lru list in the pool * @first_chunks: the size of the first buddy in chunks, 0 if free * @last_chunks: the size of the last buddy in chunks, 0 if free */ struct zbud_header {struct list_head buddy; -struct list_head lru;unsigned int first_chunks;unsigned int last_chunks; -bool under_reclaim; +};mm/zbud.c struct zbud_pool { · line 127
Quoted 10 lines
mm/zbud.c +9
-0+ +/* +* struct zbud_page_priv - zbud flags to be stored in page->private +* @under_reclaim: if a zbud page is under reclaim +* @headless: indicates a page where zbud header didn't fit +*/ +struct zbud_page_priv { +bool under_reclaim:1 ; +bool headless:1 ; };Hmm, this is just my personal opinion, but I'm not a fan of casting ->private as a struct, if we're only using it as a bitmap. I'd suggest just defining bits as an enum (like page flags), e.g.
enum
zbud_flags{
ZBUD_UNDER_RECLAIM,
ZBUD_FULL_PAGE,
};or some names similar to that. then it can be checked with a simple
test_bit()call, and set_bit()/clear_bit().alternately, there are the already-existing
PG_privateandPG_private_2bits in the page flags...but unless we need ->private for something else, it probably makes more sense to just use it instead of thePG_privateflags.Quoted 11 lines
mm/zbud.c +1
-0/***************** @@ -221,6 +228,7 @@ MODULE_ALIAS("zpool-zbud");*****************/ /* Just to make the code easier to read */ enum buddy { +HEADLESS ,FIRST ,LAST };mm/zbud.c static int size_to_chunks(size_t size) · line 252
Quoted 8 lines
mm/zbud.c +4
-0@@ -237,12 +245,15 @@ static int size_to_chunks(size_t size)/* Initializes the zbud header of a newly allocated zbud page */ static struct zbud_header *init_zbud_page (struct page *page) { +struct zbud_page_priv *ppriv = (struct zbud_page_priv *)page->private;struct zbud_header *zhdr =page_address (page); + +INIT_LIST_HEAD (&page->lru); + ppriv->under_reclaim =0 ;don't forget to initialize the headless/full bit too. (I'll mention that the page allocation code does clear ->private before handing it to us, so it should already be 0. but let's not rely on that)
Quoted 102 lines
mm/zbud.c +40
-20+ zhdr->first_chunks =0 ; zhdr->last_chunks =0 ;INIT_LIST_HEAD (&zhdr->buddy); -INIT_LIST_HEAD (&zhdr->lru); - zhdr->under_reclaim =0 ;return zhdr; }@@ -267,11 +278,22 @@ static unsigned long encode_handle(struct zbud_header *zhdr, enum buddy bud)* over the zbud header in the first chunk. */ handle = (unsigned long )zhdr; -if (bud ==FIRST ) +switch (bud) { +case FIRST :/* skip over zbud header */ handle +=ZHDR_SIZE_ALIGNED ; -else /* bud == LAST */ +break ; +case LAST : handle +=PAGE_SIZE - (zhdr->last_chunks <<CHUNK_SHIFT ); +break ; +case HEADLESS : +break ; +default : +/* this should never happen */ +pr_err ("zbud: invalid buddy value %d\n" , bud); + handle =0 ; +break ; + }return handle; }@@ -287,6 +309,7 @@ static int num_free_chunks(struct zbud_header *zhdr)/* * Rather than branch for different situations, just use the fact that * free buddies have a length of zero to simplify everything. +* NB: can't be used with HEADLESS pages. */ return NCHUNKS - zhdr->first_chunks - zhdr->last_chunks; }@@ -353,31 +376,40 @@ void zbud_destroy_pool(struct zbud_pool *pool)int zbud_alloc (struct zbud_pool *pool,size_t size,gfp_t gfp,unsigned long *handle) { -int chunks, i, freechunks; +int chunks =0 , i, freechunks;struct zbud_header *zhdr =NULL ; +struct zbud_page_priv *ppriv;enum buddy bud;struct page *page;if (!size || (gfp & __GFP_HIGHMEM))return -EINVAL ; -if (size >PAGE_SIZE -ZHDR_SIZE_ALIGNED -CHUNK_SIZE ) + +if (size >PAGE_SIZE )return -ENOSPC ; - chunks =size_to_chunks (size); -spin_lock (&pool->lock); -/* First, try to find an unbuddied zbud page. */ - zhdr =NULL ; -for_each_unbuddied_list (i, chunks) { -if (!list_empty (&pool->unbuddied[i])) { - zhdr =list_first_entry (&pool->unbuddied[i], - struct zbud_header, buddy); -list_del (&zhdr->buddy); -if (zhdr->first_chunks ==0 ) - bud =FIRST ; -else - bud =LAST ; - goto found; +if (size >PAGE_SIZE -ZHDR_SIZE_ALIGNED -CHUNK_SIZE ) + bud =HEADLESS ; +else { + chunks =size_to_chunks (size); +spin_lock (&pool->lock); + +/* First, try to find an unbuddied zbud page. */ + zhdr =NULL ; +for_each_unbuddied_list (i, chunks) { +if (!list_empty (&pool->unbuddied[i])) { + zhdr =list_first_entry (&pool->unbuddied[i], + struct zbud_header, buddy); +list_del (&zhdr->buddy); + page =virt_to_page (zhdr); +if (zhdr->first_chunks ==0 ) + bud =FIRST ; +else + bud =LAST ; + goto found; + } } + bud =FIRST ; }/* Couldn't find unbuddied zbud page, create new one */ @@ -388,27 +420,31 @@ int zbud_alloc(struct zbud_pool *pool, size_t size, gfp_t gfp,spin_lock (&pool->lock); pool->pages_nr++;mm/zbud.c int zbud_alloc(struct zbud_pool *pool, size_t size, gfp_t gfp, · line 426
Quoted 10 lines
mm/zbud.c +2
-5zhdr =init_zbud_page (page); - bud =FIRST ; found: -if (bud ==FIRST ) - zhdr->first_chunks = chunks; -else - zhdr->last_chunks = chunks; + ppriv = (struct zbud_page_priv *)page->private; +if (bud !=HEADLESS ) {another personal opinion...it might look simpler if you just do:
if (bud == FULL) {
set_bit(ZBUD_FULL_PAGE, &page->private);
goto full;
}Quoted 18 lines
mm/zbud.c +13
-4+if (bud ==FIRST ) + zhdr->first_chunks = chunks; +else + zhdr->last_chunks = chunks; + +if (zhdr->first_chunks ==0 || zhdr->last_chunks ==0 ) { +/* Add to unbuddied list */ + freechunks =num_free_chunks (zhdr); +list_add (&zhdr->buddy, &pool->unbuddied[freechunks]); + }else { +/* Add to buddied list */ +list_add (&zhdr->buddy, &pool->buddied); + } -if (zhdr->first_chunks ==0 || zhdr->last_chunks ==0 ) { -/* Add to unbuddied list */ - freechunks =num_free_chunks (zhdr); -list_add (&zhdr->buddy, &pool->unbuddied[freechunks]);mm/zbud.c int zbud_alloc(struct zbud_pool *pool, size_t size, gfp_t gfp, · line 446
Quoted 10 lines
mm/zbud.c +5
-4- }else { -/* Add to buddied list */ -list_add (&zhdr->buddy, &pool->buddied); - } +/* Add/move zbud page to beginning of LRU */ +if (!list_empty (&page->lru)) +list_del (&page->lru); + }else + ppriv->headless = true;insert label here...
full:
mm/zbud.c int zbud_alloc(struct zbud_pool *pool, size_t size, gfp_t gfp, · line 410
Quoted 3 lines
mm/zbud.c +0
-3-/* Add/move zbud page to beginning of LRU */ -if (!list_empty (&zhdr->lru)) -list_del (&zhdr->lru);no matter the case, this is safe; leave it here for simplicity. a full page will have a empty list head.
Quoted 111 lines
mm/zbud.c +48
-23-list_add (&zhdr->lru, &pool->lru); +list_add (&page->lru, &pool->lru); *handle =encode_handle (zhdr, bud);spin_unlock (&pool->lock);@@ -430,28 +466,41 @@ void zbud_free(struct zbud_pool *pool, unsigned long handle) {struct zbud_header *zhdr;int freechunks; +struct page *page; +enum buddy bud; +struct zbud_page_priv *ppriv;spin_lock (&pool->lock); zhdr =handle_to_zbud_header (handle); + page =virt_to_page (zhdr); + ppriv = (struct zbud_page_priv *)page->private; -/* If first buddy, handle will be page aligned */ -if ((handle -ZHDR_SIZE_ALIGNED ) & ~PAGE_MASK ) +if (!(handle & ~PAGE_MASK ))/* HEADLESS page stored */ + bud =HEADLESS ; +else if ((handle -ZHDR_SIZE_ALIGNED ) & ~PAGE_MASK ) { + bud =LAST ; zhdr->last_chunks =0 ; -else + }else { +/* If first buddy, handle will be page aligned */ + bud =FIRST ; zhdr->first_chunks =0 ; + } -if (zhdr->under_reclaim) { +if (ppriv->under_reclaim) {/* zbud page is under reclaim, reclaim will free */ spin_unlock (&pool->lock);return ; } -/* Remove from existing buddy list */ -list_del (&zhdr->buddy); +if (bud !=HEADLESS ) { +/* Remove from existing buddy list */ +list_del (&zhdr->buddy); + } -if (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 ) { +if (bud ==HEADLESS || + (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 )) {/* zbud page is empty, free */ -list_del (&zhdr->lru); +list_del (&page->lru);free_zbud_page (zhdr); pool->pages_nr--; }else {@@ -505,6 +554,8 @@ int zbud_reclaim_page(struct zbud_pool *pool, unsigned int retries) {int i, ret, freechunks;struct zbud_header *zhdr; +struct page *page; +struct zbud_page_priv *ppriv;unsigned long first_handle =0 , last_handle =0 ;spin_lock (&pool->lock);@@ -514,21 +565,31 @@ int zbud_reclaim_page(struct zbud_pool *pool, unsigned int retries)return -EINVAL ; }for (i =0 ; i < retries; i++) { - zhdr = list_tail_entry(&pool->lru, struct zbud_header, lru); -list_del (&zhdr->lru); -list_del (&zhdr->buddy); + page = list_tail_entry(&pool->lru, struct page, lru); + ppriv = (struct zbud_page_priv *)page->private; +list_del (&page->lru); +/* Protect zbud page against free */ - zhdr->under_reclaim = true; -/* -* We need encode the handles before unlocking, since we can -* race with free that will set (first|last)_chunks to 0 -*/ - first_handle =0 ; - last_handle =0 ; -if (zhdr->first_chunks) - first_handle =encode_handle (zhdr,FIRST ); -if (zhdr->last_chunks) - last_handle =encode_handle (zhdr,LAST ); + ppriv->under_reclaim = true; + zhdr =page_address (page); +if (!ppriv->headless) { +list_del (&zhdr->buddy); +/* +* We need encode the handles before unlocking, since +* we can race with free that will set +* (first|last)_chunks to 0 +*/ + first_handle =0 ; + last_handle =0 ; +if (zhdr->first_chunks) + first_handle =encode_handle (zhdr,FIRST ); +if (zhdr->last_chunks) + last_handle =encode_handle (zhdr,LAST ); + }else { + first_handle =encode_handle (zhdr,HEADLESS ); + last_handle =0 ; + } +spin_unlock (&pool->lock);/* Issue the eviction callback(s) */ mm/zbud.c int zbud_reclaim_page(struct zbud_pool *pool, unsigned int retries) · line 610
Quoted 8 lines
mm/zbud.c +3
-2@@ -544,8 +605,9 @@ int zbud_reclaim_page(struct zbud_pool *pool, unsigned int retries) } next:spin_lock (&pool->lock); - zhdr->under_reclaim = false; -if (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 ) { + ppriv->under_reclaim = false; +if (ppriv->headless || + (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 )) {oops, this isn't right. you're assuming ->headless means the eviction succeeded. However if
pool->ops->evict()failed, that isn't true.Quoted 6 lines
mm/zbud.c +0
-0/* * Both buddies are now free, free the zbud page and * return success. @@ -565,7 +627,7 @@ next: }
Quoted 9 lines
Quoted 9 lines
mm/zbud.c +1
-2/* add to beginning of LRU */ -list_add (&zhdr->lru, &pool->lru); +list_add (&page->lru, &pool->lru); }spin_unlock (&pool->lock);return -EAGAIN ; --2.4.2
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Okay, how about this? It's gotten smaller BTW :)
zbud: allow up to PAGE_SIZE allocations
Currently zbud is only capable of allocating not more than PAGE_SIZE - ZHDR_SIZE_ALIGNED - CHUNK_SIZE. This is okay as long as only zswap is using it, but other users of zbud may (and likely will) want to allocate up to PAGE_SIZE. This patch addresses that by skipping the creation of zbud internal structure in the beginning of an allocated page. As a zbud page is no longer guaranteed to contain zbud header, the following changes have to be applied throughout the code:
* page->lru to be used for zbud page lists
* page->private to hold 'under_reclaim' flag
page->private will also be used to indicate if this page contains a zbud header in the beginning or not ('headless' flag).
--- mm/zbud.c | 167 ++++++++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 113 insertions(+), 54 deletions(-)
mm/zbud.c +113 -54
@@ -105,18 +105,20 @@ struct zbud_pool {/* * struct zbud_header - zbud page metadata occupying the first chunk of each -* zbud page. +* zbud page, except for HEADLESS pages * @buddy: links the zbud page into the unbuddied/buddied lists in the pool -* @lru: links the zbud page into the lru list in the pool * @first_chunks: the size of the first buddy in chunks, 0 if free * @last_chunks: the size of the last buddy in chunks, 0 if free */ struct zbud_header {struct list_head buddy; -struct list_head lru;unsigned int first_chunks;unsigned int last_chunks; -bool under_reclaim; +}; + +enum zbud_page_flags { +UNDER_RECLAIM =0 , +PAGE_HEADLESS , };/*****************
@@ -221,6 +223,7 @@ MODULE_ALIAS("zpool-zbud");*****************/ /* Just to make the code easier to read */ enum buddy { +HEADLESS ,FIRST ,LAST };
@@ -238,11 +241,14 @@ static int size_to_chunks(size_t size)static struct zbud_header *init_zbud_page (struct page *page) {struct zbud_header *zhdr =page_address (page); + +INIT_LIST_HEAD (&page->lru); +clear_bit (UNDER_RECLAIM , &page->private); +clear_bit (HEADLESS , &page->private); + zhdr->first_chunks =0 ; zhdr->last_chunks =0 ;INIT_LIST_HEAD (&zhdr->buddy); -INIT_LIST_HEAD (&zhdr->lru); - zhdr->under_reclaim =0 ;return zhdr; }
@@ -267,11 +273,22 @@ static unsigned long encode_handle(struct zbud_header *zhdr, enum buddy bud)* over the zbud header in the first chunk. */ handle = (unsigned long )zhdr; -if (bud ==FIRST ) +switch (bud) { +case FIRST :/* skip over zbud header */ handle +=ZHDR_SIZE_ALIGNED ; -else /* bud == LAST */ +break ; +case LAST : handle +=PAGE_SIZE - (zhdr->last_chunks <<CHUNK_SHIFT ); +break ; +case HEADLESS : +break ; +default : +/* this should never happen */ +pr_err ("zbud: invalid buddy value %d\n" , bud); + handle =0 ; +break ; + }return handle; }
@@ -287,6 +304,7 @@ static int num_free_chunks(struct zbud_header *zhdr)/* * Rather than branch for different situations, just use the fact that * free buddies have a length of zero to simplify everything. +* NB: can't be used with HEADLESS pages. */ return NCHUNKS - zhdr->first_chunks - zhdr->last_chunks; }
@@ -353,31 +371,39 @@ void zbud_destroy_pool(struct zbud_pool *pool)int zbud_alloc (struct zbud_pool *pool,size_t size,gfp_t gfp,unsigned long *handle) { -int chunks, i, freechunks; +int chunks =0 , i, freechunks;struct zbud_header *zhdr =NULL ;enum buddy bud;struct page *page;if (!size || (gfp & __GFP_HIGHMEM))return -EINVAL ; -if (size >PAGE_SIZE -ZHDR_SIZE_ALIGNED -CHUNK_SIZE ) + +if (size >PAGE_SIZE )return -ENOSPC ; - chunks =size_to_chunks (size); -spin_lock (&pool->lock); -/* First, try to find an unbuddied zbud page. */ - zhdr =NULL ; -for_each_unbuddied_list (i, chunks) { -if (!list_empty (&pool->unbuddied[i])) { - zhdr =list_first_entry (&pool->unbuddied[i], - struct zbud_header, buddy); -list_del (&zhdr->buddy); -if (zhdr->first_chunks ==0 ) - bud =FIRST ; -else - bud =LAST ; - goto found; +if (size >PAGE_SIZE -ZHDR_SIZE_ALIGNED -CHUNK_SIZE ) + bud =HEADLESS ; +else { + chunks =size_to_chunks (size); +spin_lock (&pool->lock); + +/* First, try to find an unbuddied zbud page. */ + zhdr =NULL ; +for_each_unbuddied_list (i, chunks) { +if (!list_empty (&pool->unbuddied[i])) { + zhdr =list_first_entry (&pool->unbuddied[i], + struct zbud_header, buddy); +list_del (&zhdr->buddy); + page =virt_to_page (zhdr); +if (zhdr->first_chunks ==0 ) + bud =FIRST ; +else + bud =LAST ; + goto found; + } } + bud =FIRST ; }/* Couldn't find unbuddied zbud page, create new one */
@@ -388,7 +414,11 @@ int zbud_alloc(struct zbud_pool *pool, size_t size, gfp_t gfp,spin_lock (&pool->lock); pool->pages_nr++; zhdr =init_zbud_page (page); - bud =FIRST ; + +if (bud ==HEADLESS ) { +set_bit (PAGE_HEADLESS , &page->private); + goto headless; + } found:if (bud ==FIRST )
@@ -405,10 +435,12 @@ found:list_add (&zhdr->buddy, &pool->buddied); } +headless:/* Add/move zbud page to beginning of LRU */ -if (!list_empty (&zhdr->lru)) -list_del (&zhdr->lru); -list_add (&zhdr->lru, &pool->lru); +if (!list_empty (&page->lru)) +list_del (&page->lru); + +list_add (&page->lru, &pool->lru); *handle =encode_handle (zhdr, bud);spin_unlock (&pool->lock);
@@ -430,28 +462,39 @@ void zbud_free(struct zbud_pool *pool, unsigned long handle) {struct zbud_header *zhdr;int freechunks; +struct page *page; +enum buddy bud;spin_lock (&pool->lock); zhdr =handle_to_zbud_header (handle); + page =virt_to_page (zhdr); -/* If first buddy, handle will be page aligned */ -if ((handle -ZHDR_SIZE_ALIGNED ) & ~PAGE_MASK ) +if (!(handle & ~PAGE_MASK ))/* HEADLESS page stored */ + bud =HEADLESS ; +else if ((handle -ZHDR_SIZE_ALIGNED ) & ~PAGE_MASK ) { + bud =LAST ; zhdr->last_chunks =0 ; -else + }else { +/* If first buddy, handle will be page aligned */ + bud =FIRST ; zhdr->first_chunks =0 ; + } -if (zhdr->under_reclaim) { +if (test_bit (UNDER_RECLAIM , &page->private)) {/* zbud page is under reclaim, reclaim will free */ spin_unlock (&pool->lock);return ; } -/* Remove from existing buddy list */ -list_del (&zhdr->buddy); +if (bud !=HEADLESS ) { +/* Remove from existing buddy list */ +list_del (&zhdr->buddy); + } -if (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 ) { +if (bud ==HEADLESS || + (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 )) {/* zbud page is empty, free */ -list_del (&zhdr->lru); +list_del (&page->lru);free_zbud_page (zhdr); pool->pages_nr--; }else {
@@ -503,8 +546,9 @@ void zbud_free(struct zbud_pool *pool, unsigned long handle)*/ int zbud_reclaim_page (struct zbud_pool *pool,unsigned int retries) { -int i, ret, freechunks; +int i, ret =0 , freechunks;struct zbud_header *zhdr; +struct page *page;unsigned long first_handle =0 , last_handle =0 ;spin_lock (&pool->lock);
@@ -514,21 +558,30 @@ int zbud_reclaim_page(struct zbud_pool *pool, unsigned int retries)return -EINVAL ; }for (i =0 ; i < retries; i++) { - zhdr = list_tail_entry(&pool->lru, struct zbud_header, lru); -list_del (&zhdr->lru); -list_del (&zhdr->buddy); + page = list_tail_entry(&pool->lru, struct page, lru); +list_del (&page->lru); +/* Protect zbud page against free */ - zhdr->under_reclaim = true; -/* -* We need encode the handles before unlocking, since we can -* race with free that will set (first|last)_chunks to 0 -*/ - first_handle =0 ; - last_handle =0 ; -if (zhdr->first_chunks) - first_handle =encode_handle (zhdr,FIRST ); -if (zhdr->last_chunks) - last_handle =encode_handle (zhdr,LAST ); +set_bit (UNDER_RECLAIM , &page->private); + zhdr =page_address (page); +if (!test_bit (PAGE_HEADLESS , &page->private)) { +list_del (&zhdr->buddy); +/* +* We need encode the handles before unlocking, since +* we can race with free that will set +* (first|last)_chunks to 0 +*/ + first_handle =0 ; + last_handle =0 ; +if (zhdr->first_chunks) + first_handle =encode_handle (zhdr,FIRST ); +if (zhdr->last_chunks) + last_handle =encode_handle (zhdr,LAST ); + }else { + first_handle =encode_handle (zhdr,HEADLESS ); + last_handle =0 ; + } +spin_unlock (&pool->lock);/* Issue the eviction callback(s) */
@@ -544,8 +597,14 @@ int zbud_reclaim_page(struct zbud_pool *pool, unsigned int retries) } next:spin_lock (&pool->lock); - zhdr->under_reclaim = false; -if (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 ) { +clear_bit (UNDER_RECLAIM , &page->private); +if (test_bit (PAGE_HEADLESS , &page->private)) { +if (ret ==0 ) { +free_zbud_page (zhdr); + pool->pages_nr--; +spin_unlock (&pool->lock); + } + }else if (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 ) {/* * Both buddies are now free, free the zbud page and * return success.
@@ -565,7 +624,7 @@ next: }/* add to beginning of LRU */ -list_add (&zhdr->lru, &pool->lru); +list_add (&page->lru, &pool->lru); }spin_unlock (&pool->lock);return -EAGAIN ;
Signature · 7 lines
2.4.2 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 38 lines · Vitaly Wool
Okay, how about this? It's gotten smaller BTW :)
zbud: allow up to
PAGE_SIZEallocationsCurrently zbud is only capable of allocating not more than
PAGE_SIZE-ZHDR_SIZE_ALIGNED-CHUNK_SIZE. This is okay as long as only zswap is using it, but other users of zbud may (and likely will) want to allocate up toPAGE_SIZE. This patch addresses that by skipping the creation of zbud internal structure in the beginning of an allocated page. As a zbud page is no longer guaranteed to contain zbud header, the following changes have to be applied throughout the code:
*page->lruto be used for zbud page lists
*page->privateto hold 'under_reclaim' flag
page->privatewill also be used to indicate if this page contains a zbud header in the beginning or not ('headless' flag).Signed-off-by Vitaly Wool <[email protected]>--- mm/zbud.c | 167 ++++++++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 113 insertions(+), 54 deletions(-)mm/zbud.c +1
-2@@ -105,18 +105,20 @@ struct zbud_pool {/* * struct zbud_header - zbud page metadata occupying the first chunk of each -* zbud page. +* zbud page, except for HEADLESS pages * @buddy: links the zbud page into the unbuddied/buddied lists in the pool -* @lru: links the zbud page into the lru list in the pool * @first_chunks: the size of the first buddy in chunks, 0 if free * @last_chunks: the size of the last buddy in chunks, 0 if free */
mm/zbud.c struct zbud_pool { · line 120
Quoted 10 lines
mm/zbud.c +4
-2struct zbud_header {struct list_head buddy; -struct list_head lru;unsigned int first_chunks;unsigned int last_chunks; -bool under_reclaim; +}; + +enum zbud_page_flags { +UNDER_RECLAIM =0 ,
Don't need the "= 0"
mm/zbud.c struct zbud_pool { · line 121
Quoted 1 line
mm/zbud.c +1
-0+ PAGE_HEADLESS,
Also I think we should prefix the enum values here. With ZPF_ ?
Quoted 12 lines
mm/zbud.c +1
-0};/***************** @@ -221,6 +223,7 @@ MODULE_ALIAS("zpool-zbud");*****************/ /* Just to make the code easier to read */ enum buddy { +HEADLESS ,FIRST ,LAST };
mm/zbud.c static int size_to_chunks(size_t size) · line 247
Quoted 7 lines
mm/zbud.c +4
-0@@ -238,11 +241,14 @@ static int size_to_chunks(size_t size)static struct zbud_header *init_zbud_page (struct page *page) {struct zbud_header *zhdr =page_address (page); + +INIT_LIST_HEAD (&page->lru); +clear_bit (UNDER_RECLAIM , &page->private); +clear_bit (HEADLESS , &page->private);
I know we are using private in a bitwise flags mode, but maybe we should just init with page->private = 0
Quoted 21 lines
mm/zbud.c +5
-4+ zhdr->first_chunks =0 ; zhdr->last_chunks =0 ;INIT_LIST_HEAD (&zhdr->buddy); -INIT_LIST_HEAD (&zhdr->lru); - zhdr->under_reclaim =0 ;return zhdr; }@@ -267,11 +273,22 @@ static unsigned long encode_handle(struct zbud_header *zhdr, enum buddy bud)* over the zbud header in the first chunk. */ handle = (unsigned long )zhdr; -if (bud ==FIRST ) +switch (bud) { +case FIRST :/* skip over zbud header */ handle +=ZHDR_SIZE_ALIGNED ; -else /* bud == LAST */ +break ; +case LAST :
mm/zbud.c static unsigned long encode_handle(struct zbud_header *zhdr, enum buddy bud) · line 291
Quoted 10 lines
mm/zbud.c +9
-0handle +=PAGE_SIZE - (zhdr->last_chunks <<CHUNK_SHIFT ); +break ; +case HEADLESS : +break ; +default : +/* this should never happen */ +pr_err ("zbud: invalid buddy value %d\n" , bud); + handle =0 ; +break ; + }
Don't need this default case since we have a case for each valid value of the enum.
Also, I think we want to add some code to free_zbud_page() to clear page->private and init page->lru so we don't leave dangling pointers.
Looks good though :)
Thanks,
Seth
Quoted 33 lines
mm/zbud.c +4
-7return handle; }@@ -287,6 +304,7 @@ static int num_free_chunks(struct zbud_header *zhdr)/* * Rather than branch for different situations, just use the fact that * free buddies have a length of zero to simplify everything. +* NB: can't be used with HEADLESS pages. */ return NCHUNKS - zhdr->first_chunks - zhdr->last_chunks; }@@ -353,31 +371,39 @@ void zbud_destroy_pool(struct zbud_pool *pool)int zbud_alloc (struct zbud_pool *pool,size_t size,gfp_t gfp,unsigned long *handle) { -int chunks, i, freechunks; +int chunks =0 , i, freechunks;struct zbud_header *zhdr =NULL ;enum buddy bud;struct page *page;if (!size || (gfp & __GFP_HIGHMEM))return -EINVAL ; -if (size >PAGE_SIZE -ZHDR_SIZE_ALIGNED -CHUNK_SIZE ) + +if (size >PAGE_SIZE )return -ENOSPC ; - chunks =size_to_chunks (size); -spin_lock (&pool->lock); -/* First, try to find an unbuddied zbud page. */ - zhdr =NULL ; -for_each_unbuddied_list (i, chunks) {
mm/zbud.c void zbud_destroy_pool(struct zbud_pool *pool) · line 385
Quoted 10 lines
mm/zbud.c +1
-9-if (!list_empty (&pool->unbuddied[i])) { - zhdr =list_first_entry (&pool->unbuddied[i], - struct zbud_header, buddy); -list_del (&zhdr->buddy); -if (zhdr->first_chunks ==0 ) - bud =FIRST ; -else - bud =LAST ; - goto found; +if (size >PAGE_SIZE -ZHDR_SIZE_ALIGNED -CHUNK_SIZE )
Nit, maybe we should set chunks = 0 here so that in both branches, chunks is obviously set.
Quoted 185 lines
mm/zbud.c +83
-30+ bud =HEADLESS ; + else { + chunks =size_to_chunks (size); +spin_lock (&pool->lock); + +/* First, try to find an unbuddied zbud page. */ + zhdr =NULL ; +for_each_unbuddied_list (i, chunks) { +if (!list_empty (&pool->unbuddied[i])) { + zhdr =list_first_entry (&pool->unbuddied[i], + struct zbud_header, buddy); +list_del (&zhdr->buddy); + page =virt_to_page (zhdr); +if (zhdr->first_chunks ==0 ) + bud =FIRST ; +else + bud =LAST ; + goto found; + } } + bud =FIRST ; }/* Couldn't find unbuddied zbud page, create new one */ @@ -388,7 +414,11 @@ int zbud_alloc(struct zbud_pool *pool, size_t size, gfp_t gfp,spin_lock (&pool->lock); pool->pages_nr++; zhdr =init_zbud_page (page); - bud =FIRST ; + +if (bud ==HEADLESS ) { +set_bit (PAGE_HEADLESS , &page->private); + goto headless; + } found:if (bud ==FIRST )@@ -405,10 +435,12 @@ found:list_add (&zhdr->buddy, &pool->buddied); } +headless:/* Add/move zbud page to beginning of LRU */ -if (!list_empty (&zhdr->lru)) -list_del (&zhdr->lru); -list_add (&zhdr->lru, &pool->lru); +if (!list_empty (&page->lru)) +list_del (&page->lru); + +list_add (&page->lru, &pool->lru); *handle =encode_handle (zhdr, bud);spin_unlock (&pool->lock);@@ -430,28 +462,39 @@ void zbud_free(struct zbud_pool *pool, unsigned long handle) {struct zbud_header *zhdr;int freechunks; +struct page *page; +enum buddy bud;spin_lock (&pool->lock); zhdr =handle_to_zbud_header (handle); + page =virt_to_page (zhdr); -/* If first buddy, handle will be page aligned */ -if ((handle -ZHDR_SIZE_ALIGNED ) & ~PAGE_MASK ) +if (!(handle & ~PAGE_MASK ))/* HEADLESS page stored */ + bud =HEADLESS ; +else if ((handle -ZHDR_SIZE_ALIGNED ) & ~PAGE_MASK ) { + bud =LAST ; zhdr->last_chunks =0 ; -else + }else { +/* If first buddy, handle will be page aligned */ + bud =FIRST ; zhdr->first_chunks =0 ; + } -if (zhdr->under_reclaim) { +if (test_bit (UNDER_RECLAIM , &page->private)) {/* zbud page is under reclaim, reclaim will free */ spin_unlock (&pool->lock);return ; } -/* Remove from existing buddy list */ -list_del (&zhdr->buddy); +if (bud !=HEADLESS ) { +/* Remove from existing buddy list */ +list_del (&zhdr->buddy); + } -if (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 ) { +if (bud ==HEADLESS || + (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 )) {/* zbud page is empty, free */ -list_del (&zhdr->lru); +list_del (&page->lru);free_zbud_page (zhdr); pool->pages_nr--; }else {@@ -503,8 +546,9 @@ void zbud_free(struct zbud_pool *pool, unsigned long handle)*/ int zbud_reclaim_page (struct zbud_pool *pool,unsigned int retries) { -int i, ret, freechunks; +int i, ret =0 , freechunks;struct zbud_header *zhdr; +struct page *page;unsigned long first_handle =0 , last_handle =0 ;spin_lock (&pool->lock);@@ -514,21 +558,30 @@ int zbud_reclaim_page(struct zbud_pool *pool, unsigned int retries)return -EINVAL ; }for (i =0 ; i < retries; i++) { - zhdr = list_tail_entry(&pool->lru, struct zbud_header, lru); -list_del (&zhdr->lru); -list_del (&zhdr->buddy); + page = list_tail_entry(&pool->lru, struct page, lru); +list_del (&page->lru); +/* Protect zbud page against free */ - zhdr->under_reclaim = true; -/* -* We need encode the handles before unlocking, since we can -* race with free that will set (first|last)_chunks to 0 -*/ - first_handle =0 ; - last_handle =0 ; -if (zhdr->first_chunks) - first_handle =encode_handle (zhdr,FIRST ); -if (zhdr->last_chunks) - last_handle =encode_handle (zhdr,LAST ); +set_bit (UNDER_RECLAIM , &page->private); + zhdr =page_address (page); +if (!test_bit (PAGE_HEADLESS , &page->private)) { +list_del (&zhdr->buddy); +/* +* We need encode the handles before unlocking, since +* we can race with free that will set +* (first|last)_chunks to 0 +*/ + first_handle =0 ; + last_handle =0 ; +if (zhdr->first_chunks) + first_handle =encode_handle (zhdr,FIRST ); +if (zhdr->last_chunks) + last_handle =encode_handle (zhdr,LAST ); + }else { + first_handle =encode_handle (zhdr,HEADLESS ); + last_handle =0 ; + } +spin_unlock (&pool->lock);/* Issue the eviction callback(s) */ @@ -544,8 +597,14 @@ int zbud_reclaim_page(struct zbud_pool *pool, unsigned int retries) } next:spin_lock (&pool->lock); - zhdr->under_reclaim = false; -if (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 ) { +clear_bit (UNDER_RECLAIM , &page->private); +if (test_bit (PAGE_HEADLESS , &page->private)) { +if (ret ==0 ) { +free_zbud_page (zhdr); + pool->pages_nr--; +spin_unlock (&pool->lock); + } + }else if (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 ) {/* * Both buddies are now free, free the zbud page and * return success. @@ -565,7 +624,7 @@ next: }/* add to beginning of LRU */ -list_add (&zhdr->lru, &pool->lru); +list_add (&page->lru, &pool->lru); }spin_unlock (&pool->lock);return -EAGAIN ;Signature · 1 line
2.4.2
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Hello Seth,
Quoted 114 lines · Seth Jennings
Quoted 38 lines · Vitaly Wool
Okay, how about this? It's gotten smaller BTW :)
zbud: allow up to
PAGE_SIZEallocationsCurrently zbud is only capable of allocating not more than
PAGE_SIZE-ZHDR_SIZE_ALIGNED-CHUNK_SIZE. This is okay as long as only zswap is using it, but other users of zbud may (and likely will) want to allocate up toPAGE_SIZE. This patch addresses that by skipping the creation of zbud internal structure in the beginning of an allocated page. As a zbud page is no longer guaranteed to contain zbud header, the following changes have to be applied throughout the code:
*page->lruto be used for zbud page lists
*page->privateto hold 'under_reclaim' flag
page->privatewill also be used to indicate if this page contains a zbud header in the beginning or not ('headless' flag).Signed-off-by Vitaly Wool <[email protected]>--- mm/zbud.c | 167 ++++++++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 113 insertions(+), 54 deletions(-)mm/zbud.c +1
-2@@ -105,18 +105,20 @@ struct zbud_pool {/* * struct zbud_header - zbud page metadata occupying the first chunk of each -* zbud page. +* zbud page, except for HEADLESS pages * @buddy: links the zbud page into the unbuddied/buddied lists in the pool -* @lru: links the zbud page into the lru list in the pool * @first_chunks: the size of the first buddy in chunks, 0 if free * @last_chunks: the size of the last buddy in chunks, 0 if free */ mm/zbud.c struct zbud_pool { · line 120
Quoted 10 lines
mm/zbud.c +4
-2struct zbud_header {struct list_head buddy; -struct list_head lru;unsigned int first_chunks;unsigned int last_chunks; -bool under_reclaim; +}; + +enum zbud_page_flags { +UNDER_RECLAIM =0 ,Don't need the "= 0"
mm/zbud.c struct zbud_pool { · line 121
Quoted 1 line
mm/zbud.c +1
-0+ PAGE_HEADLESS,Also I think we should prefix the enum values here. With
ZPF_?Quoted 12 lines
mm/zbud.c +1
-0};/***************** @@ -221,6 +223,7 @@ MODULE_ALIAS("zpool-zbud");*****************/ /* Just to make the code easier to read */ enum buddy { +HEADLESS ,FIRST ,LAST };mm/zbud.c static int size_to_chunks(size_t size) · line 247
Quoted 7 lines
mm/zbud.c +4
-0@@ -238,11 +241,14 @@ static int size_to_chunks(size_t size)static struct zbud_header *init_zbud_page (struct page *page) {struct zbud_header *zhdr =page_address (page); + +INIT_LIST_HEAD (&page->lru); +clear_bit (UNDER_RECLAIM , &page->private); +clear_bit (HEADLESS , &page->private);I know we are using private in a bitwise flags mode, but maybe we should just init with
page->private= 0Quoted 21 lines
mm/zbud.c +5
-4+ zhdr->first_chunks =0 ; zhdr->last_chunks =0 ;INIT_LIST_HEAD (&zhdr->buddy); -INIT_LIST_HEAD (&zhdr->lru); - zhdr->under_reclaim =0 ;return zhdr; }@@ -267,11 +273,22 @@ static unsigned long encode_handle(struct zbud_header *zhdr, enum buddy bud)* over the zbud header in the first chunk. */ handle = (unsigned long )zhdr; -if (bud ==FIRST ) +switch (bud) { +case FIRST :/* skip over zbud header */ handle +=ZHDR_SIZE_ALIGNED ; -else /* bud == LAST */ +break ; +case LAST :mm/zbud.c static unsigned long encode_handle(struct zbud_header *zhdr, enum buddy bud) · line 291
Quoted 10 lines
mm/zbud.c +9
-0handle +=PAGE_SIZE - (zhdr->last_chunks <<CHUNK_SHIFT ); +break ; +case HEADLESS : +break ; +default : +/* this should never happen */ +pr_err ("zbud: invalid buddy value %d\n" , bud); + handle =0 ; +break ; + }Don't need this default case since we have a case for each valid value of the enum.
Quoted 2 lines
Also, I think we want to add some code to
free_zbud_page()to clearpage->privateand initpage->lruso we don't leave dangling pointers.
Right, maybe it makes sense for free_zbud_page() to take struct page pointer as an argument, too, to minimize back-and-forth conversions?
Quoted 1 line
Looks good though :)
Thanks, I'll come up with the new version shortly. :)
~vitaly
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 7 lines · Vitaly Wool
Currently zbud is only capable of allocating not more than
PAGE_SIZE-ZHDR_SIZE_ALIGNED-CHUNK_SIZE. This is okay as long as only zswap is using it, but other users of zbud may (and likely will) want to allocate up toPAGE_SIZE. This patch addresses that by skipping the creation of zbud internal structure in the beginning of an allocated page (such pages are then called 'headless').
I guess I'm having trouble with this. If you store a PAGE_SIZE allocation in zbud, then the zpage can only have one allocation as there is no room for a buddy. Sooooo... we have an allocator for that: the page allocator.
zbud doesn't support this by design because, if you are only storing one allocation per page, you don't gain anything.
This functionality creates many new edge cases for the code.
What is this use case you envision? I think we need to discuss whether the use case exists and if it justifies the added complexity.
We are crossing a boundary into zsmalloc style complexity with storing stuff in the struct page, something I really didn't want to do in zbud.
zbud is the simple one, zsmalloc is the complex one. I'd hate to have two complex ones :-/
Seth
Quoted 342 lines
As a zbud page is no longer guaranteed to contain zbud header, the following changes had to be applied throughout the code:
*page->lruto be used for zbud page lists
*page->privateto hold 'under_reclaim' flag
page->privatewill also be used to indicate if this page contains a zbud header in the beginning or not ('headless' flag).Signed-off-by Vitaly Wool <[email protected]>--- mm/zbud.c | 194 +++++++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 128 insertions(+), 66 deletions(-)mm/zbud.c +128
-66@@ -105,18 +105,25 @@ struct zbud_pool {/* * struct zbud_header - zbud page metadata occupying the first chunk of each -* zbud page. +* zbud page, except for HEADLESS pages * @buddy: links the zbud page into the unbuddied/buddied lists in the pool -* @lru: links the zbud page into the lru list in the pool * @first_chunks: the size of the first buddy in chunks, 0 if free * @last_chunks: the size of the last buddy in chunks, 0 if free */ struct zbud_header {struct list_head buddy; -struct list_head lru;unsigned int first_chunks;unsigned int last_chunks; -bool under_reclaim; +}; + +/* +* struct zbud_page_priv - zbud flags to be stored in page->private +* @under_reclaim: if a zbud page is under reclaim +* @headless: indicates a page where zbud header didn't fit +*/ +struct zbud_page_priv { +bool under_reclaim:1 ; +bool headless:1 ; };/***************** @@ -221,6 +228,7 @@ MODULE_ALIAS("zpool-zbud");*****************/ /* Just to make the code easier to read */ enum buddy { +HEADLESS ,FIRST ,LAST };@@ -237,12 +245,15 @@ static int size_to_chunks(size_t size)/* Initializes the zbud header of a newly allocated zbud page */ static struct zbud_header *init_zbud_page (struct page *page) { +struct zbud_page_priv *ppriv = (struct zbud_page_priv *)page->private;struct zbud_header *zhdr =page_address (page); + +INIT_LIST_HEAD (&page->lru); + ppriv->under_reclaim =0 ; + zhdr->first_chunks =0 ; zhdr->last_chunks =0 ;INIT_LIST_HEAD (&zhdr->buddy); -INIT_LIST_HEAD (&zhdr->lru); - zhdr->under_reclaim =0 ;return zhdr; }@@ -267,11 +278,22 @@ static unsigned long encode_handle(struct zbud_header *zhdr, enum buddy bud)* over the zbud header in the first chunk. */ handle = (unsigned long )zhdr; -if (bud ==FIRST ) +switch (bud) { +case FIRST :/* skip over zbud header */ handle +=ZHDR_SIZE_ALIGNED ; -else /* bud == LAST */ +break ; +case LAST : handle +=PAGE_SIZE - (zhdr->last_chunks <<CHUNK_SHIFT ); +break ; +case HEADLESS : +break ; +default : +/* this should never happen */ +pr_err ("zbud: invalid buddy value %d\n" , bud); + handle =0 ; +break ; + }return handle; }@@ -287,6 +309,7 @@ static int num_free_chunks(struct zbud_header *zhdr)/* * Rather than branch for different situations, just use the fact that * free buddies have a length of zero to simplify everything. +* NB: can't be used with HEADLESS pages. */ return NCHUNKS - zhdr->first_chunks - zhdr->last_chunks; }@@ -353,31 +376,40 @@ void zbud_destroy_pool(struct zbud_pool *pool)int zbud_alloc (struct zbud_pool *pool,size_t size,gfp_t gfp,unsigned long *handle) { -int chunks, i, freechunks; +int chunks =0 , i, freechunks;struct zbud_header *zhdr =NULL ; +struct zbud_page_priv *ppriv;enum buddy bud;struct page *page;if (!size || (gfp & __GFP_HIGHMEM))return -EINVAL ; -if (size >PAGE_SIZE -ZHDR_SIZE_ALIGNED -CHUNK_SIZE ) + +if (size >PAGE_SIZE )return -ENOSPC ; - chunks =size_to_chunks (size); -spin_lock (&pool->lock); -/* First, try to find an unbuddied zbud page. */ - zhdr =NULL ; -for_each_unbuddied_list (i, chunks) { -if (!list_empty (&pool->unbuddied[i])) { - zhdr =list_first_entry (&pool->unbuddied[i], - struct zbud_header, buddy); -list_del (&zhdr->buddy); -if (zhdr->first_chunks ==0 ) - bud =FIRST ; -else - bud =LAST ; - goto found; +if (size >PAGE_SIZE -ZHDR_SIZE_ALIGNED -CHUNK_SIZE ) + bud =HEADLESS ; +else { + chunks =size_to_chunks (size); +spin_lock (&pool->lock); + +/* First, try to find an unbuddied zbud page. */ + zhdr =NULL ; +for_each_unbuddied_list (i, chunks) { +if (!list_empty (&pool->unbuddied[i])) { + zhdr =list_first_entry (&pool->unbuddied[i], + struct zbud_header, buddy); +list_del (&zhdr->buddy); + page =virt_to_page (zhdr); +if (zhdr->first_chunks ==0 ) + bud =FIRST ; +else + bud =LAST ; + goto found; + } } + bud =FIRST ; }/* Couldn't find unbuddied zbud page, create new one */ @@ -388,27 +420,31 @@ int zbud_alloc(struct zbud_pool *pool, size_t size, gfp_t gfp,spin_lock (&pool->lock); pool->pages_nr++; zhdr =init_zbud_page (page); - bud =FIRST ; found: -if (bud ==FIRST ) - zhdr->first_chunks = chunks; -else - zhdr->last_chunks = chunks; + ppriv = (struct zbud_page_priv *)page->private; +if (bud !=HEADLESS ) { +if (bud ==FIRST ) + zhdr->first_chunks = chunks; +else + zhdr->last_chunks = chunks; + +if (zhdr->first_chunks ==0 || zhdr->last_chunks ==0 ) { +/* Add to unbuddied list */ + freechunks =num_free_chunks (zhdr); +list_add (&zhdr->buddy, &pool->unbuddied[freechunks]); + }else { +/* Add to buddied list */ +list_add (&zhdr->buddy, &pool->buddied); + } -if (zhdr->first_chunks ==0 || zhdr->last_chunks ==0 ) { -/* Add to unbuddied list */ - freechunks =num_free_chunks (zhdr); -list_add (&zhdr->buddy, &pool->unbuddied[freechunks]); - }else { -/* Add to buddied list */ -list_add (&zhdr->buddy, &pool->buddied); - } +/* Add/move zbud page to beginning of LRU */ +if (!list_empty (&page->lru)) +list_del (&page->lru); + }else + ppriv->headless = true; -/* Add/move zbud page to beginning of LRU */ -if (!list_empty (&zhdr->lru)) -list_del (&zhdr->lru); -list_add (&zhdr->lru, &pool->lru); +list_add (&page->lru, &pool->lru); *handle =encode_handle (zhdr, bud);spin_unlock (&pool->lock);@@ -430,28 +466,41 @@ void zbud_free(struct zbud_pool *pool, unsigned long handle) {struct zbud_header *zhdr;int freechunks; +struct page *page; +enum buddy bud; +struct zbud_page_priv *ppriv;spin_lock (&pool->lock); zhdr =handle_to_zbud_header (handle); + page =virt_to_page (zhdr); + ppriv = (struct zbud_page_priv *)page->private; -/* If first buddy, handle will be page aligned */ -if ((handle -ZHDR_SIZE_ALIGNED ) & ~PAGE_MASK ) +if (!(handle & ~PAGE_MASK ))/* HEADLESS page stored */ + bud =HEADLESS ; +else if ((handle -ZHDR_SIZE_ALIGNED ) & ~PAGE_MASK ) { + bud =LAST ; zhdr->last_chunks =0 ; -else + }else { +/* If first buddy, handle will be page aligned */ + bud =FIRST ; zhdr->first_chunks =0 ; + } -if (zhdr->under_reclaim) { +if (ppriv->under_reclaim) {/* zbud page is under reclaim, reclaim will free */ spin_unlock (&pool->lock);return ; } -/* Remove from existing buddy list */ -list_del (&zhdr->buddy); +if (bud !=HEADLESS ) { +/* Remove from existing buddy list */ +list_del (&zhdr->buddy); + } -if (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 ) { +if (bud ==HEADLESS || + (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 )) {/* zbud page is empty, free */ -list_del (&zhdr->lru); +list_del (&page->lru);free_zbud_page (zhdr); pool->pages_nr--; }else {@@ -505,6 +554,8 @@ int zbud_reclaim_page(struct zbud_pool *pool, unsigned int retries) {int i, ret, freechunks;struct zbud_header *zhdr; +struct page *page; +struct zbud_page_priv *ppriv;unsigned long first_handle =0 , last_handle =0 ;spin_lock (&pool->lock);@@ -514,21 +565,31 @@ int zbud_reclaim_page(struct zbud_pool *pool, unsigned int retries)return -EINVAL ; }for (i =0 ; i < retries; i++) { - zhdr = list_tail_entry(&pool->lru, struct zbud_header, lru); -list_del (&zhdr->lru); -list_del (&zhdr->buddy); + page = list_tail_entry(&pool->lru, struct page, lru); + ppriv = (struct zbud_page_priv *)page->private; +list_del (&page->lru); +/* Protect zbud page against free */ - zhdr->under_reclaim = true; -/* -* We need encode the handles before unlocking, since we can -* race with free that will set (first|last)_chunks to 0 -*/ - first_handle =0 ; - last_handle =0 ; -if (zhdr->first_chunks) - first_handle =encode_handle (zhdr,FIRST ); -if (zhdr->last_chunks) - last_handle =encode_handle (zhdr,LAST ); + ppriv->under_reclaim = true; + zhdr =page_address (page); +if (!ppriv->headless) { +list_del (&zhdr->buddy); +/* +* We need encode the handles before unlocking, since +* we can race with free that will set +* (first|last)_chunks to 0 +*/ + first_handle =0 ; + last_handle =0 ; +if (zhdr->first_chunks) + first_handle =encode_handle (zhdr,FIRST ); +if (zhdr->last_chunks) + last_handle =encode_handle (zhdr,LAST ); + }else { + first_handle =encode_handle (zhdr,HEADLESS ); + last_handle =0 ; + } +spin_unlock (&pool->lock);/* Issue the eviction callback(s) */ @@ -544,8 +605,9 @@ int zbud_reclaim_page(struct zbud_pool *pool, unsigned int retries) } next:spin_lock (&pool->lock); - zhdr->under_reclaim = false; -if (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 ) { + ppriv->under_reclaim = false; +if (ppriv->headless || + (zhdr->first_chunks ==0 && zhdr->last_chunks ==0 )) {/* * Both buddies are now free, free the zbud page and * return success. @@ -565,7 +627,7 @@ next: }/* add to beginning of LRU */ -list_add (&zhdr->lru, &pool->lru); +list_add (&page->lru, &pool->lru); }spin_unlock (&pool->lock);return -EAGAIN ;Signature · 1 line
2.4.2
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 14 lines · Seth Jennings
Quoted 7 lines · Vitaly Wool
Currently zbud is only capable of allocating not more than
PAGE_SIZE-ZHDR_SIZE_ALIGNED-CHUNK_SIZE. This is okay as long as only zswap is using it, but other users of zbud may (and likely will) want to allocate up toPAGE_SIZE. This patch addresses that by skipping the creation of zbud internal structure in the beginning of an allocated page (such pages are then called 'headless').I guess I'm having trouble with this. If you store a
PAGE_SIZEallocation in zbud, then the zpage can only have one allocation as there is no room for a buddy. Sooooo... we have an allocator for that: the page allocator.
Quoted 7 lines
zbud doesn't support this by design because, if you are only storing one allocation per page, you don't gain anything.
This functionality creates many new edge cases for the code.
What is this use case you envision? I think we need to discuss whether the use case exists and if it justifies the added complexity.
The use case is to use zram with zbud as allocator via the common zpool api. Sometimes determinism and better worst-case time are more important than high compression ratio.
As far as I can see, I'm not the only one who wants this case supported in mainline.
Quoted 2 lines
We are crossing a boundary into zsmalloc style complexity with storing stuff in the struct page, something I really didn't want to do in zbud.
Well, the thing is we need PAGE_SIZE allocations supported to use zram with zbud. I can of course add the code handling this in zpool but I am quite sure doing that in zbud directly is a better idea. I'm very keen on keeping the complexity down as much as possible though.
Quoted 2 lines
zbud is the simple one, zsmalloc is the complex one. I'd hate to have two complex ones :-/
Who am I to disagree :) Keeping zbud simple is my goal, too, but once again, I'd really like it to support PAGE_SIZE allocations. And if it doesn't, the whole zpool thing for it becomes useless, since there will hardly be any zbud users other than zswap.
~vitaly
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 20 lines · Vitaly Wool
Quoted 19 lines · Seth Jennings
Quoted 7 lines · Vitaly Wool
Currently zbud is only capable of allocating not more than
PAGE_SIZE-ZHDR_SIZE_ALIGNED-CHUNK_SIZE. This is okay as long as only zswap is using it, but other users of zbud may (and likely will) want to allocate up toPAGE_SIZE. This patch addresses that by skipping the creation of zbud internal structure in the beginning of an allocated page (such pages are then called 'headless').I guess I'm having trouble with this. If you store a
PAGE_SIZEallocation in zbud, then the zpage can only have one allocation as there is no room for a buddy. Sooooo... we have an allocator for that: the page allocator.zbud doesn't support this by design because, if you are only storing one allocation per page, you don't gain anything.
This functionality creates many new edge cases for the code.
Quoted 8 lines
Quoted 2 lines
What is this use case you envision? I think we need to discuss whether the use case exists and if it justifies the added complexity.
The use case is to use zram with zbud as allocator via the common zpool api. Sometimes determinism and better worst-case time are more important than high compression ratio.
As far as I can see, I'm not the only one who wants this case supported in mainline.
Ok, I can see that having the allocator backends for zpool have the same set of constraints is nice.
I'll look at your latest patch.
Thanks,
Seth
Quoted 12 lines
Quoted 3 lines
We are crossing a boundary into zsmalloc style complexity with storing stuff in the struct page, something I really didn't want to do in zbud.
Well, the thing is we need
PAGE_SIZEallocations supported to use zram with zbud. I can of course add the code handling this in zpool but I am quite sure doing that in zbud directly is a better idea. I'm very keen on keeping the complexity down as much as possible though.Quoted 2 lines
zbud is the simple one, zsmalloc is the complex one. I'd hate to have two complex ones :-/
Quoted 6 lines
Who am I to disagree :) Keeping zbud simple is my goal, too, but once again, I'd really like it to support
PAGE_SIZEallocations. And if it doesn't, the whole zpool thing for it becomes useless, since there will hardly be any zbud users other than zswap.~vitaly
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Minchan Kim
maintainerHello,
Quoted 24 lines · Seth Jennings
Quoted 23 lines · Vitaly Wool
Quoted 21 lines · Seth Jennings
Quoted 7 lines · Vitaly Wool
Currently zbud is only capable of allocating not more than
PAGE_SIZE-ZHDR_SIZE_ALIGNED-CHUNK_SIZE. This is okay as long as only zswap is using it, but other users of zbud may (and likely will) want to allocate up toPAGE_SIZE. This patch addresses that by skipping the creation of zbud internal structure in the beginning of an allocated page (such pages are then called 'headless').I guess I'm having trouble with this. If you store a
PAGE_SIZEallocation in zbud, then the zpage can only have one allocation as there is no room for a buddy. Sooooo... we have an allocator for that: the page allocator.zbud doesn't support this by design because, if you are only storing one allocation per page, you don't gain anything.
This functionality creates many new edge cases for the code.
What is this use case you envision? I think we need to discuss whether the use case exists and if it justifies the added complexity.
Quoted 8 lines
Quoted 5 lines
The use case is to use zram with zbud as allocator via the common zpool api. Sometimes determinism and better worst-case time are more important than high compression ratio.
As far as I can see, I'm not the only one who wants this case supported in mainline.Ok, I can see that having the allocator backends for zpool have the same set of constraints is nice.
Sorry for delay. I'm on vacation until next week.
It seems Seth was missed in previous discusstion which was not the end.
I already said questions, opinion and concerns but anything is not clear until now. Only clear thing I could hear is just "compaction stats are better" which is not enough for me. Sorry.
1) https://lkml.org/lkml/2015/9/15/33
2) https://lkml.org/lkml/2015/9/21/2
Vitally, Please say what's the root cause of your problem and if it is external fragmentation, what's the problem of my approach?
1) make non-LRU page migrate
2) provide zsmalloc's migratpage
We should provide it for CMA as well as external fragmentation. I think we could solve your issue with above approach and it fundamentally makes zsmalloc/zbud happy in future.
Also, please keep it in mind that zram has been in linux kernel for memory efficiency for a long time and later zswap/zbud was born for *determinism* at the cost of memory efficiency.
Thanks.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 9 lines · Minchan Kim
Quoted 2 lines
Ok, I can see that having the allocator backends for zpool have the same set of constraints is nice.
Sorry for delay. I'm on vacation until next week.
It seems Seth was missed in previous discusstion which was not the end.I already said questions, opinion and concerns but anything is not clear until now. Only clear thing I could hear is just "compaction stats are better" which is not enough for me. Sorry.
Agree.
There weren't lots of answers, really.
Vitaly,
Have you seen those symptoms before? How did you come up to a conclusion that zram->zbud will do the trick?
If those symptoms are some sort of a recent addition, then does it help when you disable zsmalloc compaction?
---
mm/zsmalloc.c +2 -2
@@ -1944,8 +1944,8 @@ struct zs_pool *zs_create_pool(const char *name, gfp_t flags)* Not critical, we still can use the pool * and user can trigger compaction manually. */ -if (zs_register_shrinker (pool) ==0 ) - pool->shrinker_enabled = true; +/* if (zs_register_shrinker(pool) == 0) +pool->shrinker_enabled = true;*/ return pool; err:
---
p.s. I'll be on vacation next week, so most likely will be quite slow to answer.
-ss
Quoted 18 lines
1) https://lkml.org/lkml/2015/9/15/33
2) https://lkml.org/lkml/2015/9/21/2Vitally, Please say what's the root cause of your problem and if it is external fragmentation, what's the problem of my approach?
1) make non-LRU page migrate
2) provide zsmalloc's migratpageWe should provide it for CMA as well as external fragmentation. I think we could solve your issue with above approach and it fundamentally makes zsmalloc/zbud happy in future.
Also, please keep it in mind that zram has been in linux kernel for memory efficiency for a long time and later zswap/zbud was born for *determinism* at the cost of memory efficiency.
Quoted 8 lines
Thanks.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 2 lines
Have you seen those symptoms before? How did you come up to a conclusion that
zram->zbudwill do the trick?
I have data from various tests (partially described here:
https://lkml.org/lkml/2015/9/17/244) and once again, I'll post a reply to https://lkml.org/lkml/2015/9/15/33 with more detailed test description and explanation why zsmalloc is not the right choice for me.
Quoted 2 lines
If those symptoms are some sort of a recent addition, then does it help when you disable zsmalloc compaction?
No it doesn't. OTOH enabled zsmalloc compaction doesn't seem to have a substantial effect either.
~vitaly
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 5 lines · Vitaly Wool
Quoted 2 lines
Have you seen those symptoms before? How did you come up to a conclusion that
zram->zbudwill do the trick?I have data from various tests (partially described here:
https://lkml.org/lkml/2015/9/17/244) and once again, I'll post a reply
yeah, I guess I'm just not so bright to quickly understand what is wrong with zsmalloc from those numbers.
Quoted 3 lines
to https://lkml.org/lkml/2015/9/15/33 with more detailed test description and explanation why zsmalloc is not the right choice for me.
great, thanks.
Quoted 5 lines
Quoted 2 lines
If those symptoms are some sort of a recent addition, then does it help when you disable zsmalloc compaction?
No it doesn't. OTOH enabled zsmalloc compaction doesn't seem to have a substantial effect either.
hm. ok, that was my quick guess.
-ss
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
<snip>
Quoted 6 lines
I already said questions, opinion and concerns but anything is not clear until now. Only clear thing I could hear is just "compaction stats are better" which is not enough for me. Sorry.
1) https://lkml.org/lkml/2015/9/15/33
2) https://lkml.org/lkml/2015/9/21/2
Could you please stop perverting the facts, I did answer to that:
https://lkml.org/lkml/2015/9/21/753.
Apart from that, an opinion is not necessarily something I would answer. Concerns about zsmalloc are not in the scope of this patch's discussion. If you have any concerns regarding this particular patch, please let us know.
Quoted 5 lines
Vitally, Please say what's the root cause of your problem and if it is external fragmentation, what's the problem of my approach?
1) make non-LRU page migrate
2) provide zsmalloc's migratpage
The problem with your approach is that in your world I need to prove my right to use zbud. This is a very strange speculation.
Quoted 3 lines
We should provide it for CMA as well as external fragmentation. I think we could solve your issue with above approach and it fundamentally makes zsmalloc/zbud happy in future.
I doubt that but I'll answer in this thread:
https://lkml.org/lkml/2015/9/15/33 as zsmalloc deficiencies do not have direct relation to this particular patch.
Quoted 3 lines
Also, please keep it in mind that zram has been in linux kernel for memory efficiency for a long time and later zswap/zbud was born for *determinism* at the cost of memory efficiency.
Yep, and determinism is more important to me than the memory efficiency. Dropping the compression ration from 3.2x to 1.8x is okay with me and stalls in UI are not.
~vitaly
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Minchan Kim
maintainerQuoted 8 lines · Vitaly Wool
<snip>
Quoted 6 lines
I already said questions, opinion and concerns but anything is not clear until now. Only clear thing I could hear is just "compaction stats are better" which is not enough for me. Sorry.
1) https://lkml.org/lkml/2015/9/15/33
2) https://lkml.org/lkml/2015/9/21/2
Quoted 7 lines
Could you please stop perverting the facts, I did answer to that:
https://lkml.org/lkml/2015/9/21/753.Apart from that, an opinion is not necessarily something I would answer. Concerns about zsmalloc are not in the scope of this patch's discussion. If you have any concerns regarding this particular patch, please let us know.
Yes, I don't want to interrupt zbud thing which is Seth should maintain and I respect his decision but the reason I nacked is you said this patch aims for supporing zbud into zsmalloc for determinism.
For that, at least, you should discuss with me and Sergey but I feel you are ignoring our comments.
Quoted 9 lines
Quoted 6 lines
Vitally, Please say what's the root cause of your problem and if it is external fragmentation, what's the problem of my approach?
1) make non-LRU page migrate
2) provide zsmalloc's migratpageThe problem with your approach is that in your world I need to prove my right to use zbud. This is a very strange speculation.
No. If you want to contribute something, you should prove why yours is better. I already said my concerns and my approach. It's your turn that you should explain why it's better.
Quoted 9 lines
Quoted 4 lines
We should provide it for CMA as well as external fragmentation. I think we could solve your issue with above approach and it fundamentally makes zsmalloc/zbud happy in future.
I doubt that but I'll answer in this thread:
https://lkml.org/lkml/2015/9/15/33 as zsmalloc deficiencies do not have direct relation to this particular patch.
Quoted 7 lines
Quoted 3 lines
Also, please keep it in mind that zram has been in linux kernel for memory efficiency for a long time and later zswap/zbud was born for *determinism* at the cost of memory efficiency.
Yep, and determinism is more important to me than the memory efficiency. Dropping the compression ration from 3.2x to 1.8x is okay with me and stalls in UI are not.
Then, you could use zswap which have aimed for it with small changes to prevent writeback.
Quoted 2 lines
~vitaly
Signature · 8 lines
Kind regards, Minchan Kim -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Minchan Kim
maintainerQuoted 12 lines · Minchan Kim
Quoted 11 lines · Vitaly Wool
<snip>
Quoted 6 lines
I already said questions, opinion and concerns but anything is not clear until now. Only clear thing I could hear is just "compaction stats are better" which is not enough for me. Sorry.
1) https://lkml.org/lkml/2015/9/15/33
2) https://lkml.org/lkml/2015/9/21/2Could you please stop perverting the facts, I did answer to that:
https://lkml.org/lkml/2015/9/21/753.
Quoted 8 lines
Quoted 4 lines
Apart from that, an opinion is not necessarily something I would answer. Concerns about zsmalloc are not in the scope of this patch's discussion. If you have any concerns regarding this particular patch, please let us know.
Yes, I don't want to interrupt zbud thing which is Seth should maintain and I respect his decision but the reason I nacked is you said this patch aims for supporing zbud into zsmalloc for determinism.
zram
Sorry for the typo.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 30 lines · Minchan Kim
Quoted 8 lines · Vitaly Wool
<snip>
Quoted 6 lines
I already said questions, opinion and concerns but anything is not clear until now. Only clear thing I could hear is just "compaction stats are better" which is not enough for me. Sorry.
1) https://lkml.org/lkml/2015/9/15/33
2) https://lkml.org/lkml/2015/9/21/2Quoted 7 lines
Could you please stop perverting the facts, I did answer to that:
https://lkml.org/lkml/2015/9/21/753.Apart from that, an opinion is not necessarily something I would answer. Concerns about zsmalloc are not in the scope of this patch's discussion. If you have any concerns regarding this particular patch, please let us know.
Yes, I don't want to interrupt zbud thing which is Seth should maintain and I respect his decision but the reason I nacked is you said this patch aims for supporing zbud into zsmalloc for determinism.
For that, at least, you should discuss with me and Sergey but I feel you are ignoring our comments.Quoted 7 lines
Quoted 6 lines
Vitally, Please say what's the root cause of your problem and if it is external fragmentation, what's the problem of my approach?
1) make non-LRU page migrate
2) provide zsmalloc's migratpage
Quoted 6 lines
Quoted 2 lines
The problem with your approach is that in your world I need to prove my right to use zbud. This is a very strange speculation.
No. If you want to contribute something, you should prove why yours is better. I already said my concerns and my approach. It's your turn that you should explain why it's better.
In fact, I do not add any specific functionality, my patches just do what should have already been done -- that is, zram should have been converted to use zpool api long ago. Your opposing to that is counter productive.
Quoted 12 lines
Quoted 12 lines
Quoted 3 lines
We should provide it for CMA as well as external fragmentation. I think we could solve your issue with above approach and it fundamentally makes zsmalloc/zbud happy in future.
I doubt that but I'll answer in this thread:
https://lkml.org/lkml/2015/9/15/33 as zsmalloc deficiencies do not have direct relation to this particular patch.Quoted 3 lines
Also, please keep it in mind that zram has been in linux kernel for memory efficiency for a long time and later zswap/zbud was born for *determinism* at the cost of memory efficiency.
Quoted 6 lines
Quoted 3 lines
Yep, and determinism is more important to me than the memory efficiency. Dropping the compression ration from 3.2x to 1.8x is okay with me and stalls in UI are not.
Then, you could use zswap which have aimed for it with small changes to prevent writeback.
Should i come with a patch to zram explicitly stating that it is not meant to be used in any environment that is deterministic / worst case critical? Is that what you are aiming for?
~vitaly
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>