mm: convert totalram_pages, totalhigh_pages and managed pages to atomic
Subsystems (54)
v4 cover mm: convert totalram_pages, totalhigh_pages and managed pages to atomic
This series convert totalram_pages, totalhigh_pages and zone->managed_pages to atomic variables.
The patch was comiple tested on x86(x86_64_defconfig & i386_defconfig) on 4.20-rc2. And memory hotplug tested on arm64, but on an older version of kernel.
totalram_pages, zone->managed_pages and totalhigh_pages updates are protected by managed_page_count_lock, but readers never care about it. Convert these variables to atomic to avoid readers potentially seeing a store tear.
Main motivation was that managed_page_count_lock handling was complicating things. It was discussed in length here,
https://lore.kernel.org/patchwork/patch/995739/#1181785
It seemes better to remove the lock and convert variables to atomic. With the change, preventing poteintial store-to-read tearing comes as a bonus.
Changes in v4:
- Fixed kbuild test robot error.
- Modified changelog.
- Rebased to 4.20.-rc2
Changes in v3:
- Fixed kbuild test robot errors.
- Modified changelogs to be more clear.
- EXPORT_SYMBOL for _totalram_pages and _totalhigh_pages.
Arun KS (4):
mm: reference totalram_pages and managed_pages once per function
mm: convert zone->managed_pages to atomic variable
mm: convert totalram_pages and totalhigh_pages variables to atomic
mm: Remove managed_page_count spinlock
arch/csky/mm/init.c | 4 +- arch/powerpc/platforms/pseries/cmm.c | 10 ++-- arch/s390/mm/init.c | 2 +- arch/um/kernel/mem.c | 3 +- arch/x86/kernel/cpu/microcode/core.c | 5 +- drivers/char/agp/backend.c | 4 +- drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 2 +- drivers/gpu/drm/i915/i915_gem.c | 2 +- drivers/gpu/drm/i915/selftests/i915_gem_gtt.c | 4 +- drivers/hv/hv_balloon.c | 19 +++---- drivers/md/dm-bufio.c | 2 +- drivers/md/dm-crypt.c | 2 +- drivers/md/dm-integrity.c | 2 +- drivers/md/dm-stats.c | 2 +- drivers/media/platform/mtk-vpu/mtk_vpu.c | 2 +- drivers/misc/vmw_balloon.c | 2 +- drivers/parisc/ccio-dma.c | 4 +- drivers/parisc/sba_iommu.c | 4 +- drivers/staging/android/ion/ion_system_heap.c | 2 +- drivers/xen/xen-selfballoon.c | 6 +-- fs/ceph/super.h | 2 +- fs/file_table.c | 7 +-- fs/fuse/inode.c | 2 +- fs/nfs/write.c | 2 +- fs/nfsd/nfscache.c | 2 +- fs/ntfs/malloc.h | 2 +- fs/proc/base.c | 2 +- include/linux/highmem.h | 28 ++++++++++- include/linux/mm.h | 27 +++++++++- include/linux/mmzone.h | 15 +++--- include/linux/swap.h | 1 - kernel/fork.c | 5 +- kernel/kexec_core.c | 5 +- kernel/power/snapshot.c | 2 +- lib/show_mem.c | 2 +- mm/highmem.c | 5 +- mm/huge_memory.c | 2 +- mm/kasan/quarantine.c | 2 +- mm/memblock.c | 6 +-- mm/mm_init.c | 2 +- mm/oom_kill.c | 2 +- mm/page_alloc.c | 72 +++++++++++++-------------- mm/shmem.c | 7 +-- mm/slab.c | 2 +- mm/swap.c | 2 +- mm/util.c | 2 +- mm/vmalloc.c | 4 +- mm/vmstat.c | 4 +- mm/workingset.c | 2 +- mm/zswap.c | 4 +- net/dccp/proto.c | 7 +-- net/decnet/dn_route.c | 2 +- net/ipv4/tcp_metrics.c | 2 +- net/netfilter/nf_conntrack_core.c | 7 +-- net/netfilter/xt_hashlimit.c | 5 +- net/sctp/protocol.c | 7 +-- security/integrity/ima/ima_kexec.c | 2 +- 57 files changed, 195 insertions(+), 142 deletions(-)
Signature · 1 line
1.9.1
PATCH v4 1/4 mm: reference totalram_pages and managed_pages once per function
This patch is in preparation to a later patch which converts totalram_pages and zone->managed_pages to atomic variables. Please note that re-reading the value might lead to a different value and as such it could lead to unexpected behavior. There are no known bugs as a result of the current code but it is better to prevent from them in principle.
--- arch/um/kernel/mem.c | 3 +-- arch/x86/kernel/cpu/microcode/core.c | 5 +++-- drivers/hv/hv_balloon.c | 19 ++++++++++--------- fs/file_table.c | 7 ++++--- kernel/fork.c | 5 +++-- kernel/kexec_core.c | 5 +++-- mm/page_alloc.c | 5 +++-- mm/shmem.c | 3 ++- net/dccp/proto.c | 7 ++++--- net/netfilter/nf_conntrack_core.c | 7 ++++--- net/netfilter/xt_hashlimit.c | 5 +++-- net/sctp/protocol.c | 7 ++++--- 12 files changed, 44 insertions(+), 34 deletions(-)
arch/um/kernel/mem.c +1 -2
@@ -51,8 +51,7 @@ void __init mem_init(void)/* this will put all low memory onto the freelists */ memblock_free_all (); - max_low_pfn = totalram_pages; - max_pfn = totalram_pages; + max_pfn = max_low_pfn = totalram_pages;mem_init_print_info (NULL ); kmalloc_ok =1 ; }
arch/x86/kernel/cpu/microcode/core.c +3 -2
@@ -434,9 +434,10 @@ static ssize_t microcode_write(struct file *file, const char __user *buf, size_t len, loff_t *ppos) { ssize_t ret = -EINVAL; +unsigned long totalram_pgs = totalram_pages; -if ((len >>PAGE_SHIFT ) > totalram_pages) { -pr_err ("too much data (max %ld pages)\n" , totalram_pages); +if ((len >>PAGE_SHIFT ) > totalram_pgs) { +pr_err ("too much data (max %ld pages)\n" , totalram_pgs);return ret; }
drivers/hv/hv_balloon.c +10 -9
@@ -1090,6 +1090,7 @@ static void process_info(struct hv_dynmem_device *dm, struct dm_info_msg *msg)static unsigned long compute_balloon_floor (void ) {unsigned long min_pages; +unsigned long totalram_pgs = totalram_pages;#define MB2PAGES (mb) ((mb) << (20 - PAGE_SHIFT))/* Simple continuous piecewiese linear function: * max MiB -> min MiB gradient
@@ -1102,16 +1103,16 @@ static unsigned long compute_balloon_floor(void)* 8192 744 (1/16) * 32768 1512 (1/32) */ -if (totalram_pages <MB2PAGES (128 )) - min_pages =MB2PAGES (8 ) + (totalram_pages >>1 ); -else if (totalram_pages <MB2PAGES (512 )) - min_pages =MB2PAGES (40 ) + (totalram_pages >>2 ); -else if (totalram_pages <MB2PAGES (2048 )) - min_pages =MB2PAGES (104 ) + (totalram_pages >>3 ); -else if (totalram_pages <MB2PAGES (8192 )) - min_pages =MB2PAGES (232 ) + (totalram_pages >>4 ); +if (totalram_pgs <MB2PAGES (128 )) + min_pages =MB2PAGES (8 ) + (totalram_pgs >>1 ); +else if (totalram_pgs <MB2PAGES (512 )) + min_pages =MB2PAGES (40 ) + (totalram_pgs >>2 ); +else if (totalram_pgs <MB2PAGES (2048 )) + min_pages =MB2PAGES (104 ) + (totalram_pgs >>3 ); +else if (totalram_pgs <MB2PAGES (8192 )) + min_pages =MB2PAGES (232 ) + (totalram_pgs >>4 );else - min_pages =MB2PAGES (488 ) + (totalram_pages >>5 ); + min_pages =MB2PAGES (488 ) + (totalram_pgs >>5 );#undef MB2PAGESreturn min_pages; }
fs/file_table.c +4 -3
@@ -380,10 +380,11 @@ void __init files_init(void)void __init files_maxfiles_init (void ) {unsigned long n; -unsigned long memreserve = (totalram_pages -nr_free_pages ()) *3 /2 ; +unsigned long totalram_pgs = totalram_pages; +unsigned long memreserve = (totalram_pgs -nr_free_pages ()) *3 /2 ; - memreserve =min (memreserve, totalram_pages -1 ); - n = ((totalram_pages - memreserve) * (PAGE_SIZE /1024 )) /10 ; + memreserve =min (memreserve, totalram_pgs -1 ); + n = ((totalram_pgs - memreserve) * (PAGE_SIZE /1024 )) /10 ; files_stat.max_files = max_t(unsigned long, n, NR_FILE); }
kernel/fork.c +3 -2
@@ -739,15 +739,16 @@ void __init __weak arch_task_cache_init(void) { }static void set_max_threads (unsigned int max_threads_suggested) {u64 threads; +unsigned long totalram_pgs = totalram_pages;/* * The number of threads shall be limited such that the thread * structures may only consume a small part of the available memory. */ -if (fls64 (totalram_pages) +fls64 (PAGE_SIZE ) >64 ) +if (fls64 (totalram_pgs) +fls64 (PAGE_SIZE ) >64 ) threads =MAX_THREADS ;else - threads =div64_u64 ((u64 ) totalram_pages * (u64 )PAGE_SIZE , + threads =div64_u64 ((u64 ) totalram_pgs * (u64 )PAGE_SIZE , (u64 )THREAD_SIZE *8UL );if (threads > max_threads_suggested)
kernel/kexec_core.c +3 -2
@@ -152,6 +152,7 @@ int sanity_check_segment_list(struct kimage *image)int i;unsigned long nr_segments = image->nr_segments;unsigned long total_pages =0 ; +unsigned long totalram_pgs = totalram_pages;/* * Verify we have good destination addresses. The caller is
@@ -217,13 +218,13 @@ int sanity_check_segment_list(struct kimage *image)* wasted allocating pages, which can cause a soft lockup. */ for (i =0 ; i < nr_segments; i++) { -if (PAGE_COUNT (image->segment[i].memsz) > totalram_pages /2 ) +if (PAGE_COUNT (image->segment[i].memsz) > totalram_pgs /2 )return -EINVAL ; total_pages +=PAGE_COUNT (image->segment[i].memsz); } -if (total_pages > totalram_pages /2 ) +if (total_pages > totalram_pgs /2 )return -EINVAL ;/*
mm/page_alloc.c +3 -2
@@ -7245,6 +7245,7 @@ static void calculate_totalreserve_pages(void)for (i =0 ; i <MAX_NR_ZONES ; i++) {struct zone *zone = pgdat->node_zones + i;long max =0 ; +unsigned long managed_pages = zone->managed_pages;/* Find valid and maximum lowmem_reserve in the zone */ for (j = i; j <MAX_NR_ZONES ; j++) {
@@ -7255,8 +7256,8 @@ static void calculate_totalreserve_pages(void)/* we treat the high watermark as reserved pages. */ max +=high_wmark_pages (zone); -if (max > zone->managed_pages) - max = zone->managed_pages; +if (max > managed_pages) + max = managed_pages; pgdat->totalreserve_pages += max;
mm/shmem.c +2 -1
@@ -114,7 +114,8 @@ static unsigned long shmem_default_max_blocks(void)static unsigned long shmem_default_max_inodes (void ) { -return min (totalram_pages - totalhigh_pages, totalram_pages /2 ); +unsigned long totalram_pgs = totalram_pages; +return min (totalram_pgs - totalhigh_pages, totalram_pgs /2 ); }#endif
net/dccp/proto.c +4 -3
@@ -1131,6 +1131,7 @@ static inline void dccp_mib_exit(void)static int __init dccp_init (void ) {unsigned long goal; +unsigned long totalram_pgs = totalram_pages;int ehash_order, bhash_order, i;int rc;
@@ -1154,10 +1155,10 @@ static int __init dccp_init(void)* * The methodology is similar to that of the buffer cache. */ -if (totalram_pages >= (128 *1024 )) - goal = totalram_pages >> (21 -PAGE_SHIFT ); +if (totalram_pgs >= (128 *1024 )) + goal = totalram_pgs >> (21 -PAGE_SHIFT );else - goal = totalram_pages >> (23 -PAGE_SHIFT ); + goal = totalram_pgs >> (23 -PAGE_SHIFT );if (thash_entries) goal = (thash_entries *
net/netfilter/nf_conntrack_core.c +4 -3
@@ -2251,6 +2251,7 @@ static __always_inline unsigned int total_extension_size(void)int nf_conntrack_init_start (void ) { +unsigned long totalram_pgs = totalram_pages;int max_factor =8 ;int ret = -ENOMEM ;int i;
@@ -2270,11 +2271,11 @@ int nf_conntrack_init_start(void)* >= 4GB machines have 65536 buckets. */ nf_conntrack_htable_size - = (((totalram_pages <<PAGE_SHIFT ) /16384 ) + = (((totalram_pgs <<PAGE_SHIFT ) /16384 ) /sizeof (struct hlist_head )); -if (totalram_pages > (4 * (1024 *1024 *1024 /PAGE_SIZE ))) +if (totalram_pgs > (4 * (1024 *1024 *1024 /PAGE_SIZE ))) nf_conntrack_htable_size =65536 ; -else if (totalram_pages > (1024 *1024 *1024 /PAGE_SIZE )) +else if (totalram_pgs > (1024 *1024 *1024 /PAGE_SIZE )) nf_conntrack_htable_size =16384 ;if (nf_conntrack_htable_size <32 ) nf_conntrack_htable_size =32 ;
net/netfilter/xt_hashlimit.c +3 -2
@@ -274,14 +274,15 @@ static int htable_create(struct net *net, struct hashlimit_cfg3 *cfg,struct xt_hashlimit_htable *hinfo;const struct seq_operations *ops;unsigned int size, i; +unsigned long totalram_pgs = totalram_pages;int ret;if (cfg->size) { size = cfg->size; }else { - size = (totalram_pages <<PAGE_SHIFT ) /16384 / + size = (totalram_pgs <<PAGE_SHIFT ) /16384 /sizeof (struct hlist_head ); -if (totalram_pages >1024 *1024 *1024 /PAGE_SIZE ) +if (totalram_pgs >1024 *1024 *1024 /PAGE_SIZE ) size =8192 ;if (size <16 ) size =16 ;
net/sctp/protocol.c +4 -3
@@ -1368,6 +1368,7 @@ static __init int sctp_init(void)int status = -EINVAL ;unsigned long goal;unsigned long limit; +unsigned long totalram_pgs = totalram_pages;int max_share;int order;int num_entries;
@@ -1426,10 +1427,10 @@ static __init int sctp_init(void)* The methodology is similar to that of the tcp hash tables. * Though not identical. Start by getting a goal size */ -if (totalram_pages >= (128 *1024 )) - goal = totalram_pages >> (22 -PAGE_SHIFT ); +if (totalram_pgs >= (128 *1024 )) + goal = totalram_pgs >> (22 -PAGE_SHIFT );else - goal = totalram_pages >> (24 -PAGE_SHIFT ); + goal = totalram_pgs >> (24 -PAGE_SHIFT );/* Then compute the page order for said goal */ order =get_order (goal);
Signature · 1 line
1.9.1
Re: [PATCH v4 1/4] mm: reference totalram_pages and managed_pages once per function
arch/um/kernel/mem.c void __init mem_init(void) · line 54
Quoted 8 lines · Arun KS
+++
b/arch/um/kernel/mem.carch/um/kernel/mem.c +1
-2@@ -51,8 +51,7 @@ void __init mem_init(void)/* this will put all low memory onto the freelists */ memblock_free_all (); - max_low_pfn = totalram_pages; - max_pfn = totalram_pages; + max_pfn = max_low_pfn = totalram_pages;
We don't normally do "a = b = c". How about:
max_low_pfn = totalram_pages;
- max_pfn = totalram_pages;
+ max_pfn = max_low_pfn;
arch/x86/kernel/cpu/microcode/core.c static ssize_t microcode_write(struct file *file, const char __user *buf, · line 437
Quoted 6 lines
arch/x86/kernel/cpu/microcode/core.c +2
-0+++ b/arch/x86/kernel/cpu/microcode/core.c@@ -434,9 +434,10 @@ static ssize_t microcode_write(struct file *file, const char __user *buf, size_t len, loff_t *ppos) { ssize_t ret = -EINVAL; +unsigned long totalram_pgs = totalram_pages;
Can't we use a better variable name here? Even nr_pages would look better to me.
Review static ssize_t microcode_write(struct file *file, const char __user *buf, · line 439
Quoted 2 lines
(quoted diff) +2
-0+++ b/drivers/hv/hv_balloon.c + unsigned long totalram_pgs = totalram_pages;
Ditto
Review static ssize_t microcode_write(struct file *file, const char __user *buf, · line 441
Quoted 2 lines
(quoted diff) +2
-0+++ b/fs/file_table.c + unsigned long totalram_pgs = totalram_pages;
... throughout, I guess.
Re: [PATCH v4 1/4] mm: reference totalram_pages and managed_pages once per function
Hello Matthew,
Thanks for reviewing.
Quoted 12 lines · Matthew Wilcox
arch/um/kernel/mem.c void __init mem_init(void) · line 54
Quoted 8 lines · Arun KS
+++
b/arch/um/kernel/mem.carch/um/kernel/mem.c +1
-2@@ -51,8 +51,7 @@ void __init mem_init(void)/* this will put all low memory onto the freelists */ memblock_free_all (); - max_low_pfn = totalram_pages; - max_pfn = totalram_pages; + max_pfn = max_low_pfn = totalram_pages;We don't normally do "a = b = c". How about:
Review
Quoted 3 lines
(quoted diff) +1
-1max_low_pfn = totalram_pages; - max_pfn = totalram_pages; + max_pfn = max_low_pfn;
Point taken. Will fix it.
Quoted 11 lines
arch/x86/kernel/cpu/microcode/core.c static ssize_t microcode_write(struct file *file, · line 438
Quoted 8 lines
arch/x86/kernel/cpu/microcode/core.c +2
-0+++ b/arch/x86/kernel/cpu/microcode/core.c@@ -434,9 +434,10 @@ static ssize_t microcode_write(struct file *file,const char __user *buf, size_t len, loff_t *ppos) { ssize_t ret = -EINVAL; +unsigned long totalram_pgs = totalram_pages;Can't we use a better variable name here? Even
nr_pageswould look better to me.
Looks better.
Regards,
Arun
Quoted 10 lines
Review static ssize_t microcode_write(struct file *file, · line 440
Quoted 3 lines
(quoted diff) +2
-0+++ b/drivers/hv/hv_balloon.c + unsigned long totalram_pgs = totalram_pages;Ditto
Review static ssize_t microcode_write(struct file *file, · line 442
Quoted 2 lines
(quoted diff) +2
-0+++ b/fs/file_table.c + unsigned long totalram_pgs = totalram_pages;... throughout, I guess.
PATCH v4 2/4 mm: convert zone->managed_pages to atomic variable
In reply to Arun KStotalram_pages, zone->managed_pages and totalhigh_pages updates are protected by managed_page_count_lock, but readers never care about it. Convert these variables to atomic to avoid readers potentially seeing a store tear.
This patch converts zone->managed_pages. Subsequent patches will convert totalram_panges, totalhigh_pages and eventually managed_page_count_lock will be removed.
Main motivation was that managed_page_count_lock handling was complicating things. It was discussed in length here,
https://lore.kernel.org/patchwork/patch/995739/#1181785
So it seemes better to remove the lock and convert variables to atomic, with preventing poteintial store-to-read tearing as a bonus.
---
Most of the changes are done by below coccinelle script,
@@
struct zone *z;
expression e1;
@@
(
- z->managed_pages = e1
+ atomic_long_set(&z->managed_pages, e1)
|
- e1->managed_pages++
+ atomic_long_inc(&e1->managed_pages)
|
- z->managed_pages
+ zone_managed_pages(z)
)
@@
expression e,e1;
@@
- e->managed_pages += e1
+ atomic_long_add(e1, &e->managed_pages)
@@
expression z;
@@
- z.managed_pages
+ zone_managed_pages(&z)
Then, manually apply following change,include/linux/mmzone.h
- unsigned long managed_pages;
+ atomic_long_t managed_pages;
+static inline unsigned long zone_managed_pages(struct zone *zone) +{
+ return (unsigned long)atomic_long_read(&zone->managed_pages);
+}
---
--- drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 2 +- include/linux/mmzone.h | 9 +++++-- lib/show_mem.c | 2 +- mm/memblock.c | 2 +- mm/page_alloc.c | 44 +++++++++++++++++------------------ mm/vmstat.c | 4 ++-- 6 files changed, 34 insertions(+), 29 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +1 -1
@@ -848,7 +848,7 @@ static int kfd_fill_mem_info_for_cpu(int numa_node_id, int *avail_size,*/ pgdat =NODE_DATA (numa_node_id);for (zone_type =0 ; zone_type <MAX_NR_ZONES ; zone_type++) - mem_in_bytes += pgdat->node_zones[zone_type].managed_pages; + mem_in_bytes +=zone_managed_pages (&pgdat->node_zones[zone_type]); mem_in_bytes <<=PAGE_SHIFT ; sub_type_hdr->length_low =lower_32_bits (mem_in_bytes);
include/linux/mmzone.h +7 -2
@@ -435,7 +435,7 @@ struct zone {* adjust_managed_page_count() should be used instead of directly * touching zone->managed_pages and totalram_pages. */ -unsigned long managed_pages; +atomic_long_t managed_pages;unsigned long spanned_pages;unsigned long present_pages;
@@ -524,6 +524,11 @@ enum pgdat_flags { PGDAT_RECLAIM_LOCKED, /* prevents concurrent reclaim */ }; +static inline unsigned long zone_managed_pages (struct zone *zone) +{ +return (unsigned long )atomic_long_read (&zone->managed_pages); +} +static inline unsigned long zone_end_pfn (const struct zone *zone) {return zone->zone_start_pfn + zone->spanned_pages;
@@ -814,7 +819,7 @@ static inline bool is_dev_zone(const struct zone *zone)*/ static inline bool managed_zone (struct zone *zone) { -return zone->managed_pages; +return zone_managed_pages (zone); }/* Returns true if a zone has memory */
lib/show_mem.c +1 -1
@@ -28,7 +28,7 @@ void show_mem(unsigned int filter, nodemask_t *nodemask)continue ; total += zone->present_pages; - reserved += zone->present_pages - zone->managed_pages; + reserved += zone->present_pages -zone_managed_pages (zone);if (is_highmem_idx (zoneid)) highmem += zone->present_pages;
mm/memblock.c +1 -1
@@ -1950,7 +1950,7 @@ void reset_node_managed_pages(pg_data_t *pgdat)struct zone *z;for (z = pgdat->node_zones; z < pgdat->node_zones +MAX_NR_ZONES ; z++) - z->managed_pages =0 ; +atomic_long_set (&z->managed_pages,0 ); }void __init reset_all_zones_managed_pages (void )
mm/page_alloc.c +22 -22
164 lines not on this page · raw message
mm/vmstat.c +2 -2
@@ -227,7 +227,7 @@ int calculate_normal_threshold(struct zone *zone)* 125 1024 10 16-32 GB 9 */ - mem = zone->managed_pages >> (27 -PAGE_SHIFT ); + mem =zone_managed_pages (zone) >> (27 -PAGE_SHIFT ); threshold =2 *fls (num_online_cpus ()) * (1 +fls (mem));
@@ -1569,7 +1569,7 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,high_wmark_pages (zone), zone->spanned_pages, zone->present_pages, - zone->managed_pages); + zone_managed_pages(zone)); seq_printf(m, "\n protection: (%ld",
Signature · 1 line
1.9.1
PATCH v4 3/4 mm: convert totalram_pages and totalhigh_pages variables to atomic
In reply to Arun KStotalram_pages and totalhigh_pages are made static inline function.
Main motivation was that managed_page_count_lock handling was complicating things. It was discussed in lenght here,
https://lore.kernel.org/patchwork/patch/995739/#1181785
So it seemes better to remove the lock and convert variables to atomic, with preventing poteintial store-to-read tearing as a bonus.
---
coccinelle script to make most of the changes,
@@
declarer name EXPORT_SYMBOL;
symbol totalram_pages;
expression e;
@@
(EXPORT_SYMBOL(totalram_pages);
|
- totalram_pages = e
+ totalram_pages_set(e)
|
- totalram_pages += e
+ totalram_pages_add(e)
|
- totalram_pages++
+ totalram_pages_inc()
|
- totalram_pages--
+ totalram_pages_dec()
|
- totalram_pages
+ totalram_pages()
)
@@
symbol totalhigh_pages;
expression e;
@@
(EXPORT_SYMBOL(totalhigh_pages);
|
- totalhigh_pages = e
+ totalhigh_pages_set(e)
|
- totalhigh_pages += e
+ totalhigh_pages_add(e)
|
- totalhigh_pages++
+ totalhigh_pages_inc()
|
- totalhigh_pages--
+ totalhigh_pages_dec()
|
- totalhigh_pages
+ totalhigh_pages()
)
Manaually apply all changes of following files,
include/linux/highmem.hinclude/linux/mm.hinclude/linux/swap.hmm/highmem.c
and for mm/page_alloc.c mannualy apply only below changes,
#include <linux/stddef.h>
#include <linux/mm.h>
+#include <linux/highmem.h>
#include <linux/swap.h>
#include <linux/interrupt.h>
#include <linux/pagemap.h>
/* Protect totalram_pages and zone->managed_pages */ static DEFINE_SPINLOCK(managed_page_count_lock);
-unsigned long totalram_pages __read_mostly;
+atomic_long_t _totalram_pages __read_mostly;
unsigned long totalreserve_pages __read_mostly;
unsigned long totalcma_pages __read_mostly;
---
--- arch/csky/mm/init.c | 4 ++-- arch/powerpc/platforms/pseries/cmm.c | 10 +++++----- arch/s390/mm/init.c | 2 +- arch/um/kernel/mem.c | 2 +- arch/x86/kernel/cpu/microcode/core.c | 2 +- drivers/char/agp/backend.c | 4 ++-- drivers/gpu/drm/i915/i915_gem.c | 2 +- drivers/gpu/drm/i915/selftests/i915_gem_gtt.c | 4 ++-- drivers/hv/hv_balloon.c | 2 +- drivers/md/dm-bufio.c | 2 +- drivers/md/dm-crypt.c | 2 +- drivers/md/dm-integrity.c | 2 +- drivers/md/dm-stats.c | 2 +- drivers/media/platform/mtk-vpu/mtk_vpu.c | 2 +- drivers/misc/vmw_balloon.c | 2 +- drivers/parisc/ccio-dma.c | 4 ++-- drivers/parisc/sba_iommu.c | 4 ++-- drivers/staging/android/ion/ion_system_heap.c | 2 +- drivers/xen/xen-selfballoon.c | 6 +++--- fs/ceph/super.h | 2 +- fs/file_table.c | 2 +- fs/fuse/inode.c | 2 +- fs/nfs/write.c | 2 +- fs/nfsd/nfscache.c | 2 +- fs/ntfs/malloc.h | 2 +- fs/proc/base.c | 2 +- include/linux/highmem.h | 28 +++++++++++++++++++++++++-- include/linux/mm.h | 27 +++++++++++++++++++++++++- include/linux/swap.h | 1 - kernel/fork.c | 2 +- kernel/kexec_core.c | 2 +- kernel/power/snapshot.c | 2 +- mm/highmem.c | 5 ++--- mm/huge_memory.c | 2 +- mm/kasan/quarantine.c | 2 +- mm/memblock.c | 4 ++-- mm/mm_init.c | 2 +- mm/oom_kill.c | 2 +- mm/page_alloc.c | 20 ++++++++++--------- mm/shmem.c | 8 ++++---- mm/slab.c | 2 +- mm/swap.c | 2 +- mm/util.c | 2 +- mm/vmalloc.c | 4 ++-- mm/workingset.c | 2 +- mm/zswap.c | 4 ++-- net/dccp/proto.c | 2 +- net/decnet/dn_route.c | 2 +- net/ipv4/tcp_metrics.c | 2 +- net/netfilter/nf_conntrack_core.c | 2 +- net/netfilter/xt_hashlimit.c | 2 +- net/sctp/protocol.c | 2 +- security/integrity/ima/ima_kexec.c | 2 +- 53 files changed, 130 insertions(+), 81 deletions(-)
arch/csky/mm/init.c +2 -2
@@ -71,7 +71,7 @@ void free_initrd_mem(unsigned long start, unsigned long end)ClearPageReserved (virt_to_page (start));init_page_count (virt_to_page (start));free_page (start); - totalram_pages++; + totalram_pages_inc(); } }#endif
@@ -88,7 +88,7 @@ void free_initmem(void)ClearPageReserved (virt_to_page (addr));init_page_count (virt_to_page (addr));free_page (addr); - totalram_pages++; +totalram_pages_inc (); addr +=PAGE_SIZE ; }
arch/powerpc/platforms/pseries/cmm.c +5 -5
@@ -208,7 +208,7 @@ static long cmm_alloc_pages(long nr) pa->page[pa->index++] = addr; loaned_pages++; - totalram_pages--; +totalram_pages_dec ();spin_unlock (&cmm_lock); nr--; }
@@ -247,7 +247,7 @@ static long cmm_free_pages(long nr)free_page (addr); loaned_pages--; nr--; - totalram_pages++; +totalram_pages_inc (); }spin_unlock (&cmm_lock);cmm_dbg ("End request with %ld pages unfulfilled\n" , nr);
@@ -291,7 +291,7 @@ static void cmm_get_mpp(void)int rc;struct hvcall_mpp_data mpp_data;signed long active_pages_target, page_loan_request, target; -signed long total_pages = totalram_pages + loaned_pages; +signed long total_pages =totalram_pages () + loaned_pages;signed long min_mem_pages = (min_mem_mb *1024 *1024 ) /PAGE_SIZE ; rc =h_get_mpp (&mpp_data);
@@ -322,7 +322,7 @@ static void cmm_get_mpp(void)cmm_dbg ("delta = %ld, loaned = %lu, target = %lu, oom = %lu, totalram = %lu\n" , page_loan_request, loaned_pages, loaned_pages_target, - oom_freed_pages, totalram_pages); + oom_freed_pages,totalram_pages ()); }static struct notifier_block cmm_oom_nb = {
@@ -581,7 +581,7 @@ static int cmm_mem_going_offline(void *arg)free_page (pa_curr->page[idx]); freed++; loaned_pages--; - totalram_pages++; +totalram_pages_inc (); pa_curr->page[idx] = pa_last->page[--pa_last->index];if (pa_last->index ==0 ) {if (pa_curr == pa_last)
arch/s390/mm/init.c +1 -1
@@ -59,7 +59,7 @@ static void __init setup_zero_pages(void) order =7 ;/* Limit number of empty zero pages for small memory sizes */ -while (order >2 && (totalram_pages >>10 ) < (1UL << order)) +while (order >2 && (totalram_pages () >>10 ) < (1UL << order)) order--; empty_zero_page =__get_free_pages (GFP_KERNEL | __GFP_ZERO, order);
arch/um/kernel/mem.c +1 -1
@@ -51,7 +51,7 @@ void __init mem_init(void)/* this will put all low memory onto the freelists */ memblock_free_all (); - max_pfn = max_low_pfn = totalram_pages; + max_pfn = max_low_pfn =totalram_pages ();mem_init_print_info (NULL ); kmalloc_ok =1 ; }
arch/x86/kernel/cpu/microcode/core.c +1 -1
@@ -434,7 +434,7 @@ static ssize_t microcode_write(struct file *file, const char __user *buf, size_t len, loff_t *ppos) { ssize_t ret = -EINVAL; -unsigned long totalram_pgs = totalram_pages; +unsigned long totalram_pgs =totalram_pages ();if ((len >>PAGE_SHIFT ) > totalram_pgs) {pr_err ("too much data (max %ld pages)\n" , totalram_pgs);
drivers/char/agp/backend.c +2 -2
@@ -115,9 +115,9 @@ static int agp_find_max(void)long memory, index, result;#if PAGE_SHIFT <20 - memory = totalram_pages >> (20 -PAGE_SHIFT ); + memory =totalram_pages () >> (20 -PAGE_SHIFT );#else - memory = totalram_pages << (PAGE_SHIFT -20 ); + memory =totalram_pages () << (PAGE_SHIFT -20 );#endif index =1 ;
drivers/gpu/drm/i915/i915_gem.c +1 -1
@@ -2539,7 +2539,7 @@ static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)* If there's no chance of allocating enough pages for the whole * object, bail early. */ -if (page_count > totalram_pages) +if (page_count >totalram_pages ())return -ENOMEM ; st =kmalloc (sizeof (*st),GFP_KERNEL );
drivers/gpu/drm/i915/selftests/i915_gem_gtt.c +2 -2
@@ -170,7 +170,7 @@ static int igt_ppgtt_alloc(void *arg)* This should ensure that we do not run into the oomkiller during * the test and take down the machine wilfully. */ - limit = totalram_pages <<PAGE_SHIFT ; + limit =totalram_pages () <<PAGE_SHIFT ; limit =min (ppgtt->vm.total, limit);/* Check we can allocate the entire range */
@@ -1244,7 +1244,7 @@ static int exercise_mock(struct drm_i915_private *i915, u64 hole_start, u64 hole_end, unsigned long end_time)) { - const u64 limit = totalram_pages << PAGE_SHIFT; + const u64 limit = totalram_pages() << PAGE_SHIFT;struct i915_gem_context *ctx;struct i915_hw_ppgtt *ppgtt;IGT_TIMEOUT (end_time);
drivers/hv/hv_balloon.c +1 -1
@@ -1090,7 +1090,7 @@ static void process_info(struct hv_dynmem_device *dm, struct dm_info_msg *msg)static unsigned long compute_balloon_floor (void ) {unsigned long min_pages; -unsigned long totalram_pgs = totalram_pages; +unsigned long totalram_pgs =totalram_pages ();#define MB2PAGES (mb) ((mb) << (20 - PAGE_SHIFT))/* Simple continuous piecewiese linear function: * max MiB -> min MiB gradient
drivers/md/dm-bufio.c +1 -1
@@ -1887,7 +1887,7 @@ static int __init dm_bufio_init(void) dm_bufio_allocated_vmalloc =0 ; dm_bufio_current_allocated =0 ; - mem = (__u64 )mult_frac (totalram_pages - totalhigh_pages, + mem = (__u64 )mult_frac (totalram_pages () -totalhigh_pages (),DM_BUFIO_MEMORY_PERCENT ,100 ) <<PAGE_SHIFT ;if (mem >ULONG_MAX )
drivers/md/dm-crypt.c +1 -1
@@ -2158,7 +2158,7 @@ static int crypt_wipe_key(struct crypt_config *cc)static void crypt_calculate_pages_per_client (void ) { -unsigned long pages = (totalram_pages - totalhigh_pages) *DM_CRYPT_MEMORY_PERCENT /100 ; +unsigned long pages = (totalram_pages () -totalhigh_pages ()) *DM_CRYPT_MEMORY_PERCENT /100 ;if (!dm_crypt_clients_n)return ;
drivers/md/dm-integrity.c +1 -1
@@ -2843,7 +2843,7 @@ static int create_journal(struct dm_integrity_c *ic, char **error) journal_pages =roundup ((__u64 )ic->journal_sections * ic->journal_section_sectors,PAGE_SIZE >>SECTOR_SHIFT ) >> (PAGE_SHIFT -SECTOR_SHIFT ); journal_desc_size = journal_pages *sizeof (struct page_list ); -if (journal_pages >= totalram_pages - totalhigh_pages || journal_desc_size >ULONG_MAX ) { +if (journal_pages >=totalram_pages () -totalhigh_pages () || journal_desc_size >ULONG_MAX ) { *error ="Journal doesn't fit into memory" ; r = -ENOMEM ; goto bad;
drivers/md/dm-stats.c +1 -1
@@ -85,7 +85,7 @@ static bool __check_shared_memory(size_t alloc_size) a = shared_memory_amount + alloc_size;if (a < shared_memory_amount)return false; -if (a >>PAGE_SHIFT > totalram_pages /DM_STATS_MEMORY_FACTOR ) +if (a >>PAGE_SHIFT >totalram_pages () /DM_STATS_MEMORY_FACTOR )return false;#ifdef CONFIG_MMU if (a > (VMALLOC_END -VMALLOC_START ) /DM_STATS_VMALLOC_FACTOR )
drivers/media/platform/mtk-vpu/mtk_vpu.c +1 -1
@@ -855,7 +855,7 @@ static int mtk_vpu_probe(struct platform_device *pdev)/* Set PTCM to 96K and DTCM to 32K */ vpu_cfg_writel (vpu,0x2 ,VPU_TCM_CFG ); - vpu->enable_4GB = !!(totalram_pages > (SZ_2G >>PAGE_SHIFT )); + vpu->enable_4GB = !!(totalram_pages () > (SZ_2G >>PAGE_SHIFT ));dev_info (dev,"4GB mode %u\n" , vpu->enable_4GB);if (vpu->enable_4GB) {
drivers/misc/vmw_balloon.c +1 -1
@@ -570,7 +570,7 @@ static int vmballoon_send_get_target(struct vmballoon *b)unsigned long status;unsigned long limit; - limit = totalram_pages; + limit =totalram_pages ();/* Ensure limit fits in 32-bits */ if (limit != (u32 )limit)
drivers/parisc/ccio-dma.c +2 -2
@@ -1251,7 +1251,7 @@ void __init ccio_cujo20_fixup(struct parisc_device *cujo, u32 iovp)** Hot-Plug/Removal of PCI cards. (aka PCI OLARD). */ - iova_space_size = (u32) (totalram_pages /count_parisc_driver (&ccio_driver)); + iova_space_size = (u32) (totalram_pages () /count_parisc_driver (&ccio_driver));/* limit IOVA space size to 1MB-1GB */
@@ -1290,7 +1290,7 @@ void __init ccio_cujo20_fixup(struct parisc_device *cujo, u32 iovp)DBG_INIT ("%s() hpa 0x%p mem %luMB IOV %dMB (%d bits)\n" , __func__, ioc->ioc_regs, - (unsigned long ) totalram_pages >> (20 -PAGE_SHIFT ), + (unsigned long )totalram_pages () >> (20 -PAGE_SHIFT ), iova_space_size>>20 , iov_order +PAGE_SHIFT );
drivers/parisc/sba_iommu.c +2 -2
@@ -1414,7 +1414,7 @@ static int setup_ibase_imask_callback(struct device *dev, void *data)** for DMA hints - ergo only 30 bits max. */ - iova_space_size = (u32) (totalram_pages/global_ioc_cnt); + iova_space_size = (u32) (totalram_pages ()/global_ioc_cnt);/* limit IOVA space size to 1MB-1GB */ if (iova_space_size < (1 << (20 -PAGE_SHIFT ))) {
@@ -1439,7 +1439,7 @@ static int setup_ibase_imask_callback(struct device *dev, void *data)DBG_INIT ("%s() hpa 0x%lx mem %ldMB IOV %dMB (%d bits)\n" , __func__, ioc->ioc_hpa, - (unsigned long ) totalram_pages >> (20 -PAGE_SHIFT ), + (unsigned long )totalram_pages () >> (20 -PAGE_SHIFT ), iova_space_size>>20 , iov_order +PAGE_SHIFT );
drivers/staging/android/ion/ion_system_heap.c +1 -1
@@ -110,7 +110,7 @@ static int ion_system_heap_allocate(struct ion_heap *heap,unsigned long size_remaining =PAGE_ALIGN (size);unsigned int max_order = orders[0 ]; -if (size /PAGE_SIZE > totalram_pages /2 ) +if (size /PAGE_SIZE >totalram_pages () /2 )return -ENOMEM ;INIT_LIST_HEAD (&pages);
drivers/xen/xen-selfballoon.c +3 -3
@@ -189,7 +189,7 @@ static void selfballoon_process(struct work_struct *work)bool reset_timer = false;if (xen_selfballooning_enabled) { - cur_pages = totalram_pages; + cur_pages =totalram_pages (); tgt_pages = cur_pages;/* default is no change */ goal_pages =vm_memory_committed () + totalreserve_pages +
@@ -227,7 +227,7 @@ static void selfballoon_process(struct work_struct *work)if (tgt_pages < floor_pages) tgt_pages = floor_pages;balloon_set_new_target (tgt_pages + - balloon_stats.current_pages - totalram_pages); + balloon_stats.current_pages -totalram_pages ()); reset_timer = true; }#ifdef CONFIG_FRONTSWAP
@@ -569,7 +569,7 @@ int xen_selfballoon_init(bool use_selfballooning, bool use_frontswap_selfshrink)* much more reliably and response faster in some cases. */ if (!selfballoon_reserved_mb) { - reserve_pages = totalram_pages /10 ; + reserve_pages =totalram_pages () /10 ; selfballoon_reserved_mb =PAGES2MB (reserve_pages); }schedule_delayed_work (&selfballoon_worker, selfballoon_interval *HZ );
fs/ceph/super.h +1 -1
@@ -808,7 +808,7 @@ static inline int default_congestion_kb(void)* This allows larger machines to have larger/more transfers. * Limit the default to 256M */ - congestion_kb = (16 *int_sqrt (totalram_pages)) << (PAGE_SHIFT -10 ); + congestion_kb = (16 *int_sqrt (totalram_pages ())) << (PAGE_SHIFT -10 );if (congestion_kb >256 *1024 ) congestion_kb =256 *1024 ;
fs/file_table.c +1 -1
@@ -380,7 +380,7 @@ void __init files_init(void)void __init files_maxfiles_init (void ) {unsigned long n; -unsigned long totalram_pgs = totalram_pages; +unsigned long totalram_pgs =totalram_pages ();unsigned long memreserve = (totalram_pgs -nr_free_pages ()) *3 /2 ; memreserve =min (memreserve, totalram_pgs -1 );
fs/fuse/inode.c +1 -1
@@ -824,7 +824,7 @@ static struct dentry *fuse_get_parent(struct dentry *child)static void sanitize_global_limit (unsigned *limit) {if (*limit ==0 ) - *limit = ((totalram_pages <<PAGE_SHIFT ) >>13 ) / + *limit = ((totalram_pages () <<PAGE_SHIFT ) >>13 ) /sizeof (struct fuse_req );if (*limit >=1 <<16 )
fs/nfs/write.c +1 -1
@@ -2121,7 +2121,7 @@ int __init nfs_init_writepagecache(void)* This allows larger machines to have larger/more transfers. * Limit the default to 256M */ - nfs_congestion_kb = (16 *int_sqrt (totalram_pages)) << (PAGE_SHIFT -10 ); + nfs_congestion_kb = (16 *int_sqrt (totalram_pages ())) << (PAGE_SHIFT -10 );if (nfs_congestion_kb >256 *1024 ) nfs_congestion_kb =256 *1024 ;
fs/nfsd/nfscache.c +1 -1
@@ -99,7 +99,7 @@ static unsigned long nfsd_reply_cache_scan(struct shrinker *shrink,nfsd_cache_size_limit (void) {unsigned int limit; -unsigned long low_pages = totalram_pages - totalhigh_pages; +unsigned long low_pages =totalram_pages () -totalhigh_pages (); limit = (16 *int_sqrt (low_pages)) << (PAGE_SHIFT -10 ); return min_t(unsigned int, limit, 256*1024);
fs/ntfs/malloc.h +1 -1
@@ -47,7 +47,7 @@ static inline void *__ntfs_malloc(unsigned long size, gfp_t gfp_mask)return kmalloc (PAGE_SIZE , gfp_mask & ~__GFP_HIGHMEM);/* return (void *)__get_free_page(gfp_mask); */ } -if (likely ((size >>PAGE_SHIFT ) < totalram_pages)) +if (likely ((size >>PAGE_SHIFT ) <totalram_pages ()))return __vmalloc (size, gfp_mask,PAGE_KERNEL );return NULL ; }
fs/proc/base.c +1 -1
@@ -530,7 +530,7 @@ static ssize_t lstats_write(struct file *file, const char __user *buf,static int proc_oom_score (struct seq_file *m,struct pid_namespace *ns,struct pid *pid,struct task_struct *task) { -unsigned long totalpages = totalram_pages + total_swap_pages; +unsigned long totalpages =totalram_pages () + total_swap_pages;unsigned long points =0 ; points = oom_badness(task, NULL, NULL, totalpages) *
include/linux/highmem.h +26 -2
@@ -36,7 +36,31 @@ static inline void invalidate_kernel_vmap_range(void *vaddr, int size)/* declarations for linux/mm/highmem.c */ unsigned int nr_free_highpages (void ); -extern unsigned long totalhigh_pages; +extern atomic_long_t _totalhigh_pages; +static inline unsigned long totalhigh_pages (void ) +{ +return (unsigned long )atomic_long_read (&_totalhigh_pages); +} + +static inline void totalhigh_pages_inc (void ) +{ +atomic_long_inc (&_totalhigh_pages); +} + +static inline void totalhigh_pages_dec (void ) +{ +atomic_long_dec (&_totalhigh_pages); +} + +static inline void totalhigh_pages_add (long count) +{ +atomic_long_add (count, &_totalhigh_pages); +} + +static inline void totalhigh_pages_set (long val) +{ +atomic_long_set (&_totalhigh_pages, val); +}void kmap_flush_unused (void );
@@ -51,7 +75,7 @@ static inline struct page *kmap_to_page(void *addr)return virt_to_page (addr); } -#define totalhigh_pages 0UL +static inline unsigned long totalhigh_pages (void ) {return 0UL ; }#ifndef ARCH_HAS_KMAP static inline void *kmap (struct page *page)
include/linux/mm.h +26 -1
@@ -48,7 +48,32 @@ static inline void set_max_mapnr(unsigned long limit)static inline void set_max_mapnr (unsigned long limit) { }#endif -extern unsigned long totalram_pages; +extern atomic_long_t _totalram_pages; +static inline unsigned long totalram_pages (void ) +{ +return (unsigned long )atomic_long_read (&_totalram_pages); +} + +static inline void totalram_pages_inc (void ) +{ +atomic_long_inc (&_totalram_pages); +} + +static inline void totalram_pages_dec (void ) +{ +atomic_long_dec (&_totalram_pages); +} + +static inline void totalram_pages_add (long count) +{ +atomic_long_add (count, &_totalram_pages); +} + +static inline void totalram_pages_set (long val) +{ +atomic_long_set (&_totalram_pages, val); +} +extern void * high_memory;extern int page_cluster;
include/linux/swap.h +0 -1
@@ -308,7 +308,6 @@ struct vma_swap_readahead { }while (0 )/* linux/mm/page_alloc.c */ -extern unsigned long totalram_pages; extern unsigned long totalreserve_pages;extern unsigned long nr_free_buffer_pages (void );extern unsigned long nr_free_pagecache_pages (void );
kernel/fork.c +1 -1
@@ -739,7 +739,7 @@ void __init __weak arch_task_cache_init(void) { }static void set_max_threads (unsigned int max_threads_suggested) {u64 threads; -unsigned long totalram_pgs = totalram_pages; +unsigned long totalram_pgs =totalram_pages ();/* * The number of threads shall be limited such that the thread
kernel/kexec_core.c +1 -1
@@ -152,7 +152,7 @@ int sanity_check_segment_list(struct kimage *image)int i;unsigned long nr_segments = image->nr_segments;unsigned long total_pages =0 ; -unsigned long totalram_pgs = totalram_pages; +unsigned long totalram_pgs =totalram_pages ();/* * Verify we have good destination addresses. The caller is
kernel/power/snapshot.c +1 -1
@@ -105,7 +105,7 @@ void __init hibernate_reserved_size_init(void)void __init hibernate_image_size_init (void ) { - image_size = ((totalram_pages *2 ) /5 ) *PAGE_SIZE ; + image_size = ((totalram_pages () *2 ) /5 ) *PAGE_SIZE ; }/*
mm/highmem.c +2 -3
@@ -105,9 +105,8 @@ static inline wait_queue_head_t *get_pkmap_wait_queue_head(unsigned int color) }#endif -unsigned long totalhigh_pages__read_mostly ; -EXPORT_SYMBOL (totalhigh_pages); - +atomic_long_t _totalhigh_pages__read_mostly ; +EXPORT_SYMBOL (_totalhigh_pages);EXPORT_PER_CPU_SYMBOL (__kmap_atomic_idx);
mm/huge_memory.c +1 -1
@@ -420,7 +420,7 @@ static int __init hugepage_init(void)* where the extra memory used could hurt more than TLB overhead * is likely to save. The admin can still enable it through /sys. */ -if (totalram_pages < (512 << (20 -PAGE_SHIFT ))) { +if (totalram_pages () < (512 << (20 -PAGE_SHIFT ))) { transparent_hugepage_flags =0 ;return 0 ; }
mm/kasan/quarantine.c +1 -1
@@ -236,7 +236,7 @@ void quarantine_reduce(void)* Update quarantine size in case of hotplug. Allocate a fraction of * the installed memory to quarantine minus per-cpu queue limits. */ - total_size = (READ_ONCE (totalram_pages) <<PAGE_SHIFT ) / + total_size = (totalram_pages () <<PAGE_SHIFT ) /QUARANTINE_FRACTION ; percpu_quarantines =QUARANTINE_PERCPU_SIZE *num_online_cpus (); new_quarantine_size = (total_size < percpu_quarantines) ?
mm/memblock.c +2 -2
@@ -1576,7 +1576,7 @@ void __init __memblock_free_late(phys_addr_t base, phys_addr_t size)for (; cursor < end; cursor++) {memblock_free_pages (pfn_to_page (cursor), cursor,0 ); - totalram_pages++; +totalram_pages_inc (); } }
@@ -1978,7 +1978,7 @@ unsigned long __init memblock_free_all(void)reset_all_zones_managed_pages (); pages =free_low_memory_core_early (); - totalram_pages += pages; +totalram_pages_add (pages);return pages; }
mm/mm_init.c +1 -1
@@ -146,7 +146,7 @@ static void __meminit mm_compute_batch(void)s32 batch =max_t (s32, nr*2 ,32 );/* batch size set to 0.4% of (total memory/#cpus), or max int32 */ - memsized_batch =min_t (u64, (totalram_pages/nr)/256 ,0x7fffffff ); + memsized_batch =min_t (u64, (totalram_pages ()/nr)/256 ,0x7fffffff ); vm_committed_as_batch =max_t (s32, memsized_batch, batch); }
mm/oom_kill.c +1 -1
@@ -269,7 +269,7 @@ static enum oom_constraint constrained_alloc(struct oom_control *oc) }/* Default to all available memory */ - oc->totalpages = totalram_pages + total_swap_pages; + oc->totalpages =totalram_pages () + total_swap_pages;if (!IS_ENABLED (CONFIG_NUMA ))return CONSTRAINT_NONE ;
mm/page_alloc.c +11 -9
@@ -16,6 +16,7 @@#include <linux/stddef.h> #include <linux/mm.h> +#include <linux/highmem.h> #include <linux/swap.h> #include <linux/interrupt.h> #include <linux/pagemap.h>
@@ -124,7 +125,8 @@/* Protect totalram_pages and zone->managed_pages */ static DEFINE_SPINLOCK (managed_page_count_lock); -unsigned long totalram_pages__read_mostly ; +atomic_long_t _totalram_pages__read_mostly ; +EXPORT_SYMBOL (_totalram_pages);unsigned long totalreserve_pages__read_mostly ;unsigned long totalcma_pages__read_mostly ;
@@ -4748,11 +4750,11 @@ long si_mem_available(void)void si_meminfo (struct sysinfo *val) { - val->totalram = totalram_pages; + val->totalram =totalram_pages (); val->sharedram =global_node_page_state (NR_SHMEM ); val->freeram =global_zone_page_state (NR_FREE_PAGES ); val->bufferram =nr_blockdev_pages (); - val->totalhigh = totalhigh_pages; + val->totalhigh =totalhigh_pages (); val->freehigh =nr_free_highpages (); val->mem_unit =PAGE_SIZE ; }
@@ -7065,10 +7067,10 @@ void adjust_managed_page_count(struct page *page, long count) {spin_lock (&managed_page_count_lock);atomic_long_add (count, &page_zone (page)->managed_pages); - totalram_pages += count; +totalram_pages_add (count);#ifdef CONFIG_HIGHMEM if (PageHighMem (page)) - totalhigh_pages += count; +totalhigh_pages_add (count);#endif spin_unlock (&managed_page_count_lock); }
@@ -7111,9 +7113,9 @@ unsigned long free_reserved_area(void *start, void *end, int poison, char *s)void free_highmem_page (struct page *page) {__free_reserved_page (page); - totalram_pages++; +totalram_pages_inc ();atomic_long_inc (&page_zone (page)->managed_pages); - totalhigh_pages++; +totalhigh_pages_inc (); }#endif
@@ -7162,10 +7164,10 @@ void __init mem_init_print_info(const char *str) physpages << (PAGE_SHIFT -10 ), codesize >>10 , datasize >>10 , rosize >>10 , (init_data_size + init_code_size) >>10 , bss_size >>10 , - (physpages - totalram_pages - totalcma_pages) << (PAGE_SHIFT -10 ), + (physpages -totalram_pages () - totalcma_pages) << (PAGE_SHIFT -10 ), totalcma_pages << (PAGE_SHIFT -10 ), #ifdef CONFIG_HIGHMEM - totalhigh_pages << (PAGE_SHIFT -10 ), +totalhigh_pages () << (PAGE_SHIFT -10 ), #endif str ? ", " : "", str ? str : ""); }
mm/shmem.c +4 -4
@@ -109,13 +109,13 @@ struct shmem_falloc {#ifdef CONFIG_TMPFS static unsigned long shmem_default_max_blocks (void ) { -return totalram_pages /2 ; +return totalram_pages () /2 ; }static unsigned long shmem_default_max_inodes (void ) { -unsigned long totalram_pgs = totalram_pages; -return min (totalram_pgs - totalhigh_pages, totalram_pgs /2 ); +unsigned long totalram_pgs =totalram_pages (); +return min (totalram_pgs -totalhigh_pages (), totalram_pgs /2 ); }#endif
@@ -3275,7 +3275,7 @@ static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo, size =memparse (value,&rest);if (*rest =='%' ) { size <<=PAGE_SHIFT ; - size *= totalram_pages; + size *=totalram_pages ();do_div (size,100 ); rest++; }
mm/slab.c +1 -1
@@ -1248,7 +1248,7 @@ void __init kmem_cache_init(void)* page orders on machines with more than 32MB of memory if * not overridden on the command line. */ -if (!slab_max_order_set && totalram_pages > (32 <<20 ) >>PAGE_SHIFT ) +if (!slab_max_order_set &&totalram_pages () > (32 <<20 ) >>PAGE_SHIFT ) slab_max_order =SLAB_MAX_ORDER_HI ;/* Bootstrap is tricky, because several objects are allocated
mm/swap.c +1 -1
@@ -1023,7 +1023,7 @@ unsigned pagevec_lookup_range_nr_tag(struct pagevec *pvec,*/ void __init swap_setup (void ) { -unsigned long megs = totalram_pages >> (20 -PAGE_SHIFT ); +unsigned long megs =totalram_pages () >> (20 -PAGE_SHIFT );/* Use a smaller cluster for small-memory machines */ if (megs <16 )
mm/util.c +1 -1
@@ -593,7 +593,7 @@ unsigned long vm_commit_limit(void)if (sysctl_overcommit_kbytes) allowed = sysctl_overcommit_kbytes >> (PAGE_SHIFT -10 );else - allowed = ((totalram_pages -hugetlb_total_pages ()) + allowed = ((totalram_pages () -hugetlb_total_pages ()) * sysctl_overcommit_ratio /100 ); allowed += total_swap_pages;
mm/vmalloc.c +2 -2
@@ -1634,7 +1634,7 @@ void *vmap(struct page **pages, unsigned int count,might_sleep (); -if (count > totalram_pages) +if (count >totalram_pages ())return NULL ; size = (unsigned long )count <<PAGE_SHIFT ;
@@ -1739,7 +1739,7 @@ void *__vmalloc_node_range(unsigned long size, unsigned long align,unsigned long real_size = size; size =PAGE_ALIGN (size); -if (!size || (size >>PAGE_SHIFT ) > totalram_pages) +if (!size || (size >>PAGE_SHIFT ) >totalram_pages ()) goto fail; area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNINITIALIZED |
mm/workingset.c +1 -1
@@ -549,7 +549,7 @@ static int __init workingset_init(void)* double the initial memory by using totalram_pages as-is. */ timestamp_bits =BITS_PER_LONG -EVICTION_SHIFT ; - max_order =fls_long (totalram_pages -1 ); + max_order =fls_long (totalram_pages () -1 );if (max_order > timestamp_bits) bucket_order = max_order - timestamp_bits; pr_info("workingset: timestamp_bits=%d max_order=%d bucket_order=%u\n",
mm/zswap.c +2 -2
@@ -219,8 +219,8 @@ struct zswap_tree {static bool zswap_is_full (void ) { -return totalram_pages * zswap_max_pool_percent /100 < -DIV_ROUND_UP (zswap_pool_total_size,PAGE_SIZE ); +return totalram_pages () * zswap_max_pool_percent /100 < +DIV_ROUND_UP (zswap_pool_total_size,PAGE_SIZE ); }static void zswap_update_total_size (void )
net/dccp/proto.c +1 -1
@@ -1131,7 +1131,7 @@ static inline void dccp_mib_exit(void)static int __init dccp_init (void ) {unsigned long goal; -unsigned long totalram_pgs = totalram_pages; +unsigned long totalram_pgs =totalram_pages ();int ehash_order, bhash_order, i;int rc;
net/decnet/dn_route.c +1 -1
@@ -1866,7 +1866,7 @@ void __init dn_route_init(void) dn_route_timer.expires = jiffies + decnet_dst_gc_interval *HZ ;add_timer (&dn_route_timer); - goal = totalram_pages >> (26 -PAGE_SHIFT ); + goal =totalram_pages () >> (26 -PAGE_SHIFT );for (order =0 ; (1UL << order) < goal; order++)/* NOTHING */ ;
net/ipv4/tcp_metrics.c +1 -1
@@ -1000,7 +1000,7 @@ static int __net_init tcp_net_metrics_init(struct net *net) slots = tcpmhash_entries;if (!slots) { -if (totalram_pages >=128 *1024 ) +if (totalram_pages () >=128 *1024 ) slots =16 *1024 ;else slots =8 *1024 ;
net/netfilter/nf_conntrack_core.c +1 -1
@@ -2251,7 +2251,7 @@ static __always_inline unsigned int total_extension_size(void)int nf_conntrack_init_start (void ) { -unsigned long totalram_pgs = totalram_pages; +unsigned long totalram_pgs =totalram_pages ();int max_factor =8 ;int ret = -ENOMEM ;int i;
net/netfilter/xt_hashlimit.c +1 -1
@@ -274,7 +274,7 @@ static int htable_create(struct net *net, struct hashlimit_cfg3 *cfg,struct xt_hashlimit_htable *hinfo;const struct seq_operations *ops;unsigned int size, i; -unsigned long totalram_pgs = totalram_pages; +unsigned long totalram_pgs =totalram_pages ();int ret;if (cfg->size) {
net/sctp/protocol.c +1 -1
@@ -1368,7 +1368,7 @@ static __init int sctp_init(void)int status = -EINVAL ;unsigned long goal;unsigned long limit; -unsigned long totalram_pgs = totalram_pages; +unsigned long totalram_pgs =totalram_pages ();int max_share;int order;int num_entries;
security/integrity/ima/ima_kexec.c +1 -1
@@ -106,7 +106,7 @@ void ima_add_kexec_buffer(struct kimage *image) kexec_segment_size =ALIGN (ima_get_binary_runtime_size () +PAGE_SIZE /2 ,PAGE_SIZE );if ((kexec_segment_size ==ULONG_MAX ) || - ((kexec_segment_size >>PAGE_SHIFT ) > totalram_pages /2 )) { + ((kexec_segment_size >>PAGE_SHIFT ) >totalram_pages () /2 )) {pr_err ("Binary measurement list too large.\n" );return ; }
Signature · 1 line
1.9.1
PATCH v4 4/4 mm: Remove managed_page_count spinlock
In reply to Arun KSNow that totalram_pages and managed_pages are atomic varibles, no need of managed_page_count spinlock. The lock had really a weak consistency guarantee. It hasn't been used for anything but the update but no reader actually cares about all the values being updated to be in sync.
--- include/linux/mmzone.h | 6 ------ mm/page_alloc.c | 5 ----- 2 files changed, 11 deletions(-)
include/linux/mmzone.h +0 -6
@@ -428,12 +428,6 @@ struct zone {* Write access to present_pages at runtime should be protected by * mem_hotplug_begin/end(). Any reader who can't tolerant drift of * present_pages should get_online_mems() to get a stable value. -* -* Read access to managed_pages should be safe because it's unsigned -* long. Write access to zone->managed_pages and totalram_pages are -* protected by managed_page_count_lock at runtime. Idealy only -* adjust_managed_page_count() should be used instead of directly -* touching zone->managed_pages and totalram_pages. */ atomic_long_t managed_pages;unsigned long spanned_pages;
mm/page_alloc.c +0 -5
@@ -122,9 +122,6 @@ };EXPORT_SYMBOL (node_states); -/* Protect totalram_pages and zone->managed_pages */ -static DEFINE_SPINLOCK (managed_page_count_lock); -atomic_long_t _totalram_pages__read_mostly ;EXPORT_SYMBOL (_totalram_pages);unsigned long totalreserve_pages__read_mostly ;
@@ -7065,14 +7062,12 @@ static int __init cmdline_parse_movablecore(char *p)void adjust_managed_page_count (struct page *page,long count) { -spin_lock (&managed_page_count_lock);atomic_long_add (count, &page_zone (page)->managed_pages);totalram_pages_add (count);#ifdef CONFIG_HIGHMEM if (PageHighMem (page))totalhigh_pages_add (count);#endif -spin_unlock (&managed_page_count_lock); }EXPORT_SYMBOL (adjust_managed_page_count);
Signature · 1 line
1.9.1