mm: cma: support sysfs
Subsystems (2)
PATCH 1/1 mm: cma: support sysfs
Since CMA is getting used more widely, it's more important to keep monitoring CMA statistics for system health since it's directly related to user experience.
This patch introduces sysfs for the CMA and exposes stats below to keep monitor for telemetric in the system.
* the number of CMA allocation attempts
* the number of CMA allocation failures
* the number of CMA page allocation attempts
* the number of CMA page allocation failures
--- Documentation/ABI/testing/sysfs-kernel-mm-cma | 39 +++++ include/linux/cma.h | 1 + mm/Makefile | 1 + mm/cma.c | 6 +- mm/cma.h | 20 +++ mm/cma_sysfs.c | 143 ++++++++++++++++++ 6 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 Documentation/ABI/testing/sysfs-kernel-mm-cma create mode 100644 mm/cma_sysfs.c
Documentation/ABI/testing/sysfs-kernel-mm-cma new file +39 -0
@@ -0,0 +1,39 @@ +What: /sys/kernel/mm/cma/ +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + /sys/kernel/mm/cma/ contains a number of subdirectories by + cma-heap name. The subdirectory contains a number of files + to represent cma allocation statistics. + + There are number of files under + /sys/kernel/mm/cma/<cma-heap-name> directory + + - cma_alloc_attempt + - cma_alloc_fail + - alloc_pages_attempt + - alloc_pages_fail + +What: /sys/kernel/mm/cma/<cma-heap-name>/cma_alloc_attempt +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + the number of cma_alloc API attempted + +What: /sys/kernel/mm/cma/<cma-heap-name>/cma_alloc_fail +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + the number of CMA_alloc API failed + +What: /sys/kernel/mm/cma/<cma-heap-name>/alloc_pages_attempt +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + the number of pages CMA API tried to allocate + +What: /sys/kernel/mm/cma/<cma-heap-name>/alloc_pages_fail +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + the number of pages CMA API failed to allocate
include/linux/cma.h +1 -0
@@ -49,4 +49,5 @@ extern struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align,extern bool cma_release (struct cma *cma,const struct page *pages,unsigned int count);extern int cma_for_each_area (int (*it)(struct cma *cma,void *data),void *data); +#endif
mm/Makefile +1 -0
@@ -106,6 +106,7 @@ obj-$(CONFIG_ZSMALLOC) += zsmalloc.o obj-$(CONFIG_Z3FOLD) += z3fold.o obj-$(CONFIG_GENERIC_EARLY_IOREMAP) += early_ioremap.o obj-$(CONFIG_CMA) += cma.o +obj-$(CONFIG_SYSFS) += cma_sysfs.o obj-$(CONFIG_MEMORY_BALLOON) += balloon_compaction.o obj-$(CONFIG_PAGE_EXTENSION) += page_ext.o obj-$(CONFIG_CMA_DEBUGFS) += cma_debug.o
mm/cma.c +5 -1
@@ -448,9 +448,10 @@ struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align, offset =cma_bitmap_aligned_offset (cma, align); bitmap_maxno =cma_bitmap_maxno (cma); bitmap_count =cma_bitmap_pages_to_bits (cma, count); +cma_sysfs_alloc_count (cma, count);if (bitmap_count > bitmap_maxno) -return NULL ; + goto out;for (;;) {mutex_lock (&cma->lock);
@@ -505,6 +506,9 @@ struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align, __func__, count, ret);cma_debug_show_areas (cma); } +out: +if (!page) +cma_sysfs_fail (cma, count);pr_debug ("%s(): returned %p\n" , __func__, page);return page;
mm/cma.h +20 -0
@@ -3,6 +3,16 @@#define __MM_CMA_H__#include <linux/debugfs.h> +#include <linux/kobject.h> + +struct cma_stat { +spinlock_t lock; +unsigned long alloc_attempt;/* the number of CMA allocation attempts */ +unsigned long alloc_fail;/* the number of CMA allocation failures */ +unsigned long pages_attempt;/* the number of CMA page allocation attempts */ +unsigned long pages_fail;/* the number of CMA page allocation failures */ +struct kobject kobj; +};struct cma {unsigned long base_pfn;
@@ -16,6 +26,9 @@ struct cma {struct debugfs_u32_array dfs_bitmap;#endif char name[CMA_MAX_NAME ]; +#ifdef CONFIG_SYSFS +struct cma_stat *stat; +#endif };extern struct cma cma_areas[MAX_CMA_AREAS ];
@@ -26,4 +39,11 @@ static inline unsigned long cma_bitmap_maxno(struct cma *cma) return cma->count >> cma->order_per_bit; } +#ifdef CONFIG_SYSFS +void cma_sysfs_alloc_count (struct cma *cma,size_t count); +void cma_sysfs_fail (struct cma *cma,size_t count); +#else +static void cma_sysfs_alloc_count (struct cma *cma,size_t count) {}; +static void cma_sysfs_fail (struct cma *cma,size_t count) {}; +#endif #endif
mm/cma_sysfs.c new file +143 -0
143 lines not on this page · raw message
Signature · 1 line
2.30.0.365.g02bc693789-goog
Quoted 11 lines · Minchan Kim
Since CMA is getting used more widely, it's more important to keep monitoring CMA statistics for system health since it's directly related to user experience.
This patch introduces sysfs for the CMA and exposes stats below to keep monitor for telemetric in the system.
* the number of CMA allocation attempts
* the number of CMA allocation failures
* the number of CMA page allocation attempts
* the number of CMA page allocation failures
The desire to report CMA data is understandable, but there are a few odd things here:
1) First of all, this has significant overlap with /sys/kernel/debug/cma items. I suspect that all of these items could instead go into /sys/kernel/debug/cma, right?
2) The overall CMA allocation attempts/failures (first two items above) seem an odd pair of things to track. Maybe that is what was easy to track, but I'd vote for just omitting them.
Quoted 19 lines
Signed-off-by Minchan Kim <[email protected]>--- Documentation/ABI/testing/sysfs-kernel-mm-cma | 39 +++++ include/linux/cma.h | 1 + mm/Makefile | 1 + mm/cma.c | 6 +- mm/cma.h | 20 +++ mm/cma_sysfs.c | 143 ++++++++++++++++++ 6 files changed, 209 insertions(+), 1 deletion(-)create mode 100644
Documentation/ABI/testing/sysfs-kernel-mm-cmacreate mode 100644mm/cma_sysfs.cDocumentation/ABI/testing/sysfs-kernel-mm-cma new file +0
-0
Documentation/ABI/testing/sysfs-kernel-mm-cma line 7
Quoted 7 lines
Documentation/ABI/testing/sysfs-kernel-mm-cma new file +7
-0@@ -0,0 +1,39 @@ +What: /sys/kernel/mm/cma/ +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + /sys/kernel/mm/cma/ contains a number of subdirectories by + cma-heap name. The subdirectory contains a number of files + to represent cma allocation statistics.
Somewhere, maybe here, there should be a mention of the closely related /sys/kernel/debug/cma files.
Documentation/ABI/testing/sysfs-kernel-mm-cma line 13
Quoted 6 lines
Documentation/ABI/testing/sysfs-kernel-mm-cma new file +6
-0+ + There are number of files under + /sys/kernel/mm/cma/<cma-heap-name> directory + + - cma_alloc_attempt + - cma_alloc_fail
Are these really useful? They a summary of the alloc_pages items, really.
Documentation/ABI/testing/sysfs-kernel-mm-cma line 15
Quoted 2 lines
Documentation/ABI/testing/sysfs-kernel-mm-cma new file +2
-0+ - alloc_pages_attempt + - alloc_pages_fail
This should also have "cma" in the name, really: cma_alloc_pages_*.
Quoted 29 lines
Documentation/ABI/testing/sysfs-kernel-mm-cma new file +24
-0+ +What: /sys/kernel/mm/cma/<cma-heap-name>/cma_alloc_attempt +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + the number of cma_alloc API attempted + +What: /sys/kernel/mm/cma/<cma-heap-name>/cma_alloc_fail +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + the number of CMA_alloc API failed + +What: /sys/kernel/mm/cma/<cma-heap-name>/alloc_pages_attempt +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + the number of pages CMA API tried to allocate + +What: /sys/kernel/mm/cma/<cma-heap-name>/alloc_pages_fail +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + the number of pages CMA API failed to allocateinclude/linux/cma.h +0
-0
include/linux/cma.h extern struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align, · line 52
Quoted 4 lines
include/linux/cma.h +1
-0@@ -49,4 +49,5 @@ extern struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align,extern bool cma_release (struct cma *cma,const struct page *pages,unsigned int count);extern int cma_for_each_area (int (*it)(struct cma *cma,void *data),void *data); +
A single additional blank line seems to be the only change to this file. :)
thanks,
Signature · 2 lines
John Hubbard NVIDIA
Quoted 13 lines · John Hubbard
Quoted 11 lines · Minchan Kim
Since CMA is getting used more widely, it's more important to keep monitoring CMA statistics for system health since it's directly related to user experience.
This patch introduces sysfs for the CMA and exposes stats below to keep monitor for telemetric in the system.
* the number of CMA allocation attempts
* the number of CMA allocation failures
* the number of CMA page allocation attempts
* the number of CMA page allocation failures
Quoted 5 lines
The desire to report CMA data is understandable, but there are a few odd things here:
1) First of all, this has significant overlap with
/sys/kernel/debug/cmaitems. I suspect that all of these items could instead go into
At this moment, I don't see any overlap with item from cma_debugfs. Could you specify what item you are mentioning?
Quoted 1 line
/sys/kernel/debug/cma, right?
Anyway, thing is I need an stable interface for that and need to use it in Android production build, too(Unfortunately, Android deprecated the debugfs
https://source.android.com/setup/start/android-11-release#debugfs
)
What should be in debugfs and in sysfs? What's the criteria?
Some statistic could be considered about debugging aid or telemetric depening on view point and usecase. And here, I want to use it for telemetric, get an stable interface and use it in production build of Android. In this chance, I'd like to get concrete guideline what should be in sysfs and debugfs so that pointing out this thread whenever folks dump their stat into sysfs to avoid waste of time for others in future. :)
Quoted 4 lines
2) The overall CMA allocation attempts/failures (first two items above) seem an odd pair of things to track. Maybe that is what was easy to track, but I'd vote for just omitting them.
Then, how to know how often CMA API failed?
There are various size allocation request for a CMA so only page allocation stat are not enough to know it.
Quoted 39 lines
Quoted 19 lines
Signed-off-by Minchan Kim <[email protected]>--- Documentation/ABI/testing/sysfs-kernel-mm-cma | 39 +++++ include/linux/cma.h | 1 + mm/Makefile | 1 + mm/cma.c | 6 +- mm/cma.h | 20 +++ mm/cma_sysfs.c | 143 ++++++++++++++++++ 6 files changed, 209 insertions(+), 1 deletion(-)create mode 100644
Documentation/ABI/testing/sysfs-kernel-mm-cmacreate mode 100644mm/cma_sysfs.cDocumentation/ABI/testing/sysfs-kernel-mm-cma new file +0
-0Documentation/ABI/testing/sysfs-kernel-mm-cma line 7
Quoted 7 lines
Documentation/ABI/testing/sysfs-kernel-mm-cma new file +7
-0@@ -0,0 +1,39 @@ +What: /sys/kernel/mm/cma/ +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + /sys/kernel/mm/cma/ contains a number of subdirectories by + cma-heap name. The subdirectory contains a number of files + to represent cma allocation statistics.Somewhere, maybe here, there should be a mention of the closely related
/sys/kernel/debug/cmafiles.Documentation/ABI/testing/sysfs-kernel-mm-cma line 13
Quoted 6 lines
Documentation/ABI/testing/sysfs-kernel-mm-cma new file +6
-0+ + There are number of files under + /sys/kernel/mm/cma/<cma-heap-name> directory + + - cma_alloc_attempt + - cma_alloc_failAre these really useful? They a summary of the
alloc_pagesitems, really.
Quoted 4 lines
Documentation/ABI/testing/sysfs-kernel-mm-cma line 15
Quoted 2 lines
Documentation/ABI/testing/sysfs-kernel-mm-cma new file +2
-0+ - alloc_pages_attempt + - alloc_pages_failThis should also have "cma" in the name, really: cma_alloc_pages_*.
No problem.
Quoted 27 lines
Quoted 27 lines
Documentation/ABI/testing/sysfs-kernel-mm-cma new file +24
-0+ +What: /sys/kernel/mm/cma/<cma-heap-name>/cma_alloc_attempt +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + the number of cma_alloc API attempted + +What: /sys/kernel/mm/cma/<cma-heap-name>/cma_alloc_fail +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + the number of CMA_alloc API failed + +What: /sys/kernel/mm/cma/<cma-heap-name>/alloc_pages_attempt +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + the number of pages CMA API tried to allocate + +What: /sys/kernel/mm/cma/<cma-heap-name>/alloc_pages_fail +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + the number of pages CMA API failed to allocateinclude/linux/cma.h +0
-0
Quoted 8 lines
include/linux/cma.h extern struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align, · line 51
Quoted 6 lines
include/linux/cma.h +1
-0@@ -49,4 +49,5 @@ extern struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align,extern bool cma_release (struct cma *cma,const struct page *pages,unsigned int count);extern int cma_for_each_area (int (*it)(struct cma *cma,void *data),void *data); +A single additional blank line seems to be the only change to this file. :)
Oops.
Quoted 14 lines · Minchan Kim
Quoted 13 lines · John Hubbard
Quoted 11 lines · Minchan Kim
Since CMA is getting used more widely, it's more important to keep monitoring CMA statistics for system health since it's directly related to user experience.
This patch introduces sysfs for the CMA and exposes stats below to keep monitor for telemetric in the system.
* the number of CMA allocation attempts
* the number of CMA allocation failures
* the number of CMA page allocation attempts
* the number of CMA page allocation failures
Quoted 8 lines
Quoted 5 lines
The desire to report CMA data is understandable, but there are a few odd things here:
1) First of all, this has significant overlap with
/sys/kernel/debug/cmaitems. I suspect that all of these items could instead go intoAt this moment, I don't see any overlap with item from
cma_debugfs. Could you specify what item you are mentioning?
Just the fact that there would be two systems under /sys, both of which are doing very very similar things: providing information that is intended to help diagnose CMA.
Quoted 8 lines
Quoted 2 lines
/sys/kernel/debug/cma, right?Anyway, thing is I need an stable interface for that and need to use it in Android production build, too(Unfortunately, Android deprecated the debugfs
https://source.android.com/setup/start/android-11-release#debugfs
)
That's the closest hint to a "why this is needed" that we've seen yet. But it's only a hint.
Quoted 2 lines
What should be in debugfs and in sysfs? What's the criteria?
Well, it's a gray area. "Debugging support" goes into debugfs, and "production-level monitoring and control" goes into sysfs, roughly speaking. And here you have items that could be classified as either.
Quoted 10 lines
Some statistic could be considered about debugging aid or telemetric depening on view point and usecase. And here, I want to use it for telemetric, get an stable interface and use it in production build of Android. In this chance, I'd like to get concrete guideline what should be in sysfs and debugfs so that pointing out this thread whenever folks dump their stat into sysfs to avoid waste of time for others in future. :)
Quoted 1 line
Quoted 5 lines
Quoted 3 lines
2) The overall CMA allocation attempts/failures (first two items above) seem an odd pair of things to track. Maybe that is what was easy to track, but I'd vote for just omitting them.
Then, how to know how often CMA API failed?
Why would you even need to know that, *in addition* to knowing specific page allocation numbers that failed? Again, there is no real-world motivation cited yet, just "this is good data". Need more stories and support here.
thanks,
Signature · 2 lines
John Hubbard NVIDIA
Quoted 87 lines
There are various size allocation request for a CMA so only page allocation stat are not enough to know it.
Quoted 39 lines
Quoted 19 lines
Signed-off-by Minchan Kim <[email protected]>--- Documentation/ABI/testing/sysfs-kernel-mm-cma | 39 +++++ include/linux/cma.h | 1 + mm/Makefile | 1 + mm/cma.c | 6 +- mm/cma.h | 20 +++ mm/cma_sysfs.c | 143 ++++++++++++++++++ 6 files changed, 209 insertions(+), 1 deletion(-)create mode 100644
Documentation/ABI/testing/sysfs-kernel-mm-cmacreate mode 100644mm/cma_sysfs.cDocumentation/ABI/testing/sysfs-kernel-mm-cma new file +0
-0Documentation/ABI/testing/sysfs-kernel-mm-cma line 7
Quoted 7 lines
Documentation/ABI/testing/sysfs-kernel-mm-cma new file +7
-0@@ -0,0 +1,39 @@ +What: /sys/kernel/mm/cma/ +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + /sys/kernel/mm/cma/ contains a number of subdirectories by + cma-heap name. The subdirectory contains a number of files + to represent cma allocation statistics.Somewhere, maybe here, there should be a mention of the closely related
/sys/kernel/debug/cmafiles.Documentation/ABI/testing/sysfs-kernel-mm-cma line 13
Quoted 6 lines
Documentation/ABI/testing/sysfs-kernel-mm-cma new file +6
-0+ + There are number of files under + /sys/kernel/mm/cma/<cma-heap-name> directory + + - cma_alloc_attempt + - cma_alloc_failAre these really useful? They a summary of the
alloc_pagesitems, really.Quoted 4 lines
Documentation/ABI/testing/sysfs-kernel-mm-cma line 15
Quoted 2 lines
Documentation/ABI/testing/sysfs-kernel-mm-cma new file +2
-0+ - alloc_pages_attempt + - alloc_pages_failThis should also have "cma" in the name, really: cma_alloc_pages_*.
No problem.
Quoted 27 lines
Quoted 27 lines
Documentation/ABI/testing/sysfs-kernel-mm-cma new file +24
-0+ +What: /sys/kernel/mm/cma/<cma-heap-name>/cma_alloc_attempt +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + the number of cma_alloc API attempted + +What: /sys/kernel/mm/cma/<cma-heap-name>/cma_alloc_fail +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + the number of CMA_alloc API failed + +What: /sys/kernel/mm/cma/<cma-heap-name>/alloc_pages_attempt +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + the number of pages CMA API tried to allocate + +What: /sys/kernel/mm/cma/<cma-heap-name>/alloc_pages_fail +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + the number of pages CMA API failed to allocateinclude/linux/cma.h +0
-0Quoted 8 lines
include/linux/cma.h extern struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align, · line 51
Quoted 6 lines
include/linux/cma.h +1
-0@@ -49,4 +49,5 @@ extern struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align,extern bool cma_release (struct cma *cma,const struct page *pages,unsigned int count);extern int cma_for_each_area (int (*it)(struct cma *cma,void *data),void *data); +A single additional blank line seems to be the only change to this file. :)
Oops.
Suren Baghdasaryan
reviewerQuoted 62 lines · John Hubbard
Quoted 14 lines · Minchan Kim
Quoted 13 lines · John Hubbard
Quoted 11 lines · Minchan Kim
Since CMA is getting used more widely, it's more important to keep monitoring CMA statistics for system health since it's directly related to user experience.
This patch introduces sysfs for the CMA and exposes stats below to keep monitor for telemetric in the system.
* the number of CMA allocation attempts
* the number of CMA allocation failures
* the number of CMA page allocation attempts
* the number of CMA page allocation failuresQuoted 8 lines
Quoted 5 lines
The desire to report CMA data is understandable, but there are a few odd things here:
1) First of all, this has significant overlap with
/sys/kernel/debug/cmaitems. I suspect that all of these items could instead go intoAt this moment, I don't see any overlap with item from
cma_debugfs. Could you specify what item you are mentioning?Just the fact that there would be two systems under /sys, both of which are doing very very similar things: providing information that is intended to help diagnose CMA.
Quoted 8 lines
Quoted 2 lines
/sys/kernel/debug/cma, right?Anyway, thing is I need an stable interface for that and need to use it in Android production build, too(Unfortunately, Android deprecated the debugfs
https://source.android.com/setup/start/android-11-release#debugfs
)That's the closest hint to a "why this is needed" that we've seen yet. But it's only a hint.
Quoted 2 lines
What should be in debugfs and in sysfs? What's the criteria?
Well, it's a gray area. "Debugging support" goes into debugfs, and "production-level monitoring and control" goes into sysfs, roughly speaking. And here you have items that could be classified as either.
Quoted 14 lines
Some statistic could be considered about debugging aid or telemetric depening on view point and usecase. And here, I want to use it for telemetric, get an stable interface and use it in production build of Android. In this chance, I'd like to get concrete guideline what should be in sysfs and debugfs so that pointing out this thread whenever folks dump their stat into sysfs to avoid waste of time for others in future. :)
Quoted 4 lines
2) The overall CMA allocation attempts/failures (first two items above) seem an odd pair of things to track. Maybe that is what was easy to track, but I'd vote for just omitting them.
Quoted 5 lines
Quoted 1 line
Then, how to know how often CMA API failed?
Why would you even need to know that, *in addition* to knowing specific page allocation numbers that failed? Again, there is no real-world motivation cited yet, just "this is good data". Need more stories and support here.
IMHO it would be very useful to see whether there are multiple small-order allocation failures or a few large-order ones, especially for CMA where large allocations are not unusual. For that I believe both alloc_pages_attempt and alloc_pages_fail would be required.
Quoted 95 lines
thanks,
--
John Hubbard
NVIDIAQuoted 87 lines
There are various size allocation request for a CMA so only page allocation stat are not enough to know it.
Quoted 39 lines
Quoted 19 lines
Signed-off-by Minchan Kim <[email protected]>--- Documentation/ABI/testing/sysfs-kernel-mm-cma | 39 +++++ include/linux/cma.h | 1 + mm/Makefile | 1 + mm/cma.c | 6 +- mm/cma.h | 20 +++ mm/cma_sysfs.c | 143 ++++++++++++++++++ 6 files changed, 209 insertions(+), 1 deletion(-)create mode 100644
Documentation/ABI/testing/sysfs-kernel-mm-cmacreate mode 100644mm/cma_sysfs.cDocumentation/ABI/testing/sysfs-kernel-mm-cma new file +0
-0Documentation/ABI/testing/sysfs-kernel-mm-cma line 7
Quoted 7 lines
Documentation/ABI/testing/sysfs-kernel-mm-cma new file +7
-0@@ -0,0 +1,39 @@ +What: /sys/kernel/mm/cma/ +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + /sys/kernel/mm/cma/ contains a number of subdirectories by + cma-heap name. The subdirectory contains a number of files + to represent cma allocation statistics.Somewhere, maybe here, there should be a mention of the closely related
/sys/kernel/debug/cmafiles.Documentation/ABI/testing/sysfs-kernel-mm-cma line 13
Quoted 6 lines
Documentation/ABI/testing/sysfs-kernel-mm-cma new file +6
-0+ + There are number of files under + /sys/kernel/mm/cma/<cma-heap-name> directory + + - cma_alloc_attempt + - cma_alloc_failAre these really useful? They a summary of the
alloc_pagesitems, really.Quoted 4 lines
Documentation/ABI/testing/sysfs-kernel-mm-cma line 15
Quoted 2 lines
Documentation/ABI/testing/sysfs-kernel-mm-cma new file +2
-0+ - alloc_pages_attempt + - alloc_pages_failThis should also have "cma" in the name, really: cma_alloc_pages_*.
No problem.
Quoted 27 lines
Quoted 27 lines
Documentation/ABI/testing/sysfs-kernel-mm-cma new file +24
-0+ +What: /sys/kernel/mm/cma/<cma-heap-name>/cma_alloc_attempt +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + the number of cma_alloc API attempted + +What: /sys/kernel/mm/cma/<cma-heap-name>/cma_alloc_fail +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + the number of CMA_alloc API failed + +What: /sys/kernel/mm/cma/<cma-heap-name>/alloc_pages_attempt +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + the number of pages CMA API tried to allocate + +What: /sys/kernel/mm/cma/<cma-heap-name>/alloc_pages_fail +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + the number of pages CMA API failed to allocateinclude/linux/cma.h +0
-0Quoted 8 lines
include/linux/cma.h extern struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align, · line 51
Quoted 6 lines
include/linux/cma.h +1
-0@@ -49,4 +49,5 @@ extern struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align,extern bool cma_release (struct cma *cma,const struct page *pages,unsigned int count);extern int cma_for_each_area (int (*it)(struct cma *cma,void *data),void *data); +A single additional blank line seems to be the only change to this file. :)
Oops.
Suren Baghdasaryan
reviewerQuoted 66 lines · Suren Baghdasaryan
Quoted 64 lines · John Hubbard
Quoted 14 lines · Minchan Kim
Quoted 13 lines · John Hubbard
Quoted 11 lines · Minchan Kim
Since CMA is getting used more widely, it's more important to keep monitoring CMA statistics for system health since it's directly related to user experience.
This patch introduces sysfs for the CMA and exposes stats below to keep monitor for telemetric in the system.
* the number of CMA allocation attempts
* the number of CMA allocation failures
* the number of CMA page allocation attempts
* the number of CMA page allocation failuresQuoted 8 lines
Quoted 5 lines
The desire to report CMA data is understandable, but there are a few odd things here:
1) First of all, this has significant overlap with
/sys/kernel/debug/cmaitems. I suspect that all of these items could instead go intoAt this moment, I don't see any overlap with item from
cma_debugfs. Could you specify what item you are mentioning?Just the fact that there would be two systems under /sys, both of which are doing very very similar things: providing information that is intended to help diagnose CMA.
Quoted 8 lines
Quoted 2 lines
/sys/kernel/debug/cma, right?Anyway, thing is I need an stable interface for that and need to use it in Android production build, too(Unfortunately, Android deprecated the debugfs
https://source.android.com/setup/start/android-11-release#debugfs
)That's the closest hint to a "why this is needed" that we've seen yet. But it's only a hint.
Quoted 2 lines
What should be in debugfs and in sysfs? What's the criteria?
Well, it's a gray area. "Debugging support" goes into debugfs, and "production-level monitoring and control" goes into sysfs, roughly speaking. And here you have items that could be classified as either.
Quoted 15 lines
Some statistic could be considered about debugging aid or telemetric depening on view point and usecase. And here, I want to use it for telemetric, get an stable interface and use it in production build of Android. In this chance, I'd like to get concrete guideline what should be in sysfs and debugfs so that pointing out this thread whenever folks dump their stat into sysfs to avoid waste of time for others in future. :)
Quoted 4 lines
2) The overall CMA allocation attempts/failures (first two items above) seem an odd pair of things to track. Maybe that is what was easy to track, but I'd vote for just omitting them.
Then, how to know how often CMA API failed?
Quoted 8 lines
Quoted 3 lines
Why would you even need to know that, *in addition* to knowing specific page allocation numbers that failed? Again, there is no real-world motivation cited yet, just "this is good data". Need more stories and support here.
IMHO it would be very useful to see whether there are multiple small-order allocation failures or a few large-order ones, especially for CMA where large allocations are not unusual. For that I believe both
alloc_pages_attemptandalloc_pages_failwould be required.
Sorry, I meant to say "both cma_alloc_fail and alloc_pages_fail would be required".
Quoted 96 lines
Quoted 96 lines
thanks,
--
John Hubbard
NVIDIAQuoted 87 lines
There are various size allocation request for a CMA so only page allocation stat are not enough to know it.
Quoted 39 lines
Quoted 19 lines
Signed-off-by Minchan Kim <[email protected]>--- Documentation/ABI/testing/sysfs-kernel-mm-cma | 39 +++++ include/linux/cma.h | 1 + mm/Makefile | 1 + mm/cma.c | 6 +- mm/cma.h | 20 +++ mm/cma_sysfs.c | 143 ++++++++++++++++++ 6 files changed, 209 insertions(+), 1 deletion(-)create mode 100644
Documentation/ABI/testing/sysfs-kernel-mm-cmacreate mode 100644mm/cma_sysfs.cDocumentation/ABI/testing/sysfs-kernel-mm-cma new file +0
-0Documentation/ABI/testing/sysfs-kernel-mm-cma line 7
Quoted 7 lines
Documentation/ABI/testing/sysfs-kernel-mm-cma new file +7
-0@@ -0,0 +1,39 @@ +What: /sys/kernel/mm/cma/ +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + /sys/kernel/mm/cma/ contains a number of subdirectories by + cma-heap name. The subdirectory contains a number of files + to represent cma allocation statistics.Somewhere, maybe here, there should be a mention of the closely related
/sys/kernel/debug/cmafiles.Documentation/ABI/testing/sysfs-kernel-mm-cma line 13
Quoted 6 lines
Documentation/ABI/testing/sysfs-kernel-mm-cma new file +6
-0+ + There are number of files under + /sys/kernel/mm/cma/<cma-heap-name> directory + + - cma_alloc_attempt + - cma_alloc_failAre these really useful? They a summary of the
alloc_pagesitems, really.Quoted 4 lines
Documentation/ABI/testing/sysfs-kernel-mm-cma line 15
Quoted 2 lines
Documentation/ABI/testing/sysfs-kernel-mm-cma new file +2
-0+ - alloc_pages_attempt + - alloc_pages_failThis should also have "cma" in the name, really: cma_alloc_pages_*.
No problem.
Quoted 27 lines
Quoted 27 lines
Documentation/ABI/testing/sysfs-kernel-mm-cma new file +24
-0+ +What: /sys/kernel/mm/cma/<cma-heap-name>/cma_alloc_attempt +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + the number of cma_alloc API attempted + +What: /sys/kernel/mm/cma/<cma-heap-name>/cma_alloc_fail +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + the number of CMA_alloc API failed + +What: /sys/kernel/mm/cma/<cma-heap-name>/alloc_pages_attempt +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + the number of pages CMA API tried to allocate + +What: /sys/kernel/mm/cma/<cma-heap-name>/alloc_pages_fail +Date: Feb 2021 +Contact: Minchan Kim <[email protected]> +Description: + the number of pages CMA API failed to allocateinclude/linux/cma.h +0
-0Quoted 8 lines
include/linux/cma.h extern struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align, · line 51
Quoted 6 lines
include/linux/cma.h +1
-0@@ -49,4 +49,5 @@ extern struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align,extern bool cma_release (struct cma *cma,const struct page *pages,unsigned int count);extern int cma_for_each_area (int (*it)(struct cma *cma,void *data),void *data); +A single additional blank line seems to be the only change to this file. :)
Oops.
On 2/4/21 3:45 PM, Suren Baghdasaryan wrote:
...
Quoted 10 lines
Quoted 10 lines
Quoted 9 lines
Quoted 5 lines
Quoted 3 lines
2) The overall CMA allocation attempts/failures (first two items above) seem an odd pair of things to track. Maybe that is what was easy to track, but I'd vote for just omitting them.
Then, how to know how often CMA API failed?
Why would you even need to know that, *in addition* to knowing specific page allocation numbers that failed? Again, there is no real-world motivation cited yet, just "this is good data". Need more stories and support here.
Quoted 7 lines
Quoted 4 lines
IMHO it would be very useful to see whether there are multiple small-order allocation failures or a few large-order ones, especially for CMA where large allocations are not unusual. For that I believe both
alloc_pages_attemptandalloc_pages_failwould be required.Sorry, I meant to say "both
cma_alloc_failandalloc_pages_failwould be required".
So if you want to know that, the existing items are still a little too indirect to really get it right. You can only know the average allocation size, by dividing. Instead, we should provide the allocation size, for each count.
The limited interface makes this a little awkward, but using zones/ranges could
work: "for this range of allocation sizes, there were the following stats". Or, some other technique that I haven't thought of (maybe two items per file?) would be better.
On the other hand, there's an argument for keeping this minimal and simple. That would probably lead us to putting in a couple of items into /proc/vmstat, as I just mentioned in my other response, and calling it good.
thanks,
Signature · 2 lines
John Hubbard NVIDIA
Quoted 24 lines · John Hubbard
On 2/4/21 3:45 PM, Suren Baghdasaryan wrote:
...Quoted 10 lines
Quoted 10 lines
Quoted 9 lines
Quoted 5 lines
Quoted 3 lines
2) The overall CMA allocation attempts/failures (first two items above) seem an odd pair of things to track. Maybe that is what was easy to track, but I'd vote for just omitting them.
Then, how to know how often CMA API failed?
Why would you even need to know that, *in addition* to knowing specific page allocation numbers that failed? Again, there is no real-world motivation cited yet, just "this is good data". Need more stories and support here.
Quoted 7 lines
Quoted 4 lines
IMHO it would be very useful to see whether there are multiple small-order allocation failures or a few large-order ones, especially for CMA where large allocations are not unusual. For that I believe both
alloc_pages_attemptandalloc_pages_failwould be required.Sorry, I meant to say "both
cma_alloc_failandalloc_pages_failwould be required".So if you want to know that, the existing items are still a little too indirect to really get it right. You can only know the average allocation size, by dividing. Instead, we should provide the allocation size, for each count.
Quoted 8 lines
The limited interface makes this a little awkward, but using zones/ranges could
work: "for this range of allocation sizes, there were the following stats". Or, some other technique that I haven't thought of (maybe two items per file?) would be better.On the other hand, there's an argument for keeping this minimal and simple. That would probably lead us to putting in a couple of items into
/proc/vmstat, as I just mentioned in my other response, and calling it good.
...and remember: if we keep it nice and minimal and clean, we can put it into /proc/vmstat and monitor it.
And then if a problem shows up, the more complex and advanced debugging data can go into debugfs's CMA area. And you're all set.
If Android made up some policy not to use debugfs, then:
a) that probably won't prevent engineers from using it anyway, for advanced debugging, and
b) If (a) somehow falls short, then we need to talk about what Android's plans are to fill the need. And "fill up sysfs with debugfs items, possibly duplicating some of them, and generally making an unecessary mess, to compensate for not using debugfs" is not my first choice. :)
thanks,
Signature · 2 lines
John Hubbard NVIDIA
Suren Baghdasaryan
reviewerQuoted 26 lines · John Hubbard
Quoted 24 lines · John Hubbard
On 2/4/21 3:45 PM, Suren Baghdasaryan wrote:
...Quoted 10 lines
Quoted 10 lines
Quoted 9 lines
Quoted 5 lines
Quoted 3 lines
2) The overall CMA allocation attempts/failures (first two items above) seem an odd pair of things to track. Maybe that is what was easy to track, but I'd vote for just omitting them.
Then, how to know how often CMA API failed?
Why would you even need to know that, *in addition* to knowing specific page allocation numbers that failed? Again, there is no real-world motivation cited yet, just "this is good data". Need more stories and support here.
Quoted 7 lines
Quoted 4 lines
IMHO it would be very useful to see whether there are multiple small-order allocation failures or a few large-order ones, especially for CMA where large allocations are not unusual. For that I believe both
alloc_pages_attemptandalloc_pages_failwould be required.Sorry, I meant to say "both
cma_alloc_failandalloc_pages_failwould be required".So if you want to know that, the existing items are still a little too indirect to really get it right. You can only know the average allocation size, by dividing. Instead, we should provide the allocation size, for each count.
Quoted 8 lines
Quoted 8 lines
The limited interface makes this a little awkward, but using zones/ranges could
work: "for this range of allocation sizes, there were the following stats". Or, some other technique that I haven't thought of (maybe two items per file?) would be better.On the other hand, there's an argument for keeping this minimal and simple. That would probably lead us to putting in a couple of items into
/proc/vmstat, as I just mentioned in my other response, and calling it good.
True. I was thinking along these lines but per-order counters felt like maybe an overkill? I'm all for keeping it simple.
Quoted 4 lines
...and remember: if we keep it nice and minimal and clean, we can put it into
/proc/vmstatand monitor it.
No objections from me.
Quoted 19 lines
And then if a problem shows up, the more complex and advanced debugging data can go into debugfs's CMA area. And you're all set.
If Android made up some policy not to use debugfs, then:
a) that probably won't prevent engineers from using it anyway, for advanced debugging, and
b) If (a) somehow falls short, then we need to talk about what Android's plans are to fill the need. And "fill up sysfs with debugfs items, possibly duplicating some of them, and generally making an unecessary mess, to compensate for not using debugfs" is not my first choice. :)
thanks,
--
John Hubbard
NVIDIA
Quoted 37 lines · John Hubbard
Quoted 14 lines · Minchan Kim
Quoted 13 lines · John Hubbard
Quoted 11 lines · Minchan Kim
Since CMA is getting used more widely, it's more important to keep monitoring CMA statistics for system health since it's directly related to user experience.
This patch introduces sysfs for the CMA and exposes stats below to keep monitor for telemetric in the system.
* the number of CMA allocation attempts
* the number of CMA allocation failures
* the number of CMA page allocation attempts
* the number of CMA page allocation failuresQuoted 8 lines
Quoted 5 lines
The desire to report CMA data is understandable, but there are a few odd things here:
1) First of all, this has significant overlap with
/sys/kernel/debug/cmaitems. I suspect that all of these items could instead go intoAt this moment, I don't see any overlap with item from
cma_debugfs. Could you specify what item you are mentioning?Just the fact that there would be two systems under /sys, both of which are doing very very similar things: providing information that is intended to help diagnose CMA.
Quoted 8 lines
Quoted 2 lines
/sys/kernel/debug/cma, right?Anyway, thing is I need an stable interface for that and need to use it in Android production build, too(Unfortunately, Android deprecated the debugfs
https://source.android.com/setup/start/android-11-release#debugfs
)
Quoted 8 lines
That's the closest hint to a "why this is needed" that we've seen yet. But it's only a hint.
Quoted 2 lines
What should be in debugfs and in sysfs? What's the criteria?
Well, it's a gray area. "Debugging support" goes into debugfs, and "production-level monitoring and control" goes into sysfs, roughly
True.
Quoted 16 lines
speaking. And here you have items that could be classified as either.
Quoted 14 lines
Some statistic could be considered about debugging aid or telemetric depening on view point and usecase. And here, I want to use it for telemetric, get an stable interface and use it in production build of Android. In this chance, I'd like to get concrete guideline what should be in sysfs and debugfs so that pointing out this thread whenever folks dump their stat into sysfs to avoid waste of time for others in future. :)
Quoted 4 lines
2) The overall CMA allocation attempts/failures (first two items above) seem an odd pair of things to track. Maybe that is what was easy to track, but I'd vote for just omitting them.
Quoted 5 lines
Quoted 1 line
Then, how to know how often CMA API failed?
Why would you even need to know that, *in addition* to knowing specific page allocation numbers that failed? Again, there is no real-world motivation cited yet, just "this is good data". Need more stories and support here.
Let me give an example.
Let' assume we use memory buffer allocation via CMA for bluetooth enable of device.
If user clicks the bluetooth button in the phone but fail to allocate the memory from CMA, user will still see bluetooth button gray. User would think his touch was not enough powerful so he try clicking again and fortunately CMA allocation was successful this time and they will see bluetooh button enabled and could listen the music.
Here, product team needs to monitor how often CMA alloc failed so if the failure ratio is steadily increased than the bar, it means engineers need to go investigation.
Make sense?
On 2/4/21 4:12 PM, Minchan Kim wrote:
...
Quoted 16 lines
Quoted 5 lines
Quoted 1 line
Then, how to know how often CMA API failed?
Why would you even need to know that, *in addition* to knowing specific page allocation numbers that failed? Again, there is no real-world motivation cited yet, just "this is good data". Need more stories and support here.
Let me give an example.
Let' assume we use memory buffer allocation via CMA for bluetooth enable of device.
If user clicks the bluetooth button in the phone but fail to allocate the memory from CMA, user will still see bluetooth button gray. User would think his touch was not enough powerful so he try clicking again and fortunately CMA allocation was successful this time and they will see bluetooh button enabled and could listen the music.
Quoted 6 lines
Here, product team needs to monitor how often CMA alloc failed so if the failure ratio is steadily increased than the bar, it means engineers need to go investigation.
Make sense?
Yes, except that it raises more questions:
1) Isn't this just standard allocation failure? Don't you already have a way to track that?
Presumably, having the source code, you can easily deduce that a bluetooth allocation failure goes directly to a CMA allocation failure, right?
Anyway, even though the above is still a little murky, I expect you're right that it's good to have *some* indication, somewhere about CMA behavior...
Thinking about this some more, I wonder if this is really /proc/vmstat sort of data that we're talking about. It seems to fit right in there, yes?
thanks,
Signature · 2 lines
John Hubbard NVIDIA
Quoted 30 lines · John Hubbard
On 2/4/21 4:12 PM, Minchan Kim wrote:
...Quoted 16 lines
Quoted 5 lines
Quoted 1 line
Then, how to know how often CMA API failed?
Why would you even need to know that, *in addition* to knowing specific page allocation numbers that failed? Again, there is no real-world motivation cited yet, just "this is good data". Need more stories and support here.
Let me give an example.
Let' assume we use memory buffer allocation via CMA for bluetooth enable of device.
If user clicks the bluetooth button in the phone but fail to allocate the memory from CMA, user will still see bluetooth button gray. User would think his touch was not enough powerful so he try clicking again and fortunately CMA allocation was successful this time and they will see bluetooh button enabled and could listen the music.Quoted 6 lines
Here, product team needs to monitor how often CMA alloc failed so if the failure ratio is steadily increased than the bar, it means engineers need to go investigation.
Make sense?
Yes, except that it raises more questions:
1) Isn't this just standard allocation failure? Don't you already have a way to track that?
Quoted 8 lines
Presumably, having the source code, you can easily deduce that a bluetooth allocation failure goes directly to a CMA allocation failure, right?
Anyway, even though the above is still a little murky, I expect you're right that it's good to have *some* indication, somewhere about CMA behavior...
Thinking about this some more, I wonder if this is really
/proc/vmstatsort of data that we're talking about. It seems to fit right in there, yes?
Thing is CMA instance are multiple, cma-A, cma-B, cma-C and each of CMA heap has own specific scenario. /proc/vmstat could be bloated a lot while CMA instance will be increased.
Suren Baghdasaryan
reviewerQuoted 38 lines · Minchan Kim
Quoted 36 lines · John Hubbard
On 2/4/21 4:12 PM, Minchan Kim wrote:
...Quoted 16 lines
Quoted 5 lines
Quoted 1 line
Then, how to know how often CMA API failed?
Why would you even need to know that, *in addition* to knowing specific page allocation numbers that failed? Again, there is no real-world motivation cited yet, just "this is good data". Need more stories and support here.
Let me give an example.
Let' assume we use memory buffer allocation via CMA for bluetooth enable of device.
If user clicks the bluetooth button in the phone but fail to allocate the memory from CMA, user will still see bluetooth button gray. User would think his touch was not enough powerful so he try clicking again and fortunately CMA allocation was successful this time and they will see bluetooh button enabled and could listen the music.Quoted 6 lines
Here, product team needs to monitor how often CMA alloc failed so if the failure ratio is steadily increased than the bar, it means engineers need to go investigation.
Make sense?
Yes, except that it raises more questions:
1) Isn't this just standard allocation failure? Don't you already have a way to track that?
Presumably, having the source code, you can easily deduce that a bluetooth allocation failure goes directly to a CMA allocation failure, right?
Anyway, even though the above is still a little murky, I expect you're right that it's good to have *some* indication, somewhere about CMA behavior...
Quoted 6 lines
Quoted 2 lines
Thinking about this some more, I wonder if this is really
/proc/vmstatsort of data that we're talking about. It seems to fit right in there, yes?Thing is CMA instance are multiple, cma-A, cma-B, cma-C and each of CMA heap has own specific scenario.
/proc/vmstatcould be bloated a lot while CMA instance will be increased.
Oh, I missed the fact that you need these stats per-CMA.
Quoted 26 lines · Minchan Kim
Quoted 25 lines · John Hubbard
On 2/4/21 4:12 PM, Minchan Kim wrote:
...Quoted 22 lines
Quoted 5 lines
Quoted 1 line
Then, how to know how often CMA API failed?
Why would you even need to know that, *in addition* to knowing specific page allocation numbers that failed? Again, there is no real-world motivation cited yet, just "this is good data". Need more stories and support here.
Let me give an example.
Let' assume we use memory buffer allocation via CMA for bluetooth enable of device.
If user clicks the bluetooth button in the phone but fail to allocate the memory from CMA, user will still see bluetooth button gray. User would think his touch was not enough powerful so he try clicking again and fortunately CMA allocation was successful this time and they will see bluetooh button enabled and could listen the music.Here, product team needs to monitor how often CMA alloc failed so if the failure ratio is steadily increased than the bar, it means engineers need to go investigation.
Make sense?
Quoted 7 lines
Quoted 7 lines
Yes, except that it raises more questions:
1) Isn't this just standard allocation failure? Don't you already have a way to track that?
Presumably, having the source code, you can easily deduce that a bluetooth allocation failure goes directly to a CMA allocation failure, right?
Still wondering about this...
Quoted 11 lines
Quoted 6 lines
Anyway, even though the above is still a little murky, I expect you're right that it's good to have *some* indication, somewhere about CMA behavior...
Thinking about this some more, I wonder if this is really
/proc/vmstatsort of data that we're talking about. It seems to fit right in there, yes?Thing is CMA instance are multiple, cma-A, cma-B, cma-C and each of CMA heap has own specific scenario.
/proc/vmstatcould be bloated a lot while CMA instance will be increased.
Yes, that would not fit in /proc/vmstat...assuming that you really require knowing--at this point--which CMA heap is involved. And that's worth poking at. If you get an overall indication in vmstat that CMA is having trouble, then maybe that's all you need to start digging further.
It's actually easier to monitor one or two simpler items than it is to monitor a larger number of complicated items. And I get the impression that this is sort of a top-level, production software indicator.
thanks,
Signature · 2 lines
John Hubbard NVIDIA
Quoted 29 lines · John Hubbard
Quoted 28 lines · Minchan Kim
Quoted 27 lines · John Hubbard
On 2/4/21 4:12 PM, Minchan Kim wrote:
...Quoted 16 lines
Quoted 5 lines
Quoted 1 line
Then, how to know how often CMA API failed?
Why would you even need to know that, *in addition* to knowing specific page allocation numbers that failed? Again, there is no real-world motivation cited yet, just "this is good data". Need more stories and support here.
Let me give an example.
Let' assume we use memory buffer allocation via CMA for bluetooth enable of device.
If user clicks the bluetooth button in the phone but fail to allocate the memory from CMA, user will still see bluetooth button gray. User would think his touch was not enough powerful so he try clicking again and fortunately CMA allocation was successful this time and they will see bluetooh button enabled and could listen the music.Quoted 6 lines
Here, product team needs to monitor how often CMA alloc failed so if the failure ratio is steadily increased than the bar, it means engineers need to go investigation.
Make sense?
Yes, except that it raises more questions:
Quoted 7 lines
Quoted 5 lines
Quoted 5 lines
1) Isn't this just standard allocation failure? Don't you already have a way to track that?
Presumably, having the source code, you can easily deduce that a bluetooth allocation failure goes directly to a CMA allocation failure, right?
Still wondering about this...
It would work if we have full source code and stack are not complicated for every usecases. Having said, having a good central place automatically popped up is also beneficial for not to add similar statistics for each call sites.
Why do we have too many item in slab sysfs instead of creating each call site inventing on each own?
Quoted 13 lines
Quoted 5 lines
Quoted 5 lines
Anyway, even though the above is still a little murky, I expect you're right that it's good to have *some* indication, somewhere about CMA behavior...
Quoted 7 lines
Quoted 2 lines
Thinking about this some more, I wonder if this is really
/proc/vmstatsort of data that we're talking about. It seems to fit right in there, yes?Thing is CMA instance are multiple, cma-A, cma-B, cma-C and each of CMA heap has own specific scenario.
/proc/vmstatcould be bloated a lot while CMA instance will be increased.
Quoted 4 lines
Yes, that would not fit in
/proc/vmstat...assumingthat you really require knowing--at this point--which CMA heap is involved. And that's worth poking at. If you get an overall indication in vmstat that CMA is having trouble, then maybe that's all you need to start digging further.
I agree it could save to decide whether I should go digging further but anyway, I need to go though each of instance once it happens. In that, what I need is per-CMA statistics, not global. I am happy to implement it but I'd like to say it's not my case.
Quoted 4 lines
It's actually easier to monitor one or two simpler items than it is to monitor a larger number of complicated items. And I get the impression that this is sort of a top-level, production software indicator.
Let me clarify one more time.
What I'd like to get ultimately is per-CMA statistics instead of global vmstat for the usecase at this moment. Global vmstat could help the decision whether I should go deeper but it ends up needing per-CMA statistics. And I'd like to keep them in sysfs, not debugfs since it should be stable as a telemetric.
What points do you disagree in this view?
On 2/4/21 9:17 PM, Minchan Kim wrote:
...
Quoted 5 lines
Quoted 4 lines
Quoted 2 lines
Quoted 2 lines
Presumably, having the source code, you can easily deduce that a bluetooth allocation failure goes directly to a CMA allocation failure, right?
Still wondering about this...
Quoted 8 lines
It would work if we have full source code and stack are not complicated for every usecases. Having said, having a good central place automatically popped up is also beneficial for not to add similar statistics for each call sites.
Why do we have too many item in slab sysfs instead of creating each call site inventing on each own?
I'm not sure I understand that question fully, but I don't think we need to invent anything unique here. So far we've discussed debugfs, sysfs, and /proc, none of which are new mechanisms.
...
Quoted 6 lines
Quoted 3 lines
It's actually easier to monitor one or two simpler items than it is to monitor a larger number of complicated items. And I get the impression that this is sort of a top-level, production software indicator.
Let me clarify one more time.
Quoted 7 lines
What I'd like to get ultimately is per-CMA statistics instead of global vmstat for the usecase at this moment. Global vmstat could help the decision whether I should go deeper but it ends up needing per-CMA statistics. And I'd like to keep them in sysfs, not debugfs since it should be stable as a telemetric.
What points do you disagree in this view?
No huge disagreements, I just want to get us down to the true essential elements of what is required--and find a good home for the data. Initial debugging always has excesses, and those should not end up in the more carefully vetted production code.
If I were doing this, I'd probably consider HugeTLB pages as an example to follow, because they have a lot in common with CMA: it's another memory allocation pool, and people also want to monitor it.
HugeTLB pages and THP pages are monitored in /proc:
/proc/meminfo and /proc/vmstat:
# cat meminfo |grep -i huge AnonHugePages: 88064 kB ShmemHugePages: 0 kB FileHugePages: 0 kB HugePages_Total: 500 HugePages_Free: 500 HugePages_Rsvd: 0 HugePages_Surp: 0 Hugepagesize: 2048 kB Hugetlb: 1024000 kB
# cat vmstat | grep -i huge nr_shmem_hugepages 0 nr_file_hugepages 0 nr_anon_transparent_hugepages 43 numa_huge_pte_updates 0
...aha, so is CMA:
# cat vmstat | grep -i cma nr_free_cma 261718
# cat meminfo | grep -i cma CmaTotal: 1048576 kB CmaFree: 1046872 kB
OK, given that CMA is already in those two locations, maybe we should put this information in one or both of those, yes?
thanks,
Signature · 2 lines
John Hubbard NVIDIA
Quoted 12 lines · John Hubbard
On 2/4/21 9:17 PM, Minchan Kim wrote:
...Quoted 10 lines
Quoted 4 lines
Quoted 2 lines
Quoted 2 lines
Presumably, having the source code, you can easily deduce that a bluetooth allocation failure goes directly to a CMA allocation failure, right?
Still wondering about this...
It would work if we have full source code and stack are not complicated for every usecases. Having said, having a good central place automatically popped up is also beneficial for not to add similar statistics for each call sites.
Quoted 7 lines
Quoted 3 lines
Why do we have too many item in slab sysfs instead of creating each call site inventing on each own?
I'm not sure I understand that question fully, but I don't think we need to invent anything unique here. So far we've discussed debugfs, sysfs, and /proc, none of which are new mechanisms.
I thought you asked why we couldn't add those stat in their call site driver syfs instead of central place. Please clarify if I misunderstood your question.
Quoted 52 lines
...
Quoted 6 lines
Quoted 3 lines
It's actually easier to monitor one or two simpler items than it is to monitor a larger number of complicated items. And I get the impression that this is sort of a top-level, production software indicator.
Let me clarify one more time.
Quoted 7 lines
What I'd like to get ultimately is per-CMA statistics instead of global vmstat for the usecase at this moment. Global vmstat could help the decision whether I should go deeper but it ends up needing per-CMA statistics. And I'd like to keep them in sysfs, not debugfs since it should be stable as a telemetric.
What points do you disagree in this view?
No huge disagreements, I just want to get us down to the true essential elements of what is required--and find a good home for the data. Initial debugging always has excesses, and those should not end up in the more carefully vetted production code.
If I were doing this, I'd probably consider HugeTLB pages as an example to follow, because they have a lot in common with CMA: it's another memory allocation pool, and people also want to monitor it.
HugeTLB pages and THP pages are monitored in /proc:
/proc/meminfoand/proc/vmstat:# cat meminfo |grep -i huge AnonHugePages: 88064 kB ShmemHugePages: 0 kB FileHugePages: 0 kB HugePages_Total: 500 HugePages_Free: 500 HugePages_Rsvd: 0 HugePages_Surp: 0 Hugepagesize: 2048 kB Hugetlb: 1024000 kB# cat vmstat | grep -i huge nr_shmem_hugepages 0 nr_file_hugepages 0 nr_anon_transparent_hugepages 43 numa_huge_pte_updates 0...aha, so is CMA:
# cat vmstat | grep -i cma nr_free_cma 261718
Quoted 6 lines
# cat meminfo | grep -i cma CmaTotal: 1048576 kB CmaFree: 1046872 kBOK, given that CMA is already in those two locations, maybe we should put this information in one or both of those, yes?
Do you suggest something liks this, for example?
cat vmstat | grep -i cmacma_a_success 125cma_a_fail 25cma_b_success 130cma_b_fail 156
..cma_f_fail xxx
Quoted 2 lines · Minchan Kim
Quoted 1 line · John Hubbard
On 2/4/21 9:17 PM, Minchan Kim wrote:
...
Quoted 13 lines
Quoted 9 lines
# cat vmstat | grep -i cma nr_free_cma 261718# cat meminfo | grep -i cma CmaTotal: 1048576 kB CmaFree: 1046872 kBOK, given that CMA is already in those two locations, maybe we should put this information in one or both of those, yes?
Do you suggest something liks this, for example?
Quoted 8 lines
cat vmstat | grep -i cma
cma_a_success125cma_a_fail25cma_b_success130cma_b_fail156
..cma_f_failxxx
Yes, approximately. I was wondering if this would suffice at least as a baseline:
cma_alloc_success 125cma_alloc_failure 25
...and then, to see if more is needed, some questions:
a) Do you know of an upper bound on how many cma areas there can be (I think Matthew also asked that)?
b) Is tracking the cma area really as valuable as other possibilities? We can put "a few" to "several" items here, so really want to get your very favorite bits of information in. If, for example, there can be *lots* of cma areas, then maybe tracking by a range of allocation sizes is better...
thanks,
Signature · 2 lines
John Hubbard NVIDIA
Quoted 26 lines · John Hubbard
Quoted 2 lines · Minchan Kim
Quoted 1 line · John Hubbard
On 2/4/21 9:17 PM, Minchan Kim wrote:
...
Quoted 13 lines
Quoted 9 lines
# cat vmstat | grep -i cma nr_free_cma 261718# cat meminfo | grep -i cma CmaTotal: 1048576 kB CmaFree: 1046872 kBOK, given that CMA is already in those two locations, maybe we should put this information in one or both of those, yes?
Do you suggest something liks this, for example?
Quoted 8 lines
cat vmstat | grep -i cma
cma_a_success125cma_a_fail25cma_b_success130cma_b_fail156
..cma_f_failxxx
Quoted 4 lines
Yes, approximately. I was wondering if this would suffice at least as a baseline:
cma_alloc_success125cma_alloc_failure25
IMO, regardless of the my patch, it would be good to have such statistics in that CMA was born to replace carved out memory with dynamic allocation ideally for memory efficiency ideally so failure should regard critical so admin could notice it how the system is hurt.
Anyway, it's not enough for me and orthgonal with my goal.
Quoted 5 lines
...and then, to see if more is needed, some questions:
a) Do you know of an upper bound on how many cma areas there can be (I think Matthew also asked that)?
There is no upper bound since it's configurable.
Quoted 4 lines
b) Is tracking the cma area really as valuable as other possibilities? We can put "a few" to "several" items here, so really want to get your very favorite bits of information in. If, for example, there can be *lots* of cma areas, then maybe tracking
At this moment, allocation/failure for each CMA area since they have particular own usecase, which makes me easy to keep which module will be affected. I think it is very useful per-CMA statistics as minimum code change so I want to enable it by default under CONFIG_CMA && CONFIG_SYSFS.
Quoted 1 line
by a range of allocation sizes is better...
I takes your suggestion something like this.
[alloc_range] could be order or range by interval
/sys/kernel/mm/cma/cma-A/[alloc_range]/success
/sys/kernel/mm/cma/cma-A/[alloc_range]/fail
..
..
/sys/kernel/mm/cma/cma-Z/[alloc_range]/success
/sys/kernel/mm/cma/cma-Z/[alloc_range]/fail
I agree it would be also useful but I'd like to enable it under CONFIG_CMA_SYSFS_ALLOC_RANGE as separate patchset.
On 2/5/21 8:15 AM, Minchan Kim wrote:
...
Quoted 9 lines
Quoted 4 lines
Yes, approximately. I was wondering if this would suffice at least as a baseline:
cma_alloc_success125cma_alloc_failure25IMO, regardless of the my patch, it would be good to have such statistics in that CMA was born to replace carved out memory with dynamic allocation ideally for memory efficiency ideally so failure should regard critical so admin could notice it how the system is hurt.
Right. So CMA failures are useful for the admin to see, understood.
Quoted 3 lines
Anyway, it's not enough for me and orthgonal with my goal.
OK. But...what *is* your goal, and why is this useless (that's what orthogonal really means here) for your goal?
Also, would you be willing to try out something simple first, such as providing indication that cma is active and it's overall success rate, like this:
/proc/vmstat:
cma_alloc_success 125cma_alloc_failure 25
...or is the only way to provide the more detailed items, complete with per-CMA details, in a non-debugfs location?
Quoted 8 lines
Quoted 5 lines
...and then, to see if more is needed, some questions:
a) Do you know of an upper bound on how many cma areas there can be (I think Matthew also asked that)?
There is no upper bound since it's configurable.
OK, thanks,so that pretty much rules out putting per-cma details into anything other than a directory or something like it.
Quoted 14 lines
Quoted 4 lines
b) Is tracking the cma area really as valuable as other possibilities? We can put "a few" to "several" items here, so really want to get your very favorite bits of information in. If, for example, there can be *lots* of cma areas, then maybe tracking
At this moment, allocation/failure for each CMA area since they have particular own usecase, which makes me easy to keep which module will be affected. I think it is very useful per-CMA statistics as minimum code change so I want to enable it by default under
CONFIG_CMA&&CONFIG_SYSFS.Quoted 1 line
by a range of allocation sizes is better...
I takes your suggestion something like this.
Quoted 8 lines
[
alloc_range] could be order or range by interval/sys/kernel/mm/cma/cma-A/[alloc_range]/success
/sys/kernel/mm/cma/cma-A/[alloc_range]/fail
..
..
/sys/kernel/mm/cma/cma-Z/[alloc_range]/success
/sys/kernel/mm/cma/cma-Z/[alloc_range]/fail
Actually, I meant, "ranges instead of cma areas", like this:
/<path-to-cma-data/[alloc_range_1]/success
/<path-to-cma-data/[alloc_range_1]/fail
/<path-to-cma-data/[alloc_range_2]/success
/<path-to-cma-data/[alloc_range_2]/fail
...
/<path-to-cma-data/[alloc_range_max]/success
/<path-to-cma-data/[alloc_range_max]/fail
The idea is that knowing the allocation sizes that succeeded and failed is maybe even more interesting and useful than knowing the cma area that contains them.
Quoted 4 lines
I agree it would be also useful but I'd like to enable it under
CONFIG_CMA_SYSFS_ALLOC_RANGEas separate patchset.
I will stop harassing you very soon, just want to bottom out on understanding the real goals first. :)
thanks,
Signature · 2 lines
John Hubbard NVIDIA
Quoted 12 lines · John Hubbard
On 2/5/21 8:15 AM, Minchan Kim wrote:
...Quoted 9 lines
Quoted 4 lines
Yes, approximately. I was wondering if this would suffice at least as a baseline:
cma_alloc_success125cma_alloc_failure25IMO, regardless of the my patch, it would be good to have such statistics in that CMA was born to replace carved out memory with dynamic allocation ideally for memory efficiency ideally so failure should regard critical so admin could notice it how the system is hurt.
Quoted 8 lines
Right. So CMA failures are useful for the admin to see, understood.
Quoted 3 lines
Anyway, it's not enough for me and orthgonal with my goal.
OK. But...what *is* your goal, and why is this useless (that's what orthogonal really means here) for your goal?
As I mentioned, the goal is to monitor the failure from each of CMA since they have each own purpose.
Let's have an example.
System has 5 CMA area and each CMA is associated with each user scenario. They have exclusive CMA area to avoid fragmentation problem.
CMA-1 depends on bluetooh
CMA-2 depends on WIFI
CMA-3 depends on sensor-A
CMA-4 depends on sensor-B
CMA-5 depends on sensor-C
With this, we could catch which module was affected but with global failure, I couldn't find who was affected.
Quoted 59 lines
Also, would you be willing to try out something simple first, such as providing indication that cma is active and it's overall success rate, like this:
/proc/vmstat:
cma_alloc_success125cma_alloc_failure25...or is the only way to provide the more detailed items, complete with per-CMA details, in a non-debugfs location?
Quoted 8 lines
Quoted 5 lines
...and then, to see if more is needed, some questions:
a) Do you know of an upper bound on how many cma areas there can be (I think Matthew also asked that)?
There is no upper bound since it's configurable.
OK, thanks,so that pretty much rules out putting per-cma details into anything other than a directory or something like it.
Quoted 14 lines
Quoted 4 lines
b) Is tracking the cma area really as valuable as other possibilities? We can put "a few" to "several" items here, so really want to get your very favorite bits of information in. If, for example, there can be *lots* of cma areas, then maybe tracking
At this moment, allocation/failure for each CMA area since they have particular own usecase, which makes me easy to keep which module will be affected. I think it is very useful per-CMA statistics as minimum code change so I want to enable it by default under
CONFIG_CMA&&CONFIG_SYSFS.Quoted 1 line
by a range of allocation sizes is better...
I takes your suggestion something like this.
Quoted 8 lines
[
alloc_range] could be order or range by interval/sys/kernel/mm/cma/cma-A/[alloc_range]/success
/sys/kernel/mm/cma/cma-A/[alloc_range]/fail
..
..
/sys/kernel/mm/cma/cma-Z/[alloc_range]/success
/sys/kernel/mm/cma/cma-Z/[alloc_range]/failActually, I meant, "ranges instead of cma areas", like this:
/<path-to-cma-data/[alloc_range_1]/success
/<path-to-cma-data/[alloc_range_1]/fail
/<path-to-cma-data/[alloc_range_2]/success
/<path-to-cma-data/[alloc_range_2]/fail
...
/<path-to-cma-data/[alloc_range_max]/success
/<path-to-cma-data/[alloc_range_max]/fail
Quoted 3 lines
The idea is that knowing the allocation sizes that succeeded and failed is maybe even more interesting and useful than knowing the cma area that contains them.
Understand your point but it would make hard to find who was affected by the failure. That's why I suggested to have your suggestion under additional config since per-cma metric with simple sucess/failure are enough.
Quoted 9 lines
Quoted 5 lines
I agree it would be also useful but I'd like to enable it under
CONFIG_CMA_SYSFS_ALLOC_RANGEas separate patchset.I will stop harassing you very soon, just want to bottom out on understanding the real goals first. :)
I hope my example makes the goal more clear for you.
Suren Baghdasaryan
reviewerQuoted 32 lines · Minchan Kim
Quoted 12 lines · John Hubbard
On 2/5/21 8:15 AM, Minchan Kim wrote:
...Quoted 9 lines
Quoted 4 lines
Yes, approximately. I was wondering if this would suffice at least as a baseline:
cma_alloc_success125cma_alloc_failure25IMO, regardless of the my patch, it would be good to have such statistics in that CMA was born to replace carved out memory with dynamic allocation ideally for memory efficiency ideally so failure should regard critical so admin could notice it how the system is hurt.
Quoted 8 lines
Right. So CMA failures are useful for the admin to see, understood.
Quoted 3 lines
Anyway, it's not enough for me and orthgonal with my goal.
OK. But...what *is* your goal, and why is this useless (that's what orthogonal really means here) for your goal?
As I mentioned, the goal is to monitor the failure from each of CMA since they have each own purpose.
Let's have an example.
System has 5 CMA area and each CMA is associated with each user scenario. They have exclusive CMA area to avoid fragmentation problem.
Quoted 57 lines
CMA-1 depends on bluetooh
CMA-2 depends on WIFI
CMA-3 depends on sensor-A
CMA-4 depends on sensor-B
CMA-5 depends on sensor-CWith this, we could catch which module was affected but with global failure, I couldn't find who was affected.
Quoted 48 lines
Also, would you be willing to try out something simple first, such as providing indication that cma is active and it's overall success rate, like this:
/proc/vmstat:
cma_alloc_success125cma_alloc_failure25...or is the only way to provide the more detailed items, complete with per-CMA details, in a non-debugfs location?
Quoted 8 lines
Quoted 5 lines
...and then, to see if more is needed, some questions:
a) Do you know of an upper bound on how many cma areas there can be (I think Matthew also asked that)?
There is no upper bound since it's configurable.
OK, thanks,so that pretty much rules out putting per-cma details into anything other than a directory or something like it.
Quoted 22 lines
Quoted 4 lines
b) Is tracking the cma area really as valuable as other possibilities? We can put "a few" to "several" items here, so really want to get your very favorite bits of information in. If, for example, there can be *lots* of cma areas, then maybe tracking
At this moment, allocation/failure for each CMA area since they have particular own usecase, which makes me easy to keep which module will be affected. I think it is very useful per-CMA statistics as minimum code change so I want to enable it by default under
CONFIG_CMA&&CONFIG_SYSFS.Quoted 1 line
by a range of allocation sizes is better...
I takes your suggestion something like this.
[
alloc_range] could be order or range by interval/sys/kernel/mm/cma/cma-A/[alloc_range]/success
/sys/kernel/mm/cma/cma-A/[alloc_range]/fail
..
..
/sys/kernel/mm/cma/cma-Z/[alloc_range]/success
/sys/kernel/mm/cma/cma-Z/[alloc_range]/fail
The interface above seems to me the most useful actually, if by [alloc_range] you mean the different allocation orders. This would cover Minchan's per-CMA failure tracking and would also allow us to understand what kind of allocations are failing and therefore if the problem is caused by pinning/fragmentation or by over-utilization.
Quoted 31 lines
Quoted 11 lines
Actually, I meant, "ranges instead of cma areas", like this:
/<path-to-cma-data/[alloc_range_1]/success
/<path-to-cma-data/[alloc_range_1]/fail
/<path-to-cma-data/[alloc_range_2]/success
/<path-to-cma-data/[alloc_range_2]/fail
...
/<path-to-cma-data/[alloc_range_max]/success
/<path-to-cma-data/[alloc_range_max]/failQuoted 3 lines
The idea is that knowing the allocation sizes that succeeded and failed is maybe even more interesting and useful than knowing the cma area that contains them.
Understand your point but it would make hard to find who was affected by the failure. That's why I suggested to have your suggestion under additional config since per-cma metric with simple sucess/failure are enough.
Quoted 9 lines
Quoted 5 lines
I agree it would be also useful but I'd like to enable it under
CONFIG_CMA_SYSFS_ALLOC_RANGEas separate patchset.I will stop harassing you very soon, just want to bottom out on understanding the real goals first. :)
I hope my example makes the goal more clear for you.
Quoted 11 lines · Suren Baghdasaryan
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
I takes your suggestion something like this.
[
alloc_range] could be order or range by interval/sys/kernel/mm/cma/cma-A/[alloc_range]/success
/sys/kernel/mm/cma/cma-A/[alloc_range]/fail
..
..
/sys/kernel/mm/cma/cma-Z/[alloc_range]/success
/sys/kernel/mm/cma/cma-Z/[alloc_range]/fail
Quoted 6 lines
The interface above seems to me the most useful actually, if by [
alloc_range] you mean the different allocation orders. This would cover Minchan's per-CMA failure tracking and would also allow us to understand what kind of allocations are failing and therefore if the problem is caused by pinning/fragmentation or by over-utilization.
I agree. That seems about right, now that we've established that cma areas are a must-have.
thanks,
Signature · 2 lines
John Hubbard NVIDIA
Quoted 19 lines · John Hubbard
Quoted 11 lines · Suren Baghdasaryan
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
I takes your suggestion something like this.
[
alloc_range] could be order or range by interval/sys/kernel/mm/cma/cma-A/[alloc_range]/success
/sys/kernel/mm/cma/cma-A/[alloc_range]/fail
..
..
/sys/kernel/mm/cma/cma-Z/[alloc_range]/success
/sys/kernel/mm/cma/cma-Z/[alloc_range]/failQuoted 6 lines
The interface above seems to me the most useful actually, if by [
alloc_range] you mean the different allocation orders. This would cover Minchan's per-CMA failure tracking and would also allow us to understand what kind of allocations are failing and therefore if the problem is caused by pinning/fragmentation or by over-utilization.
Quoted 2 lines
I agree. That seems about right, now that we've established that cma areas are a must-have.
Okay, now we agreed the dirction right now so let me do that in next version. If you don't see it's reasonable, let me know.
* I will drop the number of CMA *page* allocation attemtps/failures to make simple start
* I will keep CMA allocation attemtps/failures
* They will be under /sys/kernel/mm/cma/cma-XX/success,fail
* It will turn on CONFIG_CMA && CONFIG_SYSFS
Orthognal work(diffrent patchset)
* adding global CMA alloc/fail into vmstat
* adding alloc_range success/failure under CONFIG_CMA_ALLOC_TRACKING whatever configuration or just by default if everyone agree
Thanks.
Quoted 35 lines · Minchan Kim
Quoted 19 lines · John Hubbard
Quoted 11 lines · Suren Baghdasaryan
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
I takes your suggestion something like this.
[
alloc_range] could be order or range by interval/sys/kernel/mm/cma/cma-A/[alloc_range]/success
/sys/kernel/mm/cma/cma-A/[alloc_range]/fail
..
..
/sys/kernel/mm/cma/cma-Z/[alloc_range]/success
/sys/kernel/mm/cma/cma-Z/[alloc_range]/failQuoted 6 lines
The interface above seems to me the most useful actually, if by [
alloc_range] you mean the different allocation orders. This would cover Minchan's per-CMA failure tracking and would also allow us to understand what kind of allocations are failing and therefore if the problem is caused by pinning/fragmentation or by over-utilization.Quoted 2 lines
I agree. That seems about right, now that we've established that cma areas are a must-have.
Okay, now we agreed the dirction right now so let me do that in next version. If you don't see it's reasonable, let me know.
* I will drop the number of CMA *page* allocation attemtps/failures to make simple start
* I will keep CMA allocation attemtps/failures
* They will be under/sys/kernel/mm/cma/cma-XX/success,fail
* It will turn onCONFIG_CMA&&CONFIG_SYSFSOrthognal work(diffrent patchset)
Quoted 8 lines
* adding global CMA alloc/fail into vmstat
* addingalloc_rangesuccess/failure underCONFIG_CMA_ALLOC_TRACKINGwhatever configuration or just by default if everyone agree# cat meminfo | grep -i cma CmaTotal: 1048576 kB CmaFree: 1046872 kB
This CMA info was added by me way back in 2014.
At that time I even thought about adding this cma alloc/fail counter in vmstat. That time I also had an internal patch about it but later dropped (never released to mainline).
If required I can re-work again on this part.
And I have few more points to add here.
1) In the past I have faced this cma allocation failure (which could be common in small embedded devices).
Earlier it was even difficult to figure if cma failure actually happened. Thus I added this simple patch:
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?h=v5.11-rc6&id=5984af1082f3b115082178ed88c47033d43b924d
2) IMO just reporting CMA alloc/fail may not be enough (at times). The main point is for which client/dev is this failure happening ? Sometimes just tuning the size or fixing the alignment can resolve the failure if we know the client. For global CMA it will be just NULL (dev).
3) IMO having 2 options SYSFS and DEBUGFS may confuse the developer/user (personal experience). So are we going to remove the DEBUGFS or merge it ?
4) Sometimes CMA (or DMA) allocation failure can happen even very early in boot time itself. At that time these are anyways not useful. Some system may not proceed if CMA/DMA allocation is failing (again personal experience).
==> Anyways this is altogether is different case...
5) The default max CMA areas are defined to be 7 but user can configure it to any number, may be 20 or 50 (???)
6) Thus I would like to propose another thing here.
Just like we have /proc/vmallocinfo, /proc/slabinfo, etc., we can also
have: /proc/cmainfo
Here in /proc/cmaminfo we can capute more detailed information:
Global CMA Data: Alloc/Free
Client specific data: name, size, success, fail, flags, align, etc (just a random example).
==> This can dynamically grow as large as possible
==> It can also be enabled/disabled based on CMA config itself (no additional config required)
Any feedback on point (6) specifically ?
Thanks,
Pintu
On 2/6/21 9:08 AM, Pintu Agarwal wrote:
...
Quoted 23 lines
Quoted 3 lines
# cat meminfo | grep -i cma CmaTotal: 1048576 kB CmaFree: 1046872 kBThis CMA info was added by me way back in 2014.
At that time I even thought about adding this cma alloc/fail counter in vmstat. That time I also had an internal patch about it but later dropped (never released to mainline).
If required I can re-work again on this part.And I have few more points to add here.
1) In the past I have faced this cma allocation failure (which could be common in small embedded devices).
Earlier it was even difficult to figure if cma failure actually happened. Thus I added this simple patch:
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?h=v5.11-rc6&id=5984af1082f3b115082178ed88c47033d43b924d2) IMO just reporting CMA alloc/fail may not be enough (at times). The main point is for which client/dev is this failure happening ? Sometimes just tuning the size or fixing the alignment can resolve the failure if we know the client. For global CMA it will be just NULL (dev).
Quoted 4 lines
3) IMO having 2 options SYSFS and DEBUGFS may confuse the developer/user (personal experience). So are we going to remove the DEBUGFS or merge it ?
It is confusing to have a whole bunch of different places to find data about a system. Here, I think the key is to add to the Documentation/ directory. So far, the documentation I see is:
admin-guide/mm/cma_debugfs.rst: only covers debugfs admin-guide/kernel-parameters.txt:602: for CMA kernel parameters
If we add sysfs items then we will want a new .rst document that covers that, and lists all the places to look.
So anyway, the underlying guidelines for which fs to user are approximately:
* sysfs: used for monitoring. One value per item, stable ABI, production.
* debugfs: *theoretically* not a stable ABI (we hope no one locks us in by doing obnoxious things that break userspace if the debugfs APIs change). The intention is that developers can put *anything* in there.
debugfs has a firm place in debugging, and is probably a keeper here.
I originally thought we should combine CMA's sysfs and debugfs items, but Minchan listed an example that seems to show a pretty clear need for monitoring of CMA areas, in production systems, and that's what sysfs is for.
So it looks like we want both debugfs and sysfs for CMA, plus a new overall CMA documentation that points to everything.
Quoted 20 lines
4) Sometimes CMA (or DMA) allocation failure can happen even very early in boot time itself. At that time these are anyways not useful. Some system may not proceed if CMA/DMA allocation is failing (again personal experience).
==> Anyways this is altogether is different case...5) The default max CMA areas are defined to be 7 but user can configure it to any number, may be 20 or 50 (???)
6) Thus I would like to propose another thing here.
Just like we have/proc/vmallocinfo,/proc/slabinfo, etc., we can also
have:/proc/cmainfo
Here in/proc/cmaminfowe can capute more detailed information:
Global CMA Data: Alloc/Free
Client specific data: name, size, success, fail, flags, align, etc (just a random example).
==> This can dynamically grow as large as possible
==> It can also be enabled/disabled based on CMA config itself (no additional config required)
Quoted 2 lines
Any feedback on point (6) specifically ?
Well, /proc these days is for process-specific items. And CMA areas are system-wide. So that's an argument against it. However...if there is any process-specific CMA allocation info to report, then /proc is just the right place for it.
thanks,
Signature · 2 lines
John Hubbard NVIDIA
Quoted 15 lines · Minchan Kim
Quoted 4 lines · John Hubbard
On 2/5/21 8:15 AM, Minchan Kim wrote:
...
OK. But...what *is* your goal, and why is this useless (that's what orthogonal really means here) for your goal?As I mentioned, the goal is to monitor the failure from each of CMA since they have each own purpose.
Let's have an example.
System has 5 CMA area and each CMA is associated with each user scenario. They have exclusive CMA area to avoid fragmentation problem.
Quoted 6 lines
CMA-1 depends on bluetooh
CMA-2 depends on WIFI
CMA-3 depends on sensor-A
CMA-4 depends on sensor-B
CMA-5 depends on sensor-C
aha, finally. I had no idea that sort of use case was happening.
This would be good to put in the patch commit description.
Quoted 77 lines
With this, we could catch which module was affected but with global failure, I couldn't find who was affected.
Quoted 59 lines
Also, would you be willing to try out something simple first, such as providing indication that cma is active and it's overall success rate, like this:
/proc/vmstat:
cma_alloc_success125cma_alloc_failure25...or is the only way to provide the more detailed items, complete with per-CMA details, in a non-debugfs location?
Quoted 8 lines
Quoted 5 lines
...and then, to see if more is needed, some questions:
a) Do you know of an upper bound on how many cma areas there can be (I think Matthew also asked that)?
There is no upper bound since it's configurable.
OK, thanks,so that pretty much rules out putting per-cma details into anything other than a directory or something like it.
Quoted 14 lines
Quoted 4 lines
b) Is tracking the cma area really as valuable as other possibilities? We can put "a few" to "several" items here, so really want to get your very favorite bits of information in. If, for example, there can be *lots* of cma areas, then maybe tracking
At this moment, allocation/failure for each CMA area since they have particular own usecase, which makes me easy to keep which module will be affected. I think it is very useful per-CMA statistics as minimum code change so I want to enable it by default under
CONFIG_CMA&&CONFIG_SYSFS.Quoted 1 line
by a range of allocation sizes is better...
I takes your suggestion something like this.
Quoted 8 lines
[
alloc_range] could be order or range by interval/sys/kernel/mm/cma/cma-A/[alloc_range]/success
/sys/kernel/mm/cma/cma-A/[alloc_range]/fail
..
..
/sys/kernel/mm/cma/cma-Z/[alloc_range]/success
/sys/kernel/mm/cma/cma-Z/[alloc_range]/failActually, I meant, "ranges instead of cma areas", like this:
/<path-to-cma-data/[alloc_range_1]/success
/<path-to-cma-data/[alloc_range_1]/fail
/<path-to-cma-data/[alloc_range_2]/success
/<path-to-cma-data/[alloc_range_2]/fail
...
/<path-to-cma-data/[alloc_range_max]/success
/<path-to-cma-data/[alloc_range_max]/failQuoted 3 lines
The idea is that knowing the allocation sizes that succeeded and failed is maybe even more interesting and useful than knowing the cma area that contains them.
Understand your point but it would make hard to find who was affected by the failure. That's why I suggested to have your suggestion under additional config since per-cma metric with simple sucess/failure are enough.
Quoted 6 lines
Quoted 5 lines
I agree it would be also useful but I'd like to enable it under
CONFIG_CMA_SYSFS_ALLOC_RANGEas separate patchset.
Quoted 6 lines
Quoted 3 lines
I will stop harassing you very soon, just want to bottom out on understanding the real goals first. :)
I hope my example makes the goal more clear for you.
Yes it does. Based on the (rather surprising) use of cma-area-per-device, it seems clear that you will need this, so I'll drop my objections to putting it in sysfs.
I still think the "number of allocation failures" needs refining, probably via a range-based thing, as we've discussed. But the number of pages failed per cma looks OK now.
thanks,
Signature · 2 lines
John Hubbard NVIDIA
mm/Makefile obj-$(CONFIG_ZSMALLOC) += zsmalloc.o · line 109
Quoted 6 lines · Minchan Kim
+++ b/mm/Makefile
mm/Makefile +1
-0@@ -106,6 +106,7 @@ obj-$(CONFIG_ZSMALLOC) += zsmalloc.o obj-$(CONFIG_Z3FOLD) += z3fold.o obj-$(CONFIG_GENERIC_EARLY_IOREMAP) += early_ioremap.o obj-$(CONFIG_CMA) += cma.o +obj-$(CONFIG_SYSFS) += cma_sysfs.o
ehh ... if we have a kernel build with CMA=n, SYSFS=y, we'll get cma_sysfs built in with no cma to report on.
mm/cma_sysfs.c line 43
Quoted 9 lines
mm/cma_sysfs.c new file +9
-0+static ssize_t cma_alloc_attempt_show (struct kobject *kobj, +struct kobj_attribute *attr,char *buf) +{ +unsigned long val; + struct cma_stat *stat = container_of(kobj, struct cma_stat, kobj); + + val = stat->alloc_attempt; + +return sysfs_emit (buf,"%lu\n" , val);
Why not more simply:
{
struct cma_stat *stat = container_of(kobj, struct cma_stat, kobj);
return sysfs_emit(buf, "%lu\n", stat->alloc_attempt);
}mm/cma_sysfs.c line 121
Quoted 5 lines
mm/cma_sysfs.c new file +5
-0+for (i =0 ; i < cma_area_count; i++) { + cma = &cma_areas[i]; + stat =kzalloc (sizeof (*stat),GFP_KERNEL ); +if (!stat) + goto out;
How many cma areas are there going to be? do we really want to allocate their stat individually?
Quoted 10 lines · Matthew Wilcox
mm/Makefile obj-$(CONFIG_ZSMALLOC) += zsmalloc.o · line 109
Quoted 6 lines · Minchan Kim
+++ b/mm/Makefile
mm/Makefile +1
-0@@ -106,6 +106,7 @@ obj-$(CONFIG_ZSMALLOC) += zsmalloc.o obj-$(CONFIG_Z3FOLD) += z3fold.o obj-$(CONFIG_GENERIC_EARLY_IOREMAP) += early_ioremap.o obj-$(CONFIG_CMA) += cma.o +obj-$(CONFIG_SYSFS) += cma_sysfs.oehh ... if we have a kernel build with CMA=n, SYSFS=y, we'll get
cma_sysfsbuilt in with no cma to report on.
OMG. Let me fix it.
Quoted 13 lines
mm/cma_sysfs.c line 43
Quoted 10 lines
mm/cma_sysfs.c new file +9
-0+static ssize_t cma_alloc_attempt_show (struct kobject *kobj, +struct kobj_attribute *attr,char *buf) +{ +unsigned long val; + struct cma_stat *stat = container_of(kobj, struct cma_stat, kobj); + + val = stat->alloc_attempt; + +return sysfs_emit (buf,"%lu\n" , val);Why not more simply:
Quoted 3 lines
{
struct cma_stat *stat = container_of(kobj, struct cma_stat, kobj);
return sysfs_emit(buf, "%lu\n", stat->alloc_attempt);It's a legacy when I used the lock there but removed finally. Will follow your suggestion.
Quoted 10 lines
}
mm/cma_sysfs.c line 121
Quoted 5 lines
mm/cma_sysfs.c new file +5
-0+for (i =0 ; i < cma_area_count; i++) { + cma = &cma_areas[i]; + stat =kzalloc (sizeof (*stat),GFP_KERNEL ); +if (!stat) + goto out;How many cma areas are there going to be? do we really want to allocate their stat individually?
I am not sure what could be in the end but at least, I have 5+ candidates (but could be shrink or extend) and yes, want to keep track them individually.
Quoted 9 lines · Minchan Kim
Quoted 8 lines
mm/cma_sysfs.c line 121
Quoted 5 lines
mm/cma_sysfs.c new file +5
-0+for (i =0 ; i < cma_area_count; i++) { + cma = &cma_areas[i]; + stat =kzalloc (sizeof (*stat),GFP_KERNEL ); +if (!stat) + goto out;How many cma areas are there going to be? do we really want to allocate their stat individually?
Quoted 3 lines
I am not sure what could be in the end but at least, I have 5+ candidates (but could be shrink or extend) and yes, want to keep track them individually.
I meant, wouldn't it be better to:
cma_stats = kzalloc(array_size(sizeof(*stat), cma_area_count), GFP_KERNEL);
Quoted 10 lines · Matthew Wilcox
Quoted 9 lines · Minchan Kim
Quoted 8 lines
mm/cma_sysfs.c line 121
Quoted 5 lines
mm/cma_sysfs.c new file +5
-0+for (i =0 ; i < cma_area_count; i++) { + cma = &cma_areas[i]; + stat =kzalloc (sizeof (*stat),GFP_KERNEL ); +if (!stat) + goto out;How many cma areas are there going to be? do we really want to allocate their stat individually?
Quoted 9 lines
Quoted 3 lines
I am not sure what could be in the end but at least, I have 5+ candidates (but could be shrink or extend) and yes, want to keep track them individually.
I meant, wouldn't it be better to:
cma_stats = kzalloc(array_size(sizeof(*stat), cma_area_count), GFP_KERNEL);
Definitely.
Thanks, Matthew.