perf_event_open: improve the memory leak detection
PATCH 1/1 perf_event_open: improve the memory leak detection
The goal is to add more robust memory leak detection by periodically sampling memory availability during the test run and checking for significant decreases in available memory.
To avoid false postive:
perf_event_open03.c:95: TFAIL: Likely kernel memory leak detected
--- .../perf_event_open/perf_event_open03.c | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-)
testcases/kernel/syscalls/perf_event_open/perf_event_open03.c +26 -6
@@ -26,6 +26,7 @@const int iterations =12000000 ;static int fd =-1 ;static int runtime; +static int sample;static void setup (void ) {
@@ -77,22 +78,41 @@ static void check_progress(int i)static void run (void ) { -long diff; +long diff, diff_total, mem_avail, mem_avail_prev;int i; - diff =SAFE_READ_MEMINFO ("MemAvailable:" ); + sample =0 ; + diff_total =0 ; + + mem_avail_prev =SAFE_READ_MEMINFO ("MemAvailable:" );tst_timer_start (CLOCK_MONOTONIC );/* leak about 100MB of RAM */ for (i =0 ; i < iterations; i++) {ioctl (fd,PERF_EVENT_IOC_SET_FILTER ,"filter,0/0@abcd" );check_progress (i); - } - diff -=SAFE_READ_MEMINFO ("MemAvailable:" ); +/* +* Every 1200000 iterations, calculate the difference in memory +* availability. If the difference is greater than 10 * 1024 (10MB), +* increment the sample counter and log the event. +*/ +if ((i %1200000 ) ==0 ) { + mem_avail =SAFE_READ_MEMINFO ("MemAvailable:" ); + diff = mem_avail_prev - mem_avail; + diff_total += diff; + +if (diff >20 *1024 ) { + sample++; +tst_res (TINFO ,"MemAvailable decreased by %ld kB at iteration %d" , diff, i); + } + + mem_avail_prev = mem_avail; + } + } -if (diff >50 *1024 ) -tst_res (TFAIL ,"Likely kernel memory leak detected" ); +if ((sample >5 ) || (diff_total >100 *1024 )) +tst_res (TFAIL ,"Likely kernel memory leak detected, total decrease: %ld kB" , diff_total);else tst_res (TPASS ,"No memory leak found" ); }
Signature · 5 lines
2.45.2 -- Mailing list info: https://lists.linux.it/listinfo/ltp
Quoted 64 lines · Li Wang
The goal is to add more robust memory leak detection by periodically sampling memory availability during the test run and checking for significant decreases in available memory.
To avoid false postive:
perf_event_open03.c:95: TFAIL: Likely kernel memory leak detectedSigned-off-by Li Wang <[email protected]>--- .../perf_event_open/perf_event_open03.c | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-)testcases/kernel/syscalls/perf_event_open/perf_event_open03.c +8
-4@@ -26,6 +26,7 @@const int iterations =12000000 ;static int fd =-1 ;static int runtime; +static int sample;static void setup (void ) {@@ -77,22 +78,41 @@ static void check_progress(int i)static void run (void ) { -long diff; +long diff, diff_total, mem_avail, mem_avail_prev;int i; - diff =SAFE_READ_MEMINFO ("MemAvailable:" ); + sample =0 ; + diff_total =0 ; + + mem_avail_prev =SAFE_READ_MEMINFO ("MemAvailable:" );tst_timer_start (CLOCK_MONOTONIC );/* leak about 100MB of RAM */ for (i =0 ; i < iterations; i++) {ioctl (fd,PERF_EVENT_IOC_SET_FILTER ,"filter,0/0@abcd" );check_progress (i); - } - diff -=SAFE_READ_MEMINFO ("MemAvailable:" ); +/* +* Every 1200000 iterations, calculate the difference in memory
+ * availability. If the difference is greater than 10 *
1024 (10MB),testcases/kernel/syscalls/perf_event_open/perf_event_open03.c +10
-0+* increment the sample counter and log the event. +*/ +if ((i %1200000 ) ==0 ) { + mem_avail =SAFE_READ_MEMINFO ("MemAvailable:" ); + diff = mem_avail_prev - mem_avail; + diff_total += diff; + +if (diff >20 *1024 ) { + sample++; + tst_res(TINFO, "MemAvailable decreased by%ld kB at iteration %d", diff, i);
testcases/kernel/syscalls/perf_event_open/perf_event_open03.c static void check_progress(int i) · line 114
Quoted 9 lines
testcases/kernel/syscalls/perf_event_open/perf_event_open03.c +6
-2+ } + + mem_avail_prev = mem_avail; + } + } -if (diff >50 *1024 ) -tst_res (TFAIL ,"Likely kernel memory leak detected" ); +if ((sample >5 ) || (diff_total >100 *1024 ))
Here maybe diff_toal is still the interference factor in reporting false positives,
I'm hesitating to remove or keep it.
Or we could use it first to see how the test performs in the large-scale CI testing.
Quoted 13 lines
+ tst_res(TFAIL, "Likely kernel memory leak detected, total
decrease: %ld kB", diff_total); else tst_res(TPASS, "No memory leak found");}
--
2.45.2--
Mailing list info: https://lists.linux.it/listinfo/ltp
Signature · 5 lines
Regards, Li Wang -- Mailing list info: https://lists.linux.it/listinfo/ltp
Hi Cyril, All,
Any comments on this method?
Quoted 85 lines · Li Wang
The goal is to add more robust memory leak detection by periodically sampling memory availability during the test run and checking for significant decreases in available memory.
To avoid false postive:
perf_event_open03.c:95: TFAIL: Likely kernel memory leak detectedSigned-off-by Li Wang <[email protected]>--- .../perf_event_open/perf_event_open03.c | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-)testcases/kernel/syscalls/perf_event_open/perf_event_open03.c +8
-4@@ -26,6 +26,7 @@const int iterations =12000000 ;static int fd =-1 ;static int runtime; +static int sample;static void setup (void ) {@@ -77,22 +78,41 @@ static void check_progress(int i)static void run (void ) { -long diff; +long diff, diff_total, mem_avail, mem_avail_prev;int i; - diff =SAFE_READ_MEMINFO ("MemAvailable:" ); + sample =0 ; + diff_total =0 ; + + mem_avail_prev =SAFE_READ_MEMINFO ("MemAvailable:" );tst_timer_start (CLOCK_MONOTONIC );/* leak about 100MB of RAM */ for (i =0 ; i < iterations; i++) {ioctl (fd,PERF_EVENT_IOC_SET_FILTER ,"filter,0/0@abcd" );check_progress (i); - } - diff -=SAFE_READ_MEMINFO ("MemAvailable:" ); +/* +* Every 1200000 iterations, calculate the difference in memory
+ * availability. If the difference is greater than 10 *
1024 (10MB),testcases/kernel/syscalls/perf_event_open/perf_event_open03.c +10
-0+* increment the sample counter and log the event. +*/ +if ((i %1200000 ) ==0 ) { + mem_avail =SAFE_READ_MEMINFO ("MemAvailable:" ); + diff = mem_avail_prev - mem_avail; + diff_total += diff; + +if (diff >20 *1024 ) { + sample++; + tst_res(TINFO, "MemAvailable decreased by%ld kB at iteration %d", diff, i);
testcases/kernel/syscalls/perf_event_open/perf_event_open03.c +7
-2+ } + + mem_avail_prev = mem_avail; + } + } -if (diff >50 *1024 ) -tst_res (TFAIL ,"Likely kernel memory leak detected" ); +if ((sample >5 ) || (diff_total >100 *1024 )) + tst_res(TFAIL, "Likely kernel memory leak detected, totaldecrease: %ld kB", diff_total); else tst_res(TPASS, "No memory leak found");}
--
2.45.2--
Mailing list info: https://lists.linux.it/listinfo/ltp
Signature · 5 lines
Regards, Li Wang -- Mailing list info: https://lists.linux.it/listinfo/ltp
Hi!
testcases/kernel/syscalls/perf_event_open/perf_event_open03.c static void check_progress(int i) · line 105
Quoted 11 lines
testcases/kernel/syscalls/perf_event_open/perf_event_open03.c +11
-0+/* +* Every 1200000 iterations, calculate the difference in memory +* availability. If the difference is greater than 10 * 1024 (10MB), +* increment the sample counter and log the event. +*/ +if ((i %1200000 ) ==0 ) { + mem_avail =SAFE_READ_MEMINFO ("MemAvailable:" ); + diff = mem_avail_prev - mem_avail; + diff_total += diff; + +if (diff >20 *1024 ) {
Shouldn't this be 10 * 1024 or possibly slightly less than 10 * 1024?
testcases/kernel/syscalls/perf_event_open/perf_event_open03.c static void check_progress(int i) · line 114
Quoted 11 lines
testcases/kernel/syscalls/perf_event_open/perf_event_open03.c +8
-2+ sample++; +tst_res (TINFO ,"MemAvailable decreased by %ld kB at iteration %d" , diff, i); + } + + mem_avail_prev = mem_avail; + } + } -if (diff >50 *1024 ) -tst_res (TFAIL ,"Likely kernel memory leak detected" ); +if ((sample >5 ) || (diff_total >100 *1024 ))
Maybe this can rather be if ((sample > 5) && (diff_total > 100 * 1024))
That means that the available memory has been eaten by something and that it happened more or less in a linear fashion when the program was running.
Quoted 7 lines
testcases/kernel/syscalls/perf_event_open/perf_event_open03.c +1
-0+tst_res (TFAIL ,"Likely kernel memory leak detected, total decrease: %ld kB" , diff_total); else tst_res(TPASS, "No memory leak found"); }Signature · 1 line
2.45.2
Signature · 5 lines
Cyril Hrubis [email protected] -- Mailing list info: https://lists.linux.it/listinfo/ltp
Quoted 14 lines · Cyril Hrubis
Hi!
testcases/kernel/syscalls/perf_event_open/perf_event_open03.c static void check_progress(int i) · line 105
Quoted 11 lines
testcases/kernel/syscalls/perf_event_open/perf_event_open03.c +11
-0+/* +* Every 1200000 iterations, calculate the difference in memory +* availability. If the difference is greater than 10 * 1024 (10MB), +* increment the sample counter and log the event. +*/ +if ((i %1200000 ) ==0 ) { + mem_avail =SAFE_READ_MEMINFO ("MemAvailable:" ); + diff = mem_avail_prev - mem_avail; + diff_total += diff; + +if (diff >20 *1024 ) {Shouldn't this be 10 * 1024 or possibly slightly less than 10 * 1024?
This should be fine. The test leaks around 38MB between sample checks and over 350MB in a whole test run.
Quoted 15 lines
testcases/kernel/syscalls/perf_event_open/perf_event_open03.c static void check_progress(int i) · line 114
Quoted 12 lines
testcases/kernel/syscalls/perf_event_open/perf_event_open03.c +8
-2+ sample++; +tst_res (TINFO ,"MemAvailable decreased by %ld kB at iteration %d" , diff, i); + } + + mem_avail_prev = mem_avail; + } + } -if (diff >50 *1024 ) -tst_res (TFAIL ,"Likely kernel memory leak detected" ); +if ((sample >5 ) || (diff_total >100 *1024 ))Maybe this can rather be if ((sample > 5) && (
diff_total> 100 * 1024))
Quoted 3 lines
That means that the available memory has been eaten by something and that it happened more or less in a linear fashion when the program was running.
Imagine that some other process releases 300MB of memory while the test is running. If you change the || to &&, you'll get a false negative in that case. The sampling approach will protect against such interference.
That being said, if the available memory on your test system fluctuates by 100+MB during a test run that takes <10 seconds, I'd recommend investigating what's causing such fluctuation. On the test machine I used to verify this patch, I can see <10MB of difference before and after running the test on a fixed kernel.
Quoted 9 lines
Quoted 8 lines
testcases/kernel/syscalls/perf_event_open/perf_event_open03.c +1
-0+tst_res (TFAIL ,"Likely kernel memory leak detected, total decrease: %ld kB" , diff_total); else tst_res(TPASS, "No memory leak found"); }Signature · 1 line
2.45.2
Signature · 11 lines
Martin Doucha [email protected] SW Quality Engineer SUSE LINUX, s.r.o. CORSO IIa Krizikova 148/34 186 00 Prague 8 Czech Republic -- Mailing list info: https://lists.linux.it/listinfo/ltp
Hi!
Quoted 14 lines
Quoted 5 lines
Quoted 5 lines
testcases/kernel/syscalls/perf_event_open/perf_event_open03.c +5
-0+/* +* Every 1200000 iterations, calculate the difference in memory +* availability. If the difference is greater than 10 * 1024 (10MB), +* increment the sample counter and log the event. +*/ Quoted 8 lines
testcases/kernel/syscalls/perf_event_open/perf_event_open03.c static void check_progress(int i) · line 105
Quoted 6 lines
testcases/kernel/syscalls/perf_event_open/perf_event_open03.c +6
-0+if ((i %1200000 ) ==0 ) { + mem_avail =SAFE_READ_MEMINFO ("MemAvailable:" ); + diff = mem_avail_prev - mem_avail; + diff_total += diff; + +if (diff >20 *1024 ) {Shouldn't this be 10 * 1024 or possibly slightly less than 10 * 1024?
Quoted 2 lines
This should be fine. The test leaks around 38MB between sample checks and over 350MB in a whole test run.
Then we should fix the comment.
Quoted 23 lines
Quoted 15 lines
testcases/kernel/syscalls/perf_event_open/perf_event_open03.c static void check_progress(int i) · line 114
Quoted 12 lines
testcases/kernel/syscalls/perf_event_open/perf_event_open03.c +8
-2+ sample++; +tst_res (TINFO ,"MemAvailable decreased by %ld kB at iteration %d" , diff, i); + } + + mem_avail_prev = mem_avail; + } + } -if (diff >50 *1024 ) -tst_res (TFAIL ,"Likely kernel memory leak detected" ); +if ((sample >5 ) || (diff_total >100 *1024 ))Maybe this can rather be if ((sample > 5) && (
diff_total> 100 * 1024))Quoted 3 lines
That means that the available memory has been eaten by something and that it happened more or less in a linear fashion when the program was running.
Imagine that some other process releases 300MB of memory while the test is running. If you change the || to &&, you'll get a false negative in that case. The sampling approach will protect against such interference.
Quoted 5 lines
That being said, if the available memory on your test system fluctuates by 100+MB during a test run that takes <10 seconds, I'd recommend investigating what's causing such fluctuation. On the test machine I used to verify this patch, I can see <10MB of difference before and after running the test on a fixed kernel.
So shall we remove the diff_total completely?
Signature · 5 lines
Cyril Hrubis [email protected] -- Mailing list info: https://lists.linux.it/listinfo/ltp
Quoted 10 lines · Cyril Hrubis
Quoted 10 lines
Quoted 5 lines
Maybe this can rather be if ((sample > 5) && (
diff_total> 100 * 1024))That means that the available memory has been eaten by something and that it happened more or less in a linear fashion when the program was running.
Imagine that some other process releases 300MB of memory while the test is running. If you change the || to &&, you'll get a false negative in that case. The sampling approach will protect against such interference.
Quoted 7 lines
Quoted 5 lines
That being said, if the available memory on your test system fluctuates by 100+MB during a test run that takes <10 seconds, I'd recommend investigating what's causing such fluctuation. On the test machine I used to verify this patch, I can see <10MB of difference before and after running the test on a fixed kernel.
So shall we remove the
diff_totalcompletely?
No, diff_total is still useful because it checks for smaller leak than the sum of samples.
Signature · 11 lines
Martin Doucha [email protected] SW Quality Engineer SUSE LINUX, s.r.o. CORSO IIa Krizikova 148/34 186 00 Prague 8 Czech Republic -- Mailing list info: https://lists.linux.it/listinfo/ltp
Hi Martin, Cyril,
Quoted 17 lines · Martin Doucha
Quoted 16 lines · Cyril Hrubis
Quoted 15 lines
Quoted 5 lines
Maybe this can rather be if ((sample > 5) && (
diff_total> 100 * 1024))That means that the available memory has been eaten by something and that it happened more or less in a linear fashion when the program was running.
Imagine that some other process releases 300MB of memory while the test is running. If you change the || to &&, you'll get a false negative in that case. The sampling approach will protect against such interference.
That being said, if the available memory on your test system fluctuates by 100+MB during a test run that takes <10 seconds, I'd recommend investigating what's causing such fluctuation. On the test machine I used to verify this patch, I can see <10MB of difference before and after running the test on a fixed kernel.
Quoted 5 lines
Quoted 1 line
So shall we remove the
diff_totalcompletely?No,
diff_totalis still useful because it checks for smaller leak than the sum of samples.
Thanks so much for your confirmation and test.
I pushed the patch with improvements as you suggested. Let's see how it goes in future tests.
Signature · 5 lines
Regards, Li Wang -- Mailing list info: https://lists.linux.it/listinfo/ltp
Hi,
I've tested the patch on an affected kernel and it still reproduces the leak. Small nit below, but otherwise:
Quoted 17 lines · Li Wang
The goal is to add more robust memory leak detection by periodically sampling memory availability during the test run and checking for significant decreases in available memory.
To avoid false postive:
perf_event_open03.c:95: TFAIL: Likely kernel memory leak detectedSigned-off-by Li Wang <[email protected]>--- .../perf_event_open/perf_event_open03.c | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-)testcases/kernel/syscalls/perf_event_open/perf_event_open03.c +0
-0
testcases/kernel/syscalls/perf_event_open/perf_event_open03.c line 29
Quoted 4 lines
testcases/kernel/syscalls/perf_event_open/perf_event_open03.c +1
-0@@ -26,6 +26,7 @@const int iterations =12000000 ;static int fd =-1 ;static int runtime; +static int sample;
Does this variable need to be global? It's used only in run() and it gets reset to zero between test iterations.
Quoted 51 lines
testcases/kernel/syscalls/perf_event_open/perf_event_open03.c +25
-6static void setup (void ) {@@ -77,22 +78,41 @@ static void check_progress(int i)static void run (void ) { -long diff; +long diff, diff_total, mem_avail, mem_avail_prev;int i; - diff =SAFE_READ_MEMINFO ("MemAvailable:" ); + sample =0 ; + diff_total =0 ; + + mem_avail_prev =SAFE_READ_MEMINFO ("MemAvailable:" );tst_timer_start (CLOCK_MONOTONIC );/* leak about 100MB of RAM */ for (i =0 ; i < iterations; i++) {ioctl (fd,PERF_EVENT_IOC_SET_FILTER ,"filter,0/0@abcd" );check_progress (i); - } - diff -=SAFE_READ_MEMINFO ("MemAvailable:" ); +/* +* Every 1200000 iterations, calculate the difference in memory +* availability. If the difference is greater than 10 * 1024 (10MB), +* increment the sample counter and log the event. +*/ +if ((i %1200000 ) ==0 ) { + mem_avail =SAFE_READ_MEMINFO ("MemAvailable:" ); + diff = mem_avail_prev - mem_avail; + diff_total += diff; + +if (diff >20 *1024 ) { + sample++; +tst_res (TINFO ,"MemAvailable decreased by %ld kB at iteration %d" , diff, i); + } + + mem_avail_prev = mem_avail; + } + } -if (diff >50 *1024 ) -tst_res (TFAIL ,"Likely kernel memory leak detected" ); +if ((sample >5 ) || (diff_total >100 *1024 )) +tst_res (TFAIL ,"Likely kernel memory leak detected, total decrease: %ld kB" , diff_total);else tst_res (TPASS ,"No memory leak found" ); }
Signature · 11 lines
Martin Doucha [email protected] SW Quality Engineer SUSE LINUX, s.r.o. CORSO IIa Krizikova 148/34 186 00 Prague 8 Czech Republic -- Mailing list info: https://lists.linux.it/listinfo/ltp