Skip to contentSkip to conversation

Update cgroup_fj and memcg controller tests to work under cgroup2

PATCH In review Luke Nowakowski-Krijger 4 participants 17 messages Nov 27, 2021 – Dec 13, 2021

lore
Share & tools
lore
#0 lore raw

cover Update cgroup_fj and memcg controller tests to work under cgroup2

These tests were only testing for controllers under cgroup1. Now that many controllers are being mounted under cgroup2 by default, we should update these tests to test things where they make sense and at least report something when they don't.

Luke Nowakowski-Krijger (4):
controllers/memcg: update stress test to work under cgroup2 controllers/memcg: Skip functional tests when mounted under cgroup2
hierarchy
controllers/cgroup_fj: Update cgroup_fj_common to work under cgroup2 controllers/cgroup_fj: Update cgroup_fj_function to work under cgroup2

 .../controllers/cgroup_fj/cgroup_fj_common.sh | 73 +++++++++++++------
 .../cgroup_fj/cgroup_fj_function.sh           | 38 +++++++++-
 .../controllers/memcg/functional/memcg_lib.sh |  8 ++
 .../memcg/stress/memcg_stress_test.sh         | 73 ++++++++++++++-----
 4 files changed, 148 insertions(+), 44 deletions(-)
Signature · 5 lines
2.32.0


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
#1 lore raw

PATCH 1/4 controllers/memcg: update stress test to work under cgroup2

Update tests to be able to work when memory controller is mounted under cgroup2 hierarchy.

Remove redundant mounts so that it mounts once for both tests to keep the logic a bit simpler.

---
 .../memcg/stress/memcg_stress_test.sh         | 73 ++++++++++++++-----
 1 file changed, 56 insertions(+), 17 deletions(-)
testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh +56 -17
index c43d72116..ba7c8f386 100755
@@ -21,12 +21,62 @@ TST_TIMEOUT=2100
 
 . cgroup_lib.sh
 
+setup_cgroupv2()
+{
+	mount_point=$(grep -w cgroup2 /proc/mounts | cut -f 2 | cut -d " " -f2)
+	if ! grep -q memory "$mount_point"/cgroup.controllers; then
+		tst_res TINFO "memory controller not supported on cgroup v2."
+		return
+	fi
+
+	task_list="cgroup.procs"
+	cgroup_v="v2"
+}
+
+setup_cgroupv1()
+{
+	exist=$(grep -w memory /proc/cgroups | cut -f1);
+	if [ "$exist" = "" ]; then
+		tst_brk TCONF NULL "memory controller not supported"
+	fi
+
+	mount_point=$(grep -w memory /proc/mounts | cut -f 2 | cut -d " " -f2)
+	if [ "$mount_point" = "" ]; then
+		cgroup_mounted=0
+		mount_point="/dev/memcg"
+	fi
+
+	if [ "$cgroup_mounted" -eq "0" ]; then
+		ROD mkdir -p $mount_point
+		ROD mount -t cgroup -o memory none $mount_point
+	fi
+
+	task_list="tasks"
+	cgroup_v="v1"
+}
+
 setup()
 {
 	if ! is_cgroup_subsystem_available_and_enabled "memory"; then
 		tst_brk TCONF "Either kernel does not support Memory Resource Controller or feature not enabled"
 	fi
 
+	if tst_kvcmp -lt "2.6.30"; then
+		tst_brk TBROK "Test should be run with kernel 2.6.30 or newer"
+	fi
+
+	cgroup_mounted=1
+
+	if grep -q cgroup2 /proc/mounts; then
+		setup_cgroupv2
+	fi
+
+	if [ -z "$cgroup_v" ]; then
+		setup_cgroupv1
+	fi
+
+	tst_res TINFO "test starts with cgroup $cgroup_v"
+
 	echo 3 > /proc/sys/vm/drop_caches
 	sleep 2
 	local mem_free=`cat /proc/meminfo | grep MemFree | awk '{ print $2 }'`
@@ -43,20 +93,12 @@ setup()
 
 cleanup()
 {
-	if [ -e /dev/memcg ]; then
-		EXPECT_PASS umount /dev/memcg
-		EXPECT_PASS rmdir /dev/memcg
+	if [ $cgroup_mounted -eq "0" ]; then
+		EXPECT_PASS umount $mount_point
+		EXPECT_PASS rmdir $mount_point
 	fi
 }
 
-do_mount()
-{
-	cleanup
-
-	EXPECT_PASS mkdir /dev/memcg
-	EXPECT_PASS mount -t cgroup -omemory memcg /dev/memcg
-}
-
 # $1 Number of cgroups
 # $2 Allocated MB memory in one process
 # $3 The interval to touch memory in a process
@@ -71,13 +113,11 @@ run_stress()
 
 	tst_res TINFO "Testing $cgroups cgroups, using $mem_size MB, interval $interval"
 
-	do_mount
-
 	tst_res TINFO "Starting cgroups"
 	for i in $(seq 0 $(($cgroups-1))); do
-		mkdir /dev/memcg/$i 2> /dev/null
+		mkdir "$mount_point/$i" 2> /dev/null
 		memcg_process_stress $mem_size $interval &
-		echo $! > /dev/memcg/$i/tasks
+		echo $! > "$mount_point/$i/$task_list"
 		pids="$pids $!"
 	done
 
@@ -93,12 +133,11 @@ run_stress()
 	for pid in $pids; do
 		kill -KILL $pid 2> /dev/null
 		wait $pid 2> /dev/null
-		rmdir /dev/memcg/$i 2> /dev/null
+		rmdir "$mount_point/$i" 2> /dev/null
 		i=$((i+1))
 	done
 
 	tst_res TPASS "Test passed"
-	cleanup
 }
 
 test1()
Signature · 5 lines
2.32.0


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
#2 lore raw

Re: [LTP] [PATCH 1/4] controllers/memcg: update stress test to work under cgroup2

Hi Luke,

Thanks for looking into this.

Quoted 3 lines · Luke Nowakowski-Krijger

Update tests to be able to work when memory controller is mounted under cgroup2 hierarchy.

I'm thinking whether we could achieve these setup functions more generic in cgroup_lib.sh, which to avoid the redundancy behavior over and over again in each cgroup sub-test.

About the compatibility of cgroup V1 and V2 in test, I'd suggest following what the LTP C library did in include/tst_cgroup.h.
https://github.com/linux-test-project/ltp/blob/master/doc/c-test-api.txt#L2024

The solution may be briefly as:

1. scan system mounted CGroup path, and judge the situation as one of
below:
* only CGroup V2 mounted
* only CGroup V1 mounted
* both CGroup V2 and V1 mounted
* no CGroup mounted
2. make use of the system mounted CGroup V2 or TSKIP
* goto step 5
3. make use of the system mounted CGroup V1
* goto step 5
4. do mount Cgroup as what we needed (V1 or V2) in test
* goto step 5
5. do cleanup

What do you think?

Signature · 2 lines
Regards,
Li Wang
Signature · 2 lines
-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
#3 lore raw

Re: [LTP] [PATCH 1/4] controllers/memcg: update stress test to work under cgroup2

Hi Li,

Quoted 11 lines · Li Wang

Hi Luke,

Thanks for looking into this.

Quoted 3 lines · Luke Nowakowski-Krijger

Update tests to be able to work when memory controller is mounted under cgroup2 hierarchy.

Quoted 5 lines

I'm thinking whether we could achieve these setup functions more generic in cgroup_lib.sh, which to avoid the redundancy behavior over and over again in each cgroup sub-test.

Yes I agree. As I was doing the same things a few times I was beginning to

wonder if there was a better way. I would be willing to look further into expanding the cgroup_lib.sh and resubmitting my recent patches with those changes.

Quoted 8 lines

About the compatibility of cgroup V1 and V2 in test, I'd suggest following what the LTP C library did in include/tst_cgroup.h.

https://github.com/linux-test-project/ltp/blob/master/doc/c-test-api.txt#L2024

The solution may be briefly as:

1. scan system mounted CGroup path, and judge the situation as one of

Quoted 4 lines

below:
* only CGroup V2 mounted
* only CGroup V1 mounted
* both CGroup V2 and V1 mounted

Quoted 10 lines

* no CGroup mounted
2. make use of the system mounted CGroup V2 or TSKIP
* goto step 5
3. make use of the system mounted CGroup V1
* goto step 5
4. do mount Cgroup as what we needed (V1 or V2) in test
* goto step 5
5. do cleanup

Yes this would be the way to go through setting up a controller and checking everything.
Going through the steps you mentioned for mounting a single controller and returning that path wouldn't be too hard but
it seems to get more complicated when we want some guarantee of having multiple controllers in a hierarchy (if we even
would want to support something like that, which for testing purposes wouldnt seem unheard of).
Maybe something mimicking the tst_cgroup_require() from the C api would be useful here? I imagine this is where we would
do the checking and mounting logic that you were describing. We would just also have to include checking if the controllers
we are requiring can be mounted / already exist together.

For example I am imaging something mimicking the C api something like:
tst_cgroup_require "cpu"
tst_cgroup_require "cpuset"
root_mount_point =$(tst_cgroup_get_mountpoint)

or just combined them together

root_mount_point = $(tst_cgroup_get_mountpoint "cpu cpuset")

Again, most of the tests seem to only be testing individual controllers from what I can see
so I don't know if this would be too useful. Let me know what you think.

Quoted 6 lines

What do you think?

--
Regards,
Li Wang

Best,
- Luke

Signature · 2 lines
-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
#4 lore raw

Re: [LTP] [PATCH 1/4] controllers/memcg: update stress test to work under cgroup2

Quoted 15 lines · Luke Nowakowski-Krijger

Hi Li,

Quoted 11 lines · Li Wang

Hi Luke,

Thanks for looking into this.

Quoted 3 lines · Luke Nowakowski-Krijger

Update tests to be able to work when memory controller is mounted under cgroup2 hierarchy.

Quoted 9 lines
Quoted 5 lines

I'm thinking whether we could achieve these setup functions more generic in cgroup_lib.sh, which to avoid the redundancy behavior over and over again in each cgroup sub-test.

Yes I agree. As I was doing the same things a few times I was beginning

to wonder if there was a better way. I would be willing to look further into expanding the cgroup_lib.sh and resubmitting my recent patches with those changes.

Thanks a lot!

Quoted 24 lines

About the compatibility of cgroup V1 and V2 in test, I'd suggest

Quoted 7 lines

following what the LTP C library did in include/tst_cgroup.h.

https://github.com/linux-test-project/ltp/blob/master/doc/c-test-api.txt#L2024

The solution may be briefly as:

1. scan system mounted CGroup path, and judge the situation as one of

Quoted 4 lines

below:
* only CGroup V2 mounted
* only CGroup V1 mounted
* both CGroup V2 and V1 mounted

Quoted 10 lines

* no CGroup mounted
2. make use of the system mounted CGroup V2 or TSKIP
* goto step 5
3. make use of the system mounted CGroup V1
* goto step 5
4. do mount Cgroup as what we needed (V1 or V2) in test
* goto step 5
5. do cleanup

Quoted 9 lines

Yes this would be the way to go through setting up a controller and checking everything.
Going through the steps you mentioned for mounting a single controller and returning that path wouldn't be too hard but
it seems to get more complicated when we want some guarantee of having multiple controllers in a hierarchy (if we even
would want to support something like that, which for testing purposes wouldnt seem unheard of).

Right, it is the complicated part and you can take a reference how the current C API handles it.

TBH, I even think to skip the handling on mixed mounting with V1 and V2, that is too messy/overthinking and not suggested using way.

We'd better face the future and ideally as myself hoping, V2 will replace V1 everywhere someday:).

Quoted 11 lines

Maybe something mimicking the tst_cgroup_require() from the C api would be useful here? I imagine this is where we would
do the checking and mounting logic that you were describing. We would just also have to include checking if the controllers
we are requiring can be mounted / already exist together.

For example I am imaging something mimicking the C api something like:
tst_cgroup_require "cpu"
tst_cgroup_require "cpuset"
root_mount_point =$(tst_cgroup_get_mountpoint)

I prefer this one a bit, not only it's consistent with C API but it also can do CGroup mounting in tst_cgroup_require for a system without V1 nor V2 mounting. Then I'd suggest having tst_cgroup_cleanup to help umount that which makes things more clear to understand.

Anyway, it depends on the details achieve, maybe there is a better solution we haven't found.

Quoted 10 lines

or just combined them together

root_mount_point = $(tst_cgroup_get_mountpoint "cpu cpuset")

Again, most of the tests seem to only be testing individual controllers from what I can see
so I don't know if this would be too useful. Let me know what you think.

Yes, the existing legacy tests based on one controller should just work well to use the new APIs you're going to achieve. But eventually, I think, those tests also need further refactoring or rewriting someday.

And, the most important part is that it (CGroup shell APIs) will be recommended to be used when a new test is written in the future, so now it requires interfaces that have good scalability and compatibility. No matter the test target for CGroup itself or just as an auxiliary part, these shell APIs will play a key ingredient.

Signature · 2 lines
Regards,
Li Wang
Signature · 2 lines
-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
#5 lore raw

Re: [LTP] [PATCH 1/4] controllers/memcg: update stress test to work under cgroup2

Hello Li and Luke

Quoted 10 lines · Li Wang

On Thu, Dec 2, 2021 at 6:24 AM Luke Nowakowski-Krijger <[email protected]> wrote:

Hi Li,

On Wed, Dec 1, 2021 at 1:13 AM Li Wang <[email protected]> wrote:

Hi Luke,

Thanks for looking into this.

Quoted 8 lines

On Sat, Nov 27, 2021 at 8:05 AM Luke Nowakowski-Krijger <[email protected]> wrote:

Update tests to be able to work when memory controller is mounted under cgroup2 hierarchy.

I'm thinking whether we could achieve these setup functions more generic in cgroup_lib.sh, which to avoid the redundancy behavior over and over again in each cgroup sub-test.

+1 this is very necessary IMO

Quoted 27 lines

Yes I agree. As I was doing the same things a few times I was beginning to wonder if there was a better way. I would be willing to look further into expanding the cgroup_lib.sh and resubmitting my recent patches with
those changes.

Thanks a lot!

About the compatibility of cgroup V1 and V2 in test, I'd suggest following what the LTP C library did in include/tst_cgroup.h.
https://github.com/linux-test-project/ltp/blob/master/doc/c-test-api.txt#L2024

The solution may be briefly as:

1. scan system mounted CGroup path, and judge the situation as one of below:
* only CGroup V2 mounted
* only CGroup V1 mounted
* both CGroup V2 and V1 mounted
* no CGroup mounted
2. make use of the system mounted CGroup V2 or TSKIP
* goto step 5
3. make use of the system mounted CGroup V1
* goto step 5
4. do mount Cgroup as what we needed (V1 or V2) in test
* goto step 5
5. do cleanup

Quoted 7 lines

Yes this would be the way to go through setting up a controller and checking everything.
Going through the steps you mentioned for mounting a single controller and returning that path wouldn't be too hard but it seems to get more complicated when we want some guarantee of having multiple controllers in a hierarchy (if we even would want to support something like that, which for testing purposes wouldnt seem unheard of).

Right, it is the complicated part and you can take a reference how the current C API handles it.

Actually we can use the C API. This would avoid a whole bunch of issues. It requires creating a utility which we can use from shell (e.g. tst_cgctl).

We *may* have to track state somehow. Either we could run the utility as a daemon initially and communicate with a socket to execute commands. Or we could save/serialise some state to environment/shell variables. Alternatively we can probably rescan the system each time. The only state really required is the test's PID which is needed to find the correct CGroup in the LTP sub-hierarchy.

Still that is probably easier than dealing with all of the corner cases twice.

Quoted 6 lines

TBH, I even think to skip the handling on mixed mounting with V1 and V2, that is too messy/overthinking and not suggested using way.

We'd better face the future and ideally as myself hoping, V2 will replace V1 everywhere someday:).

There are still controllers/features that don't exist on V2. Meanwhile there are features that only exist on V2. So if someone wants both, then they have to mount both. Regardless, this was the default in systemd, so we are stuck with it for about 20 years and can't ignore it ;-)

Quoted 11 lines

Maybe something mimicking the tst_cgroup_require() from the C api would be useful here? I imagine this is where we would do the checking and mounting logic that you were describing. We would just also have to include checking if the controllers we are requiring can be mounted / already exist together.

For example I am imaging something mimicking the C api something like:
tst_cgroup_require "cpu"
tst_cgroup_require "cpuset"
root_mount_point =$(tst_cgroup_get_mountpoint)

Quoted 7 lines

I prefer this one a bit, not only it's consistent with C API but it also can do CGroup mounting in tst_cgroup_require for a system without V1 nor V2 mounting. Then I'd suggest having tst_cgroup_cleanup to help umount that which makes things more clear to understand.

Anyway, it depends on the details achieve, maybe there is a better solution we haven't found.

Yes, or if it is a utility then

$ test_cpu_cg_dir = $(tst_cgroup require cpu)
$ test_cpu_cg_dir = $(tst_cgroup require memory)

maybe with an option to pass the PID to indetify the test. I guess there might be some existing env variable the shell API sets we could use as well.

$ tst_cgroup cleanup --pid $MAIN_PID
Signature · 5 lines
Thank you,
Richard.

-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
#6 lore raw

Re: [LTP] [PATCH 1/4] controllers/memcg: update stress test to work under cgroup2

Hi Richard and Li,

Quoted 37 lines · Richard Palethorpe

Hello Li and Luke

Quoted 1 line · Li Wang

On Thu, Dec 2, 2021 at 6:24 AM Luke Nowakowski-Krijger <

Quoted 10 lines · [email protected]> wrote:

Hi Li,

On Wed, Dec 1, 2021 at 1:13 AM Li Wang <[email protected]> wrote:

Hi Luke,

Thanks for looking into this.

On Sat, Nov 27, 2021 at 8:05 AM Luke Nowakowski-Krijger <

Quoted 7 lines · [email protected]> wrote:

Update tests to be able to work when memory controller is mounted under cgroup2 hierarchy.

I'm thinking whether we could achieve these setup functions more generic in cgroup_lib.sh, which to avoid the redundancy behavior over and over again in each cgroup sub-test.

+1 this is very necessary IMO

Actually we can use the C API. This would avoid a whole bunch of issues. It requires creating a utility which we can use from shell (e.g. tst_cgctl).

We *may* have to track state somehow. Either we could run the utility as a daemon initially and communicate with a socket to execute commands. Or we could save/serialise some state to environment/shell variables. Alternatively we can probably rescan the system each time. The only state really required is the test's PID which is needed to find the correct CGroup in the LTP sub-hierarchy.

Quoted 4 lines

Still that is probably easier than dealing with all of the corner cases twice.

I rather like this idea. If I understand you correctly we could use it in

an RPC sort of way which would make a lot of things simpler and use the existing C API which is nice.

My one question would be if we would want this daemon to run as a test suite utility, like it seems you are suggesting, or as a per process utility.

The nice part of having a daemon that we could fork off for every test that uses it would be that the cleanup / tracking of sub-groups would get cleaned up in the normal way when we want to close the daemon and just call tst_cgroup_cleanup(). The daemons state would be tied to the test that's issuing commands to it. We could also send out the commands via a shared buffer or pipe that we read and write to.

But is a daemon per test (that uses the cgroup shell api) overkill? It seems it would spare us from having to track the test PID to sub-hierarchy like you were mentioning. Or maybe there are some other drawbacks to the per-test daemon idea that I'm not seeing?

Looking forward to hearing what you think.

Quoted 42 lines
Quoted 6 lines

TBH, I even think to skip the handling on mixed mounting with V1 and V2, that is too messy/overthinking and not suggested using way.

We'd better face the future and ideally as myself hoping, V2 will replace V1 everywhere someday:).

There are still controllers/features that don't exist on V2. Meanwhile there are features that only exist on V2. So if someone wants both, then they have to mount both. Regardless, this was the default in systemd, so we are stuck with it for about 20 years and can't ignore it ;-)

Quoted 3 lines

Maybe something mimicking the tst_cgroup_require() from the C api would

be useful here? I imagine this is where we would

Quoted 1 line

do the checking and mounting logic that you were describing. We would

just also have to include checking if the controllers

Quoted 7 lines

we are requiring can be mounted / already exist together.

For example I am imaging something mimicking the C api something like:
tst_cgroup_require "cpu"
tst_cgroup_require "cpuset"
root_mount_point =$(tst_cgroup_get_mountpoint)

Quoted 7 lines

I prefer this one a bit, not only it's consistent with C API but it also can do CGroup mounting in tst_cgroup_require for a system without V1 nor V2 mounting. Then I'd suggest having tst_cgroup_cleanup to help umount that which makes things more clear to understand.

Anyway, it depends on the details achieve, maybe there is a better solution we haven't found.

Yes, or if it is a utility then

$ test_cpu_cg_dir = $(tst_cgroup require cpu)
$ test_cpu_cg_dir = $(tst_cgroup require memory)

maybe with an option to pass the PID to indetify the test. I guess there might be some existing env variable the shell API sets we could use as well.

Quoted 6 lines
$ tst_cgroup cleanup --pid $MAIN_PID

--
Thank you,
Richard.

Thanks,
- Luke

Signature · 2 lines
-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
#7 lore raw

Re: [LTP] [PATCH 1/4] controllers/memcg: update stress test to work under cgroup2

Quoted 9 lines · Luke Nowakowski-Krijger
Quoted 5 lines · Richard Palethorpe

Actually we can use the C API. This would avoid a whole bunch of issues. It requires creating a utility which we can use from shell (e.g. tst_cgctl).

+1 This is a good idea.

Quoted 11 lines
Quoted 11 lines

We *may* have to track state somehow. Either we could run the utility as a daemon initially and communicate with a socket to execute commands. Or we could save/serialise some state to environment/shell variables. Alternatively we can probably rescan the system each time. The only state really required is the test's PID which is needed to find the correct CGroup in the LTP sub-hierarchy.

Still that is probably easier than dealing with all of the corner cases twice.

Quoted 8 lines
Quoted 1 line

I rather like this idea. If I understand you correctly we could use it in

an RPC sort of way which would make a lot of things simpler and use the existing C API which is nice.

My one question would be if we would want this daemon to run as a test suite utility, like it seems you are suggesting, or as a per process utility.

Hmm, why do we need that utility as a daemon in the background? Isn't it easier to execute a binary utility to get the CGroup path only when needed? Just like Richard mentioned the alternative way, rescan system each time and only distinguish correct CGroup path via the test PID.

Quoted 9 lines

The nice part of having a daemon that we could fork off for every test that uses it would be that the cleanup / tracking of sub-groups would get cleaned up in the normal way when we want to close the daemon and just call tst_cgroup_cleanup(). The daemons state would be tied to the test that's issuing commands to it. We could also send out the commands via a shared buffer or pipe that we read and write to.

Quoted 5 lines

But is a daemon per test (that uses the cgroup shell api) overkill? It seems it would spare us from having to track the test PID to sub-hierarchy like you were mentioning. Or maybe there are some other drawbacks to the per-test daemon idea that I'm not seeing?

I think yes, starting a daemon per test is not wise.

Another drawback I can think of is that will definitely affect paralleling things,
we must guarantee the CGroup mounted by testA shouldn't be scanned/used by testB, otherwise, it will fail in the cleanup phase. But, we can make the LTP
test mounted CGroup path is transparent to others just by adding a special string
like "ltp-cgroup".

Signature · 2 lines
Regards,
Li Wang
Signature · 2 lines
-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
#8 lore raw

Re: [LTP] [PATCH 1/4] controllers/memcg: update stress test to work under cgroup2

Hi Richard and Li,

Hmm, why do we need that utility as a daemon in the background?

Quoted 6 lines

Isn't it easier to execute a binary utility to get the CGroup path only when needed? Just like Richard mentioned the alternative way, rescan system each time and only distinguish correct CGroup path via the test PID.

Yes this would be easier if we only wanted to get access to the path. I

was getting ahead of myself and thinking about using the C-api to keep track of creating sub-cgroups and all the state that comes with that. Which we wouldn't want to do for a shell utility because creation/cleanup of subgroups is for the test to take care of and that is more trivial in a shell environment.

Quoted 28 lines
Quoted 11 lines

The nice part of having a daemon that we could fork off for every test that uses it would be that the cleanup / tracking of sub-groups would get cleaned up in the normal way when we want to close the daemon and just call tst_cgroup_cleanup(). The daemons state would be tied to the test that's issuing commands to it. We could also send out the commands via a shared buffer or pipe that we read and write to.

Quoted 5 lines

But is a daemon per test (that uses the cgroup shell api) overkill? It seems it would spare us from having to track the test PID to sub-hierarchy like you were mentioning. Or maybe there are some other drawbacks to the per-test daemon idea that I'm not seeing?

I think yes, starting a daemon per test is not wise.

Another drawback I can think of is that will definitely affect paralleling things,
we must guarantee the CGroup mounted by testA shouldn't be scanned/used by testB, otherwise, it will fail in the cleanup phase. But, we can make the LTP
test mounted CGroup path is transparent to others just by adding a special string
like "ltp-cgroup".

Quoted 1 line

If I understand it correctly, all the cgroup tests are mounted and created

like "controller_mount_point/ltp/test-$PID" where every test shares the mount_point and ltp dir. And when tst_cgroup_cleanup() gets called it only cleans up its own test_dir and removes the ltp dir and unmounts the mount point if it was the first test to do it. So none of the tests should be touching each other's directories and so what you're describing should already be taken care of. Maybe I'm not understanding you correctly.

I think the only problem with the binary utility approach where we rescan every time we execute it is that
1) The test-$PID dir that the test would be created with the PID of the program which if we were executing the utility could be different between calls. This could be easily solved by adding an arg to tst_cgroup_opts for a specific PID, which would be the test that is calling it.
2) We lose the reference between calls to the root->test_dir that is filled in when we call tst_cgroup_require(), which is what does the cleanup of the test specific dir. This is where I believe Richard was mentioning passing the PID as "tst_cgroup cleanup --pid $MAIN_PID". Which if we wanted to use the C api for this we would have to expose it to knowing about specific PIDs? Or for the root->test_dir to be reset somewhere?

Quoted 4 lines

--
Regards,
Li Wang

Let me know if this makes sense and what you think about it, I might be getting confused somewhere. But if I understand you correctly I believe that the binary utility approach where we rescan and call tst_cgroup_require() or tst_cgroup_cleanup() is a good approach.

Best,
- Luke

Signature · 2 lines
-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
#9 lore raw

Re: [LTP] [PATCH 1/4] controllers/memcg: update stress test to work under cgroup2

Hello Luke and Li,

Quoted 10 lines · Luke Nowakowski-Krijger

Hi Richard and Li,

Hmm, why do we need that utility as a daemon in the background? Isn't it easier to execute a binary utility to get the CGroup path only when needed? Just like Richard mentioned the alternative way, rescan system each time and only distinguish correct CGroup path via the test PID.

Yes this would be easier if we only wanted to get access to the path. I was getting ahead of myself and thinking about using the C-api to keep track of creating sub-cgroups and all the state that comes with that. Which we wouldn't want to do for a shell utility because creation/cleanup of subgroups is for the test to take care of and that is more trivial in a shell environment.

Yes; tests which require complicated hierarchies have to cleanup themselves. I think it would be a waste of time to add a generic sub-directory cleanup mechanism until we have a number of tests which would benefit from it.

Especially because some controllers might place special restrictions on directory and process manipulation. (cpuset for example?).

Quoted 17 lines

The nice part of having a daemon that we could fork off for every test that uses it would be that the cleanup / tracking of sub-groups would get cleaned up in the normal way when we want to close the daemon and just call tst_cgroup_cleanup(). The daemons state would be tied to the test that's issuing commands to it. We could also send out the commands via a shared buffer or pipe that
we read and write to.

But is a daemon per test (that uses the cgroup shell api) overkill? It seems it would spare us from having to track the test PID to sub-hierarchy like you were mentioning. Or maybe there are some other drawbacks to the per-test daemon idea that I'm not seeing?

I think yes, starting a daemon per test is not wise.

Another drawback I can think of is that will definitely affect paralleling things, we must guarantee the CGroup mounted by testA shouldn't be scanned/used by testB, otherwise, it will fail in the cleanup phase. But, we can make the LTP test mounted CGroup path is transparent to others just by adding a special string like "ltp-cgroup".

Quoted 4 lines

If I understand it correctly, all the cgroup tests are mounted and created like "controller_mount_point/ltp/test-$PID" where every test shares the mount_point and ltp dir. And when tst_cgroup_cleanup() gets called it only cleans up its own test_dir and removes the ltp dir and unmounts the mount point if it was the first test to do it. So none of the tests should be touching each other's directories and so
what you're describing should already be taken care of. Maybe I'm not
understanding you correctly.

I think maybe you are talking about two different things. However IMO daemons should be avoided in general. There is no hard requirement here to have a long running process.

Quoted 6 lines

I think the only problem with the binary utility approach where we rescan every time we execute it is that
1) The test-$PID dir that the test would be created with the PID of the program which if we were executing the utility could be different between calls. This could be easily solved by adding an arg to tst_cgroup_opts for a specific PID, which would be the test that is calling it.
2) We lose the reference between calls to the root->test_dir that is filled in when we call tst_cgroup_require(), which is what does the cleanup of the test specific dir. This is where I believe Richard was mentioning passing the PID as "tst_cgroup cleanup --pid $MAIN_PID". Which if we wanted to use the C api for this we would have to expose it to knowing about specific PIDs? Or for the root->test_dir to
be reset somewhere?

I suppose we also need to remember if the current test created the ltp subdirectory or mounted any controllers.

We could do this by just printing the required command line options to stdout when the utility exits. Then save this to a variable for the next use.

Quoted 9 lines

--
Regards,
Li Wang

Let me know if this makes sense and what you think about it, I might be getting confused somewhere. But if I understand you correctly I believe that the binary utility approach where we rescan and call tst_cgroup_require() or tst_cgroup_cleanup() is a good approach.

Best,
- Luke

Yes, sounds good.

Signature · 5 lines
Thank you,
Richard.

-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
#10 lore raw

Re: [LTP] [PATCH 1/4] controllers/memcg: update stress test to work under cgroup2

Hi Richard and Li,

Quoted 4 lines · Richard Palethorpe

I suppose we also need to remember if the current test created the ltp subdirectory or mounted any controllers.

We could do this by just printing the required command line options to

Quoted 4 lines

stdout when the utility exits. Then save this to a variable for the next use.

Yes, we would have to remember what we mounted. I think the part I am most curious about is how we would generate that state i.e what we mounted, because the Cgroup library does not expose any of this as far as I'm aware.

If we want to use the tst_cgroup C lib to cleanup as well we would have to find a way to reintroduce test state to the lib that we are losing between calls of the utility, which the only way I could think of is introducing a way to export and import test state within the lib. e.g. tst_cgroup_print_test_state() tst_cgroup_load_test_state(), which doesn't feel good as it exposes some of the nice API you have going on. This is the easiest way to tell if we are mounting things because we can just print what we mounted, what the test dir of the test is, and reload that state. This could have further applications to not just this scenario but also to scenarios where if a test dies its state can be reloaded, etc, almost in a checkpoint way. Not saying its common but adds some flexibility to the API and I could see it having applications outside of this utility.

Alternatively we could inspect what we created and generate state that way, i.e. make a call to tst_cgroup_require() and see if new things were mounted. Then we would have to manually be freeing things. I don't like this approach because it goes against the whole point of this which was code reuse. But the cleanup of things isnt the most difficult part so it wouldn't be the biggest deal to redo the logic.

Yes, sounds good.

Quoted 2 lines

Let me know what you think. I wouldn't want to add anything huge to the API without your blessing :)

Quoted 4 lines

--
Thank you,
Richard.

Thanks,
- Luke

Signature · 2 lines
-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
#11 lore raw

Re: [LTP] [PATCH 1/4] controllers/memcg: update stress test to work under cgroup2

Hi!

Quoted 24 lines

Yes, we would have to remember what we mounted. I think the part I am most curious about is how we would generate that state i.e what we mounted, because the Cgroup library does not expose any of this as far as I'm aware.

If we want to use the tst_cgroup C lib to cleanup as well we would have to find a way to reintroduce test state to the lib that we are losing between calls of the utility, which the only way I could think of is introducing a way to export and import test state within the lib. e.g. tst_cgroup_print_test_state() tst_cgroup_load_test_state(), which doesn't feel good as it exposes some of the nice API you have going on. This is the easiest way to tell if we are mounting things because we can just print what we mounted, what the test dir of the test is, and reload that state. This could have further applications to not just this scenario but also to scenarios where if a test dies its state can be reloaded, etc, almost in a checkpoint way. Not saying its common but adds some flexibility to the API and I could see it having applications outside of this utility.

Alternatively we could inspect what we created and generate state that way, i.e. make a call to tst_cgroup_require() and see if new things were mounted. Then we would have to manually be freeing things. I don't like this approach because it goes against the whole point of this which was code reuse. But the cleanup of things isnt the most difficult part so it wouldn't be the biggest deal to redo the logic.

Quoted 5 lines

Yes, sounds good.

Quoted 2 lines

Let me know what you think. I wouldn't want to add anything huge to the API without your blessing :)

Wouldn't it be easier to rewrite these test to the C then? I think that error handling in shell CGroup tests would always be more difficuilt than in C and given that we have a nice library for C it actually sounds like a better solution.

Signature · 5 lines
Cyril Hrubis
[email protected]

-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
#12 lore raw

Re: [LTP] [PATCH 1/4] controllers/memcg: update stress test to work under cgroup2

Hi!

Quoted 7 lines
Quoted 2 lines

Let me know what you think. I wouldn't want to add anything huge to the API without your blessing :)

Wouldn't it be easier to rewrite these test to the C then? I think that error handling in shell CGroup tests would always be more difficuilt than in C and given that we have a nice library for C it actually sounds like a better solution.

Also not to mention other problems, in shell you actually need at least one fork to do anything which isn't ideal when you are stress testing memory, which means that the fork() may fail. While in C you just call a syscall which is less likely to fail or be stalled due to the load...

Signature · 5 lines
Cyril Hrubis
[email protected]

-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
#13 lore raw

Re: [LTP] [PATCH 1/4] controllers/memcg: update stress test to work under cgroup2

In reply to Cyril Hrubis

Hello,

Quoted 12 lines · Cyril Hrubis

Hi!

Quoted 11 lines

Yes, we would have to remember what we mounted. I think the part I am most curious about is how we would generate that state i.e what we mounted, because the Cgroup library does not expose any of this as far as I'm aware.

If we want to use the tst_cgroup C lib to cleanup as well we would have to find a way to reintroduce test state to the lib that we are losing between calls of the utility, which the only way I could think of is introducing a way to export and import test state within the lib. e.g. tst_cgroup_print_test_state() tst_cgroup_load_test_state(), which doesn't feel good as it exposes some of the nice API you have going on. This is the

struct tst_cgroup_opts and tst_cgroup_print_config could be extended. These are presently only used for debugging IIRC.

BTW don't worry about the purity of API too much or how it's implemented to begin with. The current API is nice, but that is only after a lengthy review process.

Quoted 6 lines
Quoted 6 lines

easiest way to tell if we are mounting things because we can just print what we mounted, what the test dir of the test is, and reload that state. This could have further applications to not just this scenario but also to scenarios where if a test dies its state can be reloaded, etc, almost in a checkpoint way. Not saying its common but adds some flexibility to the API and I could see it having applications outside of this utility.

Perhaps, but if a test dies then we usually assume a kernel bug happened and abort. In the future we want tests to be able to restart the system, but we don't have concrete information about such scenarios.

Quoted 7 lines
Quoted 7 lines

Alternatively we could inspect what we created and generate state that way, i.e. make a call to tst_cgroup_require() and see if new things were mounted. Then we would have to manually be freeing things. I don't like this approach because it goes against the whole point of this which was code reuse. But the cleanup of things isnt the most difficult part so it wouldn't be the biggest deal to redo the logic.

We have to save the PID (or some kind of test ID) that is used to identify the CGroup. So may as well save what CGroups we created as well.

Quoted 11 lines
Quoted 6 lines

Yes, sounds good.

Quoted 2 lines

Let me know what you think. I wouldn't want to add anything huge to the API without your blessing :)

Wouldn't it be easier to rewrite these test to the C then? I think that error handling in shell CGroup tests would always be more difficuilt than in C and given that we have a nice library for C it actually sounds like a better solution.

Yeah, I would prefer C in general. I'm not sure what is the path of least resistance though.

Signature · 5 lines
Thank you,
Richard.

-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
#14 lore raw

PATCH 2/4 controllers/memcg: Skip functional tests when mounted under cgroup2 hierarchy

In reply to Luke Nowakowski-Krijger

Skip all of the tests when the memory controller is mounted under the cgroup2 hierarchy due to the tests mainly testing features related to the cgroup1 memory controller interface.

---
 .../kernel/controllers/memcg/functional/memcg_lib.sh      | 8 ++++++++
 1 file changed, 8 insertions(+)
testcases/kernel/controllers/memcg/functional/memcg_lib.sh +8 -0
index 1b76b6597..38e494ea6 100755
@@ -104,6 +104,14 @@ memcg_setup()
 		tst_brk TCONF "Either kernel does not support Memory Resource Controller or feature not enabled"
 	fi
 
+	# Most of the tests here are testing specific parts of the cgroup v1 memory interface that is
+	# not present for cgroup2, so if it is already mounted on a cgroup2 hierarchy we should skip
+	# the test.
+	cgroup_mount_point=$(grep -w cgroup2 /proc/mounts | cut -f 2 | cut -d " " -f2)
+	if grep -q memory "$cgroup_mount_point"/cgroup.controllers; then
+		tst_brk TCONF "memory controller mounted on cgroup v2 hierarchy, skipping test."
+	fi
+
 	ROD mkdir /dev/memcg
 	ROD mount -t cgroup -omemory memcg /dev/memcg
 
Signature · 5 lines
2.32.0


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
#15 lore raw

PATCH 3/4 controllers/cgroup_fj: Update cgroup_fj_common to work under cgroup2

In reply to Luke Nowakowski-Krijger

Check under which hierachy a controller is mounted on and set the appropriate paths and variables to be able to access it.

---
 .../controllers/cgroup_fj/cgroup_fj_common.sh | 73 +++++++++++++------
 1 file changed, 50 insertions(+), 23 deletions(-)
testcases/kernel/controllers/cgroup_fj/cgroup_fj_common.sh +50 -23
index 53ab637e8..6c7cb1fa5 100755
@@ -48,11 +48,11 @@ attach_and_check()
 
     tst_resm TINFO "Attaching task $pid to $path"
 
-    ROD echo "$pid" \> "$path/tasks"
+    ROD echo "$pid" \> "$path/$task_list"
 
-    for task in $(cat "$path/tasks"); do
+    for task in $(cat "$path/$task_list"); do
         if [ "$task" -ne "$pid" ]; then
-            tst_resm TINFO "Unexpected pid $task in $path/tasks, expected $pid"
+            tst_resm TINFO "Unexpected pid $task in $path/$task_list, expected $pid"
             return 1
         fi
     done
@@ -68,7 +68,9 @@ create_subgroup()
 
     # cpuset.cpus and cpuset.mems must be initialized with suitable value
     # before any pids are attached
-    if [ "$subsystem" = "cpuset" ]; then
+    # Only needs to be done for cgroup v1 as sets are inherited from parents
+    # by default in cgroup v2
+    if [ "$cgroup_v" = "v1" ] && [ "$subsystem" = "cpuset" ]; then
         if [ -e "$mount_point/cpus" ]; then
             ROD cat "$mount_point/cpus" \> "$path/cpus"
             ROD cat "$mount_point/mems" \> "$path/mems"
@@ -79,6 +81,39 @@ create_subgroup()
     fi
 }
 
+setup_cgroupv2()
+{
+    mount_point=$(grep -w cgroup2 /proc/mounts | cut -f 2 | cut -d " " -f2)
+    if ! grep -q "$subsystem" "$mount_point"/cgroup.controllers; then
+        tst_resm TINFO "$subsystem controller not supported on cgroup v2."
+        return
+    fi
+
+    task_list="cgroup.procs"
+    cgroup_v="v2"
+}
+
+setup_cgroupv1()
+{
+    exist=$(grep -w "$subsystem" /proc/cgroups | cut -f1);
+    if [ "$exist" = "" ]; then
+        tst_brk TCONF NULL "$subsystem controller not supported"
+    fi
+
+    mount_point=$(grep -w "$subsystem" /proc/mounts | cut -f 2 | cut -d " " -f2)
+    if [ "$mount_point" = "" ]; then
+        cgroup_mounted=0
+	mount_point="/dev/cgroup"
+    fi
+
+    if [ "$cgroup_mounted" -eq "0" ]; then
+	ROD mkdir -p $mount_point
+	ROD mount -t cgroup -o "$subsystem" none $mount_point
+    fi
+
+    task_list="tasks"
+    cgroup_v="v1"
+}
 
 setup()
 {
@@ -94,18 +129,17 @@ setup()
     tst_tmpdir
     TST_CLEANUP=cleanup
 
-    mount_point=`grep -w $subsystem /proc/mounts | grep -w "cgroup" | \
-	cut -f 2 | cut -d " " -f2`
-
-    if [ -z "$mount_point" ]; then
-        try_umount=1
-        mount_point="/dev/cgroup"
-	tst_resm TINFO "Subsystem $subsystem is not mounted, mounting it at $mount_point"
-        ROD mkdir $mount_point
-        ROD mount -t cgroup -o "$subsystem" "ltp_cgroup" "$mount_point"
-    else
-	tst_resm TINFO "Subsystem $subsystem is mounted at $mount_point"
+    cgroup_mounted=1
+
+    if grep -q cgroup2 /proc/mounts; then
+        setup_cgroupv2
+    fi
+
+    if [ -z "$cgroup_v" ]; then
+        setup_cgroupv1
     fi
+
+    tst_resm TINFO "test starts with cgroup $cgroup_v"
 }
 
 cleanup()
@@ -118,15 +152,8 @@ cleanup()
 
     find "$mount_point/ltp/" -depth -type d -exec rmdir '{}' \;
 
-    if [ -z "$try_umount" ]; then
-	return
-    fi
-
-    if grep -q "$mount_point" /proc/mounts; then
+    if [ "$cgroup_mounted" -eq "0" ]; then
         EXPECT_PASS umount "$mount_point"
-    fi
-
-    if [ -e "$mount_point" ]; then
         EXPECT_PASS rmdir "$mount_point"
     fi
 }
Signature · 5 lines
2.32.0


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
#16 lore raw

PATCH 4/4 controllers/cgroup_fj: Update cgroup_fj_function to work under cgroup2

In reply to Luke Nowakowski-Krijger

Most of the tests still make sense when a controller is mounted under cgroup2 hierarchy, but some are skipped as they don't make sense and would fail.

---
 .../cgroup_fj/cgroup_fj_function.sh           | 38 +++++++++++++++++--
 1 file changed, 34 insertions(+), 4 deletions(-)
testcases/kernel/controllers/cgroup_fj/cgroup_fj_function.sh +34 -4
index fc3ad1b63..7bff480df 100755
@@ -62,6 +62,12 @@ test1()
 # Group can be renamed with mv
 test2()
 {
+    # mv'ing cgroups is not available in cgroup2
+    if [ "$cgroup_v" = "v2" ]; then
+        tst_resm TCONF "Controller mounted on cgroup2 hierachy, skipping test"
+        return
+    fi
+
     create_subgroup "$start_path/ltp_2"
 
     if ! mv "$start_path/ltp_2" "$start_path/ltp_3"; then
@@ -81,6 +87,12 @@ test2()
 # Group can be renamed with mv unless the target name exists
 test3()
 {
+    # mv'ing cgroups is not available in cgroup2
+    if [ "$cgroup_v" = "v2" ]; then
+        tst_resm TCONF "Controller mounted on cgroup2 hierachy, skipping test"
+        return
+    fi
+
     create_subgroup "$start_path/ltp_2"
 
     if mv "$start_path/ltp_2" "$start_path/ltp_1" > /dev/null 2>&1; then
@@ -124,12 +136,18 @@ test5()
 
     ROD rmdir "$start_path/ltp_1/a"
 
-    ROD echo "$pid" \> "$start_path/tasks"
+    ROD echo "$pid" \> "$start_path/$task_list"
 }
 
 # Group cannot be moved outside of hierarchy
 test6()
 {
+    # mv'ing cgroups is not available in cgroup2
+    if [ "$cgroup_v" = "v2" ]; then
+        tst_resm TCONF "Controller mounted on cgroup2 hierachy, skipping test"
+        return
+    fi
+
     if mv "$start_path/ltp_1" "$PWD/ltp" > /dev/null 2>&1; then
         tst_resm TFAIL "Subgroup $start_path/ltp_1 outside hierarchy to $PWD/ltp"
         return
@@ -141,17 +159,23 @@ test6()
 # Tasks file cannot be removed
 test7()
 {
-    if rm "$start_path/ltp_1/tasks" > /dev/null 2>&1; then
-        tst_resm TFAIL "Tasks file $start_path/ltp_1/tasks could be removed"
+    if rm "$start_path/ltp_1/$task_list" > /dev/null 2>&1; then
+        tst_resm TFAIL "Tasks file $start_path/ltp_1/$task_list could be removed"
         return
     fi
 
-    tst_resm TPASS "Tasks file $start_path/ltp_1/tasks cannot be removed"
+    tst_resm TPASS "Tasks file $start_path/ltp_1/$task_list cannot be removed"
 }
 
 # Test notify_on_release with invalid inputs
 test8()
 {
+    # notify_on_release is not available in cgroup2 so skip the test
+    if [ "$cgroup_v" = "v2" ]; then
+        tst_resm TCONF "Controller mounted on cgroup2 hierachy, skipping test"
+        return
+    fi
+
     if echo "-1" > "$start_path/ltp_1/notify_on_release" 2>/dev/null; then
         tst_resm TFAIL "Can write -1 to $start_path/ltp_1/notify_on_release"
         return
@@ -168,6 +192,12 @@ test8()
 # Test that notify_on_release can be changed
 test9()
 {
+    # notify_on_release is not available in cgroup2 so skip the test
+    if [ "$cgroup_v" = "v2" ]; then
+        tst_resm TCONF "Controller mounted on cgroup2 hierachy, skipping test"
+        return
+    fi
+
     local notify=$(ROD cat "$start_path/ltp_1/notify_on_release")
     local value
 
Signature · 5 lines
2.32.0


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp