Add SGX selftest `augment_via_eaccept_long`
PATCH 1/1 Add SGX selftest `augment_via_eaccept_long`
From: Vijay Dhanraj <[email protected]>
This commit adds a new test case which is same as augment_via_eaccept but adds more number of EPC pages to stress test EAUG via EACCEPT.
--- tools/testing/selftests/sgx/load.c | 5 +- tools/testing/selftests/sgx/main.c | 120 +++++++++++++++++++++++- tools/testing/selftests/sgx/main.h | 3 +- tools/testing/selftests/sgx/sigstruct.c | 2 +- 4 files changed, 125 insertions(+), 5 deletions(-)
tools/testing/selftests/sgx/load.c +3 -2
@@ -171,7 +171,8 @@ uint64_t encl_get_entry(struct encl *encl, const char *symbol)return 0 ; } -bool encl_load (const char *path,struct encl *encl,unsigned long heap_size) +bool encl_load (const char *path,struct encl *encl,unsigned long heap_size, +unsigned long edmm_size) {const char device_path[] ="/dev/sgx_enclave" ;struct encl_segment *seg;
@@ -300,7 +301,7 @@ bool encl_load(const char *path, struct encl *encl, unsigned long heap_size) encl->src_size = encl->segment_tbl[j].offset + encl->segment_tbl[j].size; -for (encl->encl_size =4096 ; encl->encl_size < encl->src_size; ) +for (encl->encl_size =4096 ; encl->encl_size < encl->src_size + edmm_size;) encl->encl_size <<=1 ;return true;
tools/testing/selftests/sgx/main.c +119 -1
138 lines not on this page · raw message
tools/testing/selftests/sgx/main.h +2 -1
@@ -35,7 +35,8 @@ extern unsigned char sign_key[];extern unsigned char sign_key_end[];void encl_delete (struct encl *ctx); -bool encl_load (const char *path,struct encl *encl,unsigned long heap_size); +bool encl_load (const char *path,struct encl *encl,unsigned long heap_size, +unsigned long edmm_size);bool encl_measure (struct encl *encl);bool encl_build (struct encl *encl);uint64_t encl_get_entry (struct encl *encl,const char *symbol);
tools/testing/selftests/sgx/sigstruct.c +1 -1
@@ -343,7 +343,7 @@ bool encl_measure(struct encl *encl)if (!ctx) goto err; -if (!mrenclave_ecreate (ctx, encl->src_size)) +if (!mrenclave_ecreate (ctx, encl->encl_size)) goto err;for (i =0 ; i < encl->nr_segments; i++) {
Signature · 1 line
2.17.1
Jarkko Sakkinen
maintainerQuoted 7 lines · On Thu, Aug 04, 2022 at 01:14:56PM -0700, [email protected] wrote:
From: Vijay Dhanraj <[email protected]>
This commit adds a new test case which is same as
augment_via_eacceptbut adds more number of EPC pages to stress testEAUGviaEACCEPT.Signed-off-by Vijay Dhanraj <[email protected]>Signed-off-by Haitao Huang <[email protected]>
Thank you. I'll run this with Icelake system.
Quoted 205 lines
--- tools/testing/selftests/sgx/load.c | 5 +- tools/testing/selftests/sgx/main.c | 120 +++++++++++++++++++++++- tools/testing/selftests/sgx/main.h | 3 +- tools/testing/selftests/sgx/sigstruct.c | 2 +- 4 files changed, 125 insertions(+), 5 deletions(-)tools/testing/selftests/sgx/load.c +3
-2@@ -171,7 +171,8 @@ uint64_t encl_get_entry(struct encl *encl, const char *symbol)return 0 ; } -bool encl_load (const char *path,struct encl *encl,unsigned long heap_size) +bool encl_load (const char *path,struct encl *encl,unsigned long heap_size, +unsigned long edmm_size) {const char device_path[] ="/dev/sgx_enclave" ;struct encl_segment *seg;@@ -300,7 +301,7 @@ bool encl_load(const char *path, struct encl *encl, unsigned long heap_size) encl->src_size = encl->segment_tbl[j].offset + encl->segment_tbl[j].size; -for (encl->encl_size =4096 ; encl->encl_size < encl->src_size; ) +for (encl->encl_size =4096 ; encl->encl_size < encl->src_size + edmm_size;) encl->encl_size <<=1 ;return true;tools/testing/selftests/sgx/main.c +119
-1@@ -25,6 +25,8 @@ static const uint64_t MAGIC = 0x1122334455667788ULL;static const uint64_t MAGIC2 =0x8877665544332211ULL ;vdso_sgx_enter_enclave_t vdso_sgx_enter_enclave; +static const unsigned long edmm_size =8589934592 ;//8G +/* * Security Information (SECINFO) data structure needed by a few SGX * instructions (eg. ENCLU[EACCEPT] and ENCLU[EMODPE]) holds meta-data @@ -183,7 +185,7 @@ static bool setup_test_encl(unsigned long heap_size, struct encl *encl,unsigned int i;void *addr; -if (!encl_load ("test_encl.elf" , encl, heap_size)) { +if (!encl_load ("test_encl.elf" , encl, heap_size, edmm_size)) {encl_delete (encl);TH_LOG ("Failed to load the test enclave." );return false;@@ -1210,6 +1212,122 @@ TEST_F(enclave, augment_via_eaccept)munmap (addr,PAGE_SIZE ); } +/* +* Test for the addition of large number of pages to an initialized enclave via +* a pre-emptive run of EACCEPT on page to be added. +*/ +#define TIMEOUT_LONG 900/* seconds */ +TEST_F_TIMEOUT(enclave, augment_via_eaccept_long, TIMEOUT_LONG) +{ +struct encl_op_get_from_addr get_addr_op; +struct encl_op_put_to_addr put_addr_op; +struct encl_op_eaccept eaccept_op; +size_t total_size =0 ; +void *addr; +unsigned long i; + +if (!sgx2_supported ()) +SKIP (return,"SGX2 not supported" ); + +ASSERT_TRUE (setup_test_encl (ENCL_HEAP_SIZE_DEFAULT , &self->encl, _metadata)); + +memset (&self->run,0 ,sizeof (self->run)); + self->run.tcs = self->encl.encl_base; + +for (i =0 ; i < self->encl.nr_segments; i++) { +struct encl_segment *seg = &self->encl.segment_tbl[i]; + + total_size += seg->size; +TH_LOG ("test enclave: total_size = %ld, seg->size = %ld" , total_size, seg->size); + } + +/* +* Actual enclave size is expected to be larger than the loaded +* test enclave since enclave size must be a power of 2 in bytes while +* test_encl does not consume it all. +*/ +EXPECT_LT (total_size + edmm_size, self->encl.encl_size); + +/* +* mmap() a page at end of existing enclave to be used for dynamic +* EPC page. +* +* Kernel will allow new mapping using any permissions if it +* falls into the enclave's address range but not backed +* by existing enclave pages. +*/ +TH_LOG ("mmaping pages at end of enclave..." ); + addr =mmap ((void *)self->encl.encl_base + total_size, edmm_size, +PROT_READ |PROT_WRITE |PROT_EXEC ,MAP_SHARED |MAP_FIXED , + self->encl.fd,0 ); +EXPECT_NE (addr,MAP_FAILED ); + + self->run.exception_vector =0 ; + self->run.exception_error_code =0 ; + self->run.exception_addr =0 ; + +/* +* Run EACCEPT on new page to trigger the #PF->EAUG->EACCEPT(again +* without a #PF). All should be transparent to userspace. +*/ +TH_LOG ("Entering enclave to run EACCEPT for each page of %zd bytes may take a while ..." , + edmm_size); + eaccept_op.flags =SGX_SECINFO_R |SGX_SECINFO_W |SGX_SECINFO_REG |SGX_SECINFO_PENDING ; + eaccept_op.ret =0 ; + eaccept_op.header.type =ENCL_OP_EACCEPT ; + +for (i =0 ; i < edmm_size; i +=4096 ) { + eaccept_op.epc_addr = (uint64_t )(addr + i); + +EXPECT_EQ (ENCL_CALL (&eaccept_op, &self->run, true),0 ); +if (self->run.exception_vector ==14 && + self->run.exception_error_code ==4 && + self->run.exception_addr == self->encl.encl_base) { +munmap (addr, edmm_size); +SKIP (return,"Kernel does not support adding pages to initialized enclave" ); + } + +EXPECT_EQ (self->run.exception_vector,0 ); +EXPECT_EQ (self->run.exception_error_code,0 ); +EXPECT_EQ (self->run.exception_addr,0 ); +ASSERT_EQ (eaccept_op.ret,0 ); +ASSERT_EQ (self->run.function,EEXIT ); + } + +/* +* New page should be accessible from within enclave - attempt to +* write to it. +*/ + put_addr_op.value =MAGIC ; + put_addr_op.addr = (unsigned long )addr; + put_addr_op.header.type =ENCL_OP_PUT_TO_ADDRESS ; + +EXPECT_EQ (ENCL_CALL (&put_addr_op, &self->run, true),0 ); + +EXPECT_EEXIT (&self->run); +EXPECT_EQ (self->run.exception_vector,0 ); +EXPECT_EQ (self->run.exception_error_code,0 ); +EXPECT_EQ (self->run.exception_addr,0 ); + +/* +* Read memory from newly added page that was just written to, +* confirming that data previously written (MAGIC) is present. +*/ + get_addr_op.value =0 ; + get_addr_op.addr = (unsigned long )addr; + get_addr_op.header.type =ENCL_OP_GET_FROM_ADDRESS ; + +EXPECT_EQ (ENCL_CALL (&get_addr_op, &self->run, true),0 ); + +EXPECT_EQ (get_addr_op.value,MAGIC ); +EXPECT_EEXIT (&self->run); +EXPECT_EQ (self->run.exception_vector,0 ); +EXPECT_EQ (self->run.exception_error_code,0 ); +EXPECT_EQ (self->run.exception_addr,0 ); + +munmap (addr, edmm_size); +} +/* * SGX2 page type modification test in two phases: * Phase 1: tools/testing/selftests/sgx/main.h +2
-1@@ -35,7 +35,8 @@ extern unsigned char sign_key[];extern unsigned char sign_key_end[];void encl_delete (struct encl *ctx); -bool encl_load (const char *path,struct encl *encl,unsigned long heap_size); +bool encl_load (const char *path,struct encl *encl,unsigned long heap_size, +unsigned long edmm_size);bool encl_measure (struct encl *encl);bool encl_build (struct encl *encl);uint64_t encl_get_entry (struct encl *encl,const char *symbol);tools/testing/selftests/sgx/sigstruct.c +1
-1@@ -343,7 +343,7 @@ bool encl_measure(struct encl *encl)if (!ctx) goto err; -if (!mrenclave_ecreate (ctx, encl->src_size)) +if (!mrenclave_ecreate (ctx, encl->encl_size)) goto err;for (i =0 ; i < encl->nr_segments; i++) {Signature · 1 line
2.17.1
BR, Jarkko
Jarkko Sakkinen
maintainerQuoted 7 lines · On Thu, Aug 04, 2022 at 01:14:56PM -0700, [email protected] wrote:
From: Vijay Dhanraj <[email protected]>
This commit adds a new test case which is same as
augment_via_eacceptbut adds more number of EPC pages to stress testEAUGviaEACCEPT.Signed-off-by Vijay Dhanraj <[email protected]>Signed-off-by Haitao Huang <[email protected]>
Hey, to reproduce the original issue: does it reproduce on VM or should I run baremetal kernel?
BR, Jarkko
Quoted 17 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Monday, August 8, 2022 5:18 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 4 lines · On Thu, Aug 04, 2022 at 01:14:56PM -0700, [email protected] wrote:
From: Vijay Dhanraj <[email protected]>
This commit adds a new test case which is same as
augment_via_eacceptbut adds more number of EPC pages to stress test
EAUGviaEACCEPT.Quoted 1 line
Quoted 7 lines
Quoted 2 lines
Signed-off-by Vijay Dhanraj <[email protected]>Signed-off-by Haitao Huang <[email protected]>Hey, to reproduce the original issue: does it reproduce on VM or should I run baremetal kernel?
BR, Jarkko
Hi Jarkko, The issue should be reproducible on baremetal kernel. -Vijay
Jarkko Sakkinen
maintainerQuoted 20 lines · Vijay
Quoted 20 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Monday, August 8, 2022 5:18 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 4 lines · On Thu, Aug 04, 2022 at 01:14:56PM -0700, [email protected] wrote:
From: Vijay Dhanraj <[email protected]>
This commit adds a new test case which is same as
augment_via_eacceptbut adds more number of EPC pages to stress test
EAUGviaEACCEPT.Quoted 3 lines
Signed-off-by Vijay Dhanraj <[email protected]>Signed-off-by Haitao Huang <[email protected]>
Quoted 6 lines
Quoted 4 lines
Hey, to reproduce the original issue: does it reproduce on VM or should I run baremetal kernel?
BR, Jarkko
Hi Jarkko, The issue should be reproducible on baremetal kernel.
Thanks.
BR, Jarkko
Jarkko Sakkinen
maintainerQuoted 21 lines · Jarkko Sakkinen
Quoted 20 lines · Vijay
Quoted 20 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Monday, August 8, 2022 5:18 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 4 lines · On Thu, Aug 04, 2022 at 01:14:56PM -0700, [email protected] wrote:
From: Vijay Dhanraj <[email protected]>
This commit adds a new test case which is same as
augment_via_eacceptbut adds more number of EPC pages to stress test
EAUGviaEACCEPT.Quoted 3 lines
Signed-off-by Vijay Dhanraj <[email protected]>Signed-off-by Haitao Huang <[email protected]>
Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
Hey, to reproduce the original issue: does it reproduce on VM or should I run baremetal kernel?
BR, Jarkko
Hi Jarkko, The issue should be reproducible on baremetal kernel.
Thanks.
I need comment out other tests in order to make sane out of this :-)
Mentioning this because came into realization that stress tests should be IMHO moved each to a separate binary (so that they can be run separately). Just a note (TODO) to myself.
I'll work on this today again and *possibly* split your test to its own application to get a starting point for forementioned.
BR, Jarkko
Jarkko Sakkinen
maintainerQuoted 31 lines · Jarkko Sakkinen
Quoted 21 lines · Jarkko Sakkinen
Quoted 20 lines · Vijay
Quoted 20 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Monday, August 8, 2022 5:18 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 4 lines · On Thu, Aug 04, 2022 at 01:14:56PM -0700, [email protected] wrote:
From: Vijay Dhanraj <[email protected]>
This commit adds a new test case which is same as
augment_via_eacceptbut adds more number of EPC pages to stress test
EAUGviaEACCEPT.Quoted 3 lines
Signed-off-by Vijay Dhanraj <[email protected]>Signed-off-by Haitao Huang <[email protected]>Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
Hey, to reproduce the original issue: does it reproduce on VM or should I run baremetal kernel?
BR, Jarkko
Hi Jarkko, The issue should be reproducible on baremetal kernel.
Thanks.
Quoted 8 lines
I need comment out other tests in order to make sane out of this :-)
Mentioning this because came into realization that stress tests should be IMHO moved each to a separate binary (so that they can be run separately). Just a note (TODO) to myself.
I'll work on this today again and *possibly* split your test to its own application to get a starting point for forementioned.
I got
# RUN enclave.augment_via_eaccept_long ...
# main.c:1241:augment_via_eaccept_long:test enclave: total_size = 8192, seg->size = 8192
# main.c:1241:augment_via_eaccept_long:test enclave: total_size = 12288, seg->size = 4096
# main.c:1241:augment_via_eaccept_long:test enclave: total_size = 36864, seg->size = 24576
# main.c:1241:augment_via_eaccept_long:test enclave: total_size = 40960, seg->size = 4096
# main.c:1259:augment_via_eaccept_long:mmaping pages at end of enclave...
# main.c:1273:augment_via_eaccept_long:Entering enclave to run EACCEPT for each page of 8589934592 bytes may take a while ...
# OK enclave.augment_via_eaccept_long
The CPU used for testing was according to /proc/cpuinfo:
model name : Intel(R) Xeon(R) Gold 6338 CPU @ 2.00GHz
I have couple of queries:
1. Is it possible to get dmesg output?
2. Do I have to repeat the test multiple times, or does it occur unconditionaly?
BR, Jarkko
Quoted 68 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 9:10 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 13 lines · Jarkko Sakkinen
Quoted 12 lines · Jarkko Sakkinen
Quoted 11 lines · Vijay
Quoted 11 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Monday, August 8, 2022 5:18 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longOn Thu, Aug 04, 2022 at 01:14:56PM -0700, [email protected]
Quoted 19 lines · wrote:
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 5 lines
From: Vijay Dhanraj <[email protected]>
This commit adds a new test case which is same as
augment_via_eacceptbut adds more number of EPC pages to stress test
EAUGviaEACCEPT.Quoted 3 lines
Signed-off-by Vijay Dhanraj <[email protected]>Signed-off-by Haitao Huang <[email protected]>Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
Hey, to reproduce the original issue: does it reproduce on VM or should I run baremetal kernel?
BR, Jarkko
Hi Jarkko, The issue should be reproducible on baremetal kernel.
Thanks.
Quoted 8 lines
I need comment out other tests in order to make sane out of this :-)
Mentioning this because came into realization that stress tests should be IMHO moved each to a separate binary (so that they can be run separately). Just a note (TODO) to myself.
I'll work on this today again and *possibly* split your test to its own application to get a starting point for forementioned.
I got
# RUN enclave.augment_via_eaccept_long ...
# main.c:1241:augment_via_eaccept_long:test enclave:total_size= 8192,seg->size= 8192 # main.c:1241:augment_via_eaccept_long:test enclave:total_size= 12288,seg->size= 4096 # main.c:1241:augment_via_eaccept_long:test enclave:total_size= 36864,seg->size= 24576 # main.c:1241:augment_via_eaccept_long:test enclave:total_size= 40960,seg->size= 4096 # main.c:1259:augment_via_eaccept_long:mmaping pages at end of enclave... # main.c:1273:augment_via_eaccept_long:Entering enclave to run EACCEPT for each page of 8589934592 bytes may take a while ...
# OK enclave.augment_via_eaccept_long
Quoted 7 lines
The CPU used for testing was according to
/proc/cpuinfo:model name : Intel(R) Xeon(R) Gold 6338 CPU @ 2.00GHz
I have couple of queries:
1. Is it possible to get dmesg output?
I did check the dmesg output but couldn't find anything related to the failure. Just the general log messages.
Quoted 3 lines
2. Do I have to repeat the test multiple times, or does it occur unconditionaly?
I was able to repro every time but it was a bit sporadic for Haitao.
Quoted 1 line
BR, Jarkko
Also, did you set the PRMRR size to 2GB per socket in the BIOS? The issue is only reproduced for oversubscribed scenario. When I set my PRMRR to 64GB per socket, I wasn't able to repro the issue.
Regards, Vijay
Jarkko Sakkinen
maintainerQuoted 80 lines · Vijay
Quoted 68 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 9:10 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 13 lines · Jarkko Sakkinen
Quoted 12 lines · Jarkko Sakkinen
Quoted 11 lines · Vijay
Quoted 11 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Monday, August 8, 2022 5:18 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longOn Thu, Aug 04, 2022 at 01:14:56PM -0700, [email protected]
Quoted 19 lines · wrote:
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 5 lines
From: Vijay Dhanraj <[email protected]>
This commit adds a new test case which is same as
augment_via_eacceptbut adds more number of EPC pages to stress test
EAUGviaEACCEPT.Quoted 3 lines
Signed-off-by Vijay Dhanraj <[email protected]>Signed-off-by Haitao Huang <[email protected]>Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
Hey, to reproduce the original issue: does it reproduce on VM or should I run baremetal kernel?
BR, Jarkko
Hi Jarkko, The issue should be reproducible on baremetal kernel.
Thanks.
Quoted 8 lines
I need comment out other tests in order to make sane out of this :-)
Mentioning this because came into realization that stress tests should be IMHO moved each to a separate binary (so that they can be run separately). Just a note (TODO) to myself.
I'll work on this today again and *possibly* split your test to its own application to get a starting point for forementioned.
I got
# RUN enclave.augment_via_eaccept_long ...
# main.c:1241:augment_via_eaccept_long:test enclave:total_size= 8192,seg->size= 8192 # main.c:1241:augment_via_eaccept_long:test enclave:total_size= 12288,seg->size= 4096 # main.c:1241:augment_via_eaccept_long:test enclave:total_size= 36864,seg->size= 24576 # main.c:1241:augment_via_eaccept_long:test enclave:total_size= 40960,seg->size= 4096 # main.c:1259:augment_via_eaccept_long:mmaping pages at end of enclave... # main.c:1273:augment_via_eaccept_long:Entering enclave to run EACCEPT for each page of 8589934592 bytes may take a while ...
# OK enclave.augment_via_eaccept_longQuoted 7 lines
The CPU used for testing was according to
/proc/cpuinfo:model name : Intel(R) Xeon(R) Gold 6338 CPU @ 2.00GHz
I have couple of queries:
1. Is it possible to get dmesg output?
I did check the dmesg output but couldn't find anything related to the failure. Just the general log messages.
Quoted 3 lines
2. Do I have to repeat the test multiple times, or does it occur unconditionaly?
Quoted 7 lines
I was able to repro every time but it was a bit sporadic for Haitao.
Quoted 1 line
BR, Jarkko
Also, did you set the PRMRR size to 2GB per socket in the BIOS? The issue is only reproduced for oversubscribed scenario. When I set my PRMRR to 64GB per socket, I wasn't able to repro the issue.
I need to revisit this.
Can you simply run test_sgx with gdb and see where it hits? HOST_CFLAGS has apparently "-g" already.
Quoted 1 line
Regards, Vijay
BR, Jarkko
Jarkko Sakkinen
maintainerQuoted 85 lines · Jarkko Sakkinen
Quoted 84 lines · Vijay
Quoted 68 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 9:10 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 13 lines · Jarkko Sakkinen
Quoted 12 lines · Jarkko Sakkinen
Quoted 11 lines · Vijay
Quoted 11 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Monday, August 8, 2022 5:18 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longOn Thu, Aug 04, 2022 at 01:14:56PM -0700, [email protected]
Quoted 19 lines · wrote:
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 5 lines
From: Vijay Dhanraj <[email protected]>
This commit adds a new test case which is same as
augment_via_eacceptbut adds more number of EPC pages to stress test
EAUGviaEACCEPT.Quoted 3 lines
Signed-off-by Vijay Dhanraj <[email protected]>Signed-off-by Haitao Huang <[email protected]>Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
Hey, to reproduce the original issue: does it reproduce on VM or should I run baremetal kernel?
BR, Jarkko
Hi Jarkko, The issue should be reproducible on baremetal kernel.
Thanks.
Quoted 8 lines
I need comment out other tests in order to make sane out of this :-)
Mentioning this because came into realization that stress tests should be IMHO moved each to a separate binary (so that they can be run separately). Just a note (TODO) to myself.
I'll work on this today again and *possibly* split your test to its own application to get a starting point for forementioned.
I got
# RUN enclave.augment_via_eaccept_long ...
# main.c:1241:augment_via_eaccept_long:test enclave:total_size= 8192,seg->size= 8192 # main.c:1241:augment_via_eaccept_long:test enclave:total_size= 12288,seg->size= 4096 # main.c:1241:augment_via_eaccept_long:test enclave:total_size= 36864,seg->size= 24576 # main.c:1241:augment_via_eaccept_long:test enclave:total_size= 40960,seg->size= 4096 # main.c:1259:augment_via_eaccept_long:mmaping pages at end of enclave... # main.c:1273:augment_via_eaccept_long:Entering enclave to run EACCEPT for each page of 8589934592 bytes may take a while ...
# OK enclave.augment_via_eaccept_longQuoted 7 lines
The CPU used for testing was according to
/proc/cpuinfo:model name : Intel(R) Xeon(R) Gold 6338 CPU @ 2.00GHz
I have couple of queries:
1. Is it possible to get dmesg output?
I did check the dmesg output but couldn't find anything related to the failure. Just the general log messages.
Quoted 3 lines
2. Do I have to repeat the test multiple times, or does it occur unconditionaly?
I was able to repro every time but it was a bit sporadic for Haitao.
Quoted 1 line
BR, Jarkko
Quoted 8 lines
Quoted 3 lines
Also, did you set the PRMRR size to 2GB per socket in the BIOS? The issue is only reproduced for oversubscribed scenario. When I set my PRMRR to 64GB per socket, I wasn't able to repro the issue.
I need to revisit this.
Can you simply run
test_sgxwith gdb and see where it hits?HOST_CFLAGShas apparently "-g" already.
I'll check the socket configuration but since you can steadily reproduce the possible bug, this would be useful information to dig out.
BR, Jarkko
Quoted 107 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 11:53 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 59 lines · Vijay
Quoted 59 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 9:10 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 15 lines · Jarkko Sakkinen
Quoted 14 lines · Jarkko Sakkinen
Quoted 13 lines · Vijay
Quoted 13 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Monday, August 8, 2022 5:18 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longOn Thu, Aug 04, 2022 at 01:14:56PM -0700,
[email protected]Quoted 22 lines · wrote:
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 5 lines
From: Vijay Dhanraj <[email protected]>
This commit adds a new test case which is same as
augment_via_eacceptbut adds more number of EPC pages to stress test
EAUGviaEACCEPT.Quoted 3 lines
Signed-off-by Vijay Dhanraj <[email protected]>Signed-off-by Haitao Huang <[email protected]>Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
Hey, to reproduce the original issue: does it reproduce on VM or should I run baremetal kernel?
BR, Jarkko
Hi Jarkko, The issue should be reproducible on baremetal kernel.
Thanks.
I need comment out other tests in order to make sane out of this :-)
Quoted 6 lines
Mentioning this because came into realization that stress tests should be IMHO moved each to a separate binary (so that they can be run separately). Just a note (TODO) to myself.
I'll work on this today again and *possibly* split your test to its own application to get a starting point for forementioned.
I got
Quoted 10 lines
Quoted 10 lines
# RUN enclave.augment_via_eaccept_long ...
# main.c:1241:augment_via_eaccept_long:test enclave:total_size= 8192,seg->size= 8192 # main.c:1241:augment_via_eaccept_long:test enclave:total_size= 12288,seg->size= 4096 # main.c:1241:augment_via_eaccept_long:test enclave:total_size= 36864,seg->size= 24576 # main.c:1241:augment_via_eaccept_long:test enclave:total_size= 40960,seg->size= 4096 # main.c:1259:augment_via_eaccept_long:mmaping pages at end ofenclave...
Quoted 4 lines
Quoted 4 lines
# main.c:1273:augment_via_eaccept_long:Entering enclave to run EACCEPT for each page of 8589934592 bytes may take a while ...
# OK enclave.augment_via_eaccept_longQuoted 8 lines
Quoted 7 lines
The CPU used for testing was according to
/proc/cpuinfo:model name : Intel(R) Xeon(R) Gold 6338 CPU @ 2.00GHz
I have couple of queries:
1. Is it possible to get dmesg output?
I did check the dmesg output but couldn't find anything related to the
failure. Just the general log messages.
Quoted 11 lines
Quoted 4 lines
2. Do I have to repeat the test multiple times, or does it occur unconditionaly?
I was able to repro every time but it was a bit sporadic for Haitao.
Quoted 1 line
BR, Jarkko
Also, did you set the PRMRR size to 2GB per socket in the BIOS? The issue is only reproduced for oversubscribed scenario. When I set my PRMRR to 64GB per socket, I wasn't able to repro the issue.
Quoted 8 lines
I need to revisit this.
Can you simply run
test_sgxwith gdb and see where it hits?HOST_CFLAGShas apparently "-g" already.Quoted 1 line
Regards, Vijay
BR, Jarkko
I am able to repro the issue when I reduce the PRMRR to 2B/socket but not but not able to break on the assertion failure with gdb. I also enabled debug attribute in the secs but still no avail. Anything I am missing here?
tools/testing/selftests/sgx/load.c +1 -1
@@ -87,7 +87,7 @@ static bool encl_ioc_create(struct encl *encl)memset (secs,0 ,sizeof (*secs)); secs->ssa_frame_size =1 ; - secs->attributes =SGX_ATTR_MODE64BIT ; + secs->attributes =SGX_ATTR_MODE64BIT |SGX_ATTR_DEBUG ; secs->xfrm =3 ; secs->base = encl->encl_base; secs->size = encl->encl_size;
Regards, Vijay
Jarkko Sakkinen
maintainerQuoted 132 lines · Vijay
Quoted 107 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 11:53 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 59 lines · Vijay
Quoted 59 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 9:10 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 15 lines · Jarkko Sakkinen
Quoted 14 lines · Jarkko Sakkinen
Quoted 13 lines · Vijay
Quoted 13 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Monday, August 8, 2022 5:18 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longOn Thu, Aug 04, 2022 at 01:14:56PM -0700,
[email protected]Quoted 22 lines · wrote:
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 5 lines
From: Vijay Dhanraj <[email protected]>
This commit adds a new test case which is same as
augment_via_eacceptbut adds more number of EPC pages to stress test
EAUGviaEACCEPT.Quoted 3 lines
Signed-off-by Vijay Dhanraj <[email protected]>Signed-off-by Haitao Huang <[email protected]>Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
Hey, to reproduce the original issue: does it reproduce on VM or should I run baremetal kernel?
BR, Jarkko
Hi Jarkko, The issue should be reproducible on baremetal kernel.
Thanks.
I need comment out other tests in order to make sane out of this :-)
Quoted 6 lines
Mentioning this because came into realization that stress tests should be IMHO moved each to a separate binary (so that they can be run separately). Just a note (TODO) to myself.
I'll work on this today again and *possibly* split your test to its own application to get a starting point for forementioned.
I got
Quoted 10 lines
Quoted 10 lines
# RUN enclave.augment_via_eaccept_long ...
# main.c:1241:augment_via_eaccept_long:test enclave:total_size= 8192,seg->size= 8192 # main.c:1241:augment_via_eaccept_long:test enclave:total_size= 12288,seg->size= 4096 # main.c:1241:augment_via_eaccept_long:test enclave:total_size= 36864,seg->size= 24576 # main.c:1241:augment_via_eaccept_long:test enclave:total_size= 40960,seg->size= 4096 # main.c:1259:augment_via_eaccept_long:mmaping pages at end ofenclave...
Quoted 4 lines
Quoted 4 lines
# main.c:1273:augment_via_eaccept_long:Entering enclave to run EACCEPT for each page of 8589934592 bytes may take a while ...
# OK enclave.augment_via_eaccept_longQuoted 8 lines
Quoted 7 lines
The CPU used for testing was according to
/proc/cpuinfo:model name : Intel(R) Xeon(R) Gold 6338 CPU @ 2.00GHz
I have couple of queries:
1. Is it possible to get dmesg output?
I did check the dmesg output but couldn't find anything related to the
failure. Just the general log messages.
Quoted 11 lines
Quoted 4 lines
2. Do I have to repeat the test multiple times, or does it occur unconditionaly?
I was able to repro every time but it was a bit sporadic for Haitao.
Quoted 1 line
BR, Jarkko
Also, did you set the PRMRR size to 2GB per socket in the BIOS? The issue is only reproduced for oversubscribed scenario. When I set my PRMRR to 64GB per socket, I wasn't able to repro the issue.
Quoted 8 lines
I need to revisit this.
Can you simply run
test_sgxwith gdb and see where it hits?HOST_CFLAGShas apparently "-g" already.Quoted 1 line
Regards, Vijay
BR, Jarkko
I am able to repro the issue when I reduce the PRMRR to 2B/socket but not but not able to break on the assertion failure with gdb. I also enabled debug attribute in the secs but still no avail. Anything I am missing here?
tools/testing/selftests/sgx/load.c +1
-1@@ -87,7 +87,7 @@ static bool encl_ioc_create(struct encl *encl)memset (secs,0 ,sizeof (*secs)); secs->ssa_frame_size =1 ; - secs->attributes =SGX_ATTR_MODE64BIT ; + secs->attributes =SGX_ATTR_MODE64BIT |SGX_ATTR_DEBUG ; secs->xfrm =3 ; secs->base = encl->encl_base; secs->size = encl->encl_size;
Quoted 1 line
Regards, Vijay
I get also full pass with 2GB configuration (and also observed that kselftest runs much faster with this configuration).
But I looked at sgx_alloc_epc_page() and saw this:
if (list_empty(&sgx_active_page_list))
return ERR_PTR(-ENOMEM);
if (!reclaim) {
page = ERR_PTR(-EBUSY);
break;
}In sgx_vma_fault(), when running completely out of reclaimable pages, this causes VM_FAULT_SIGBUS returned instead of VM_FAULT_NOPAGE:
entry = sgx_encl_load_page(encl, addr, vma->vm_flags);
if (IS_ERR(entry)) {
mutex_unlock(&encl->lock);
if (PTR_ERR(entry) == -EBUSY)
return VM_FAULT_NOPAGE;
return VM_FAULT_SIGBUS;
}Not sure if those should be re-ordered that would keep the process stuck up until there is something to reclaim. Now we use NOPAGE to address situation when there is actually something to reclaim but because of locking side of things we pass reclaim=false to sgx_alloc_epc_page().
So this is kind of OOM behaviour how it works now instead of stalling processes.
BR, Jarkko
Jarkko Sakkinen
maintainerQuoted 161 lines · Jarkko Sakkinen
Quoted 132 lines · Vijay
Quoted 107 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 11:53 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 59 lines · Vijay
Quoted 59 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 9:10 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 15 lines · Jarkko Sakkinen
Quoted 14 lines · Jarkko Sakkinen
Quoted 13 lines · Vijay
Quoted 13 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Monday, August 8, 2022 5:18 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longOn Thu, Aug 04, 2022 at 01:14:56PM -0700,
[email protected]Quoted 22 lines · wrote:
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 5 lines
From: Vijay Dhanraj <[email protected]>
This commit adds a new test case which is same as
augment_via_eacceptbut adds more number of EPC pages to stress test
EAUGviaEACCEPT.Quoted 3 lines
Signed-off-by Vijay Dhanraj <[email protected]>Signed-off-by Haitao Huang <[email protected]>Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
Hey, to reproduce the original issue: does it reproduce on VM or should I run baremetal kernel?
BR, Jarkko
Hi Jarkko, The issue should be reproducible on baremetal kernel.
Thanks.
I need comment out other tests in order to make sane out of this :-)
Quoted 6 lines
Mentioning this because came into realization that stress tests should be IMHO moved each to a separate binary (so that they can be run separately). Just a note (TODO) to myself.
I'll work on this today again and *possibly* split your test to its own application to get a starting point for forementioned.
I got
Quoted 10 lines
Quoted 10 lines
# RUN enclave.augment_via_eaccept_long ...
# main.c:1241:augment_via_eaccept_long:test enclave:total_size= 8192,seg->size= 8192 # main.c:1241:augment_via_eaccept_long:test enclave:total_size= 12288,seg->size= 4096 # main.c:1241:augment_via_eaccept_long:test enclave:total_size= 36864,seg->size= 24576 # main.c:1241:augment_via_eaccept_long:test enclave:total_size= 40960,seg->size= 4096 # main.c:1259:augment_via_eaccept_long:mmaping pages at end ofenclave...
Quoted 4 lines
Quoted 4 lines
# main.c:1273:augment_via_eaccept_long:Entering enclave to run EACCEPT for each page of 8589934592 bytes may take a while ...
# OK enclave.augment_via_eaccept_longQuoted 8 lines
Quoted 7 lines
The CPU used for testing was according to
/proc/cpuinfo:model name : Intel(R) Xeon(R) Gold 6338 CPU @ 2.00GHz
I have couple of queries:
1. Is it possible to get dmesg output?
I did check the dmesg output but couldn't find anything related to the
failure. Just the general log messages.
Quoted 11 lines
Quoted 4 lines
2. Do I have to repeat the test multiple times, or does it occur unconditionaly?
I was able to repro every time but it was a bit sporadic for Haitao.
Quoted 1 line
BR, Jarkko
Also, did you set the PRMRR size to 2GB per socket in the BIOS? The issue is only reproduced for oversubscribed scenario. When I set my PRMRR to 64GB per socket, I wasn't able to repro the issue.
Quoted 8 lines
I need to revisit this.
Can you simply run
test_sgxwith gdb and see where it hits?HOST_CFLAGShas apparently "-g" already.Quoted 1 line
Regards, Vijay
BR, Jarkko
I am able to repro the issue when I reduce the PRMRR to 2B/socket but not but not able to break on the assertion failure with gdb. I also enabled debug attribute in the secs but still no avail. Anything I am missing here?
tools/testing/selftests/sgx/load.c +1
-1@@ -87,7 +87,7 @@ static bool encl_ioc_create(struct encl *encl)memset (secs,0 ,sizeof (*secs)); secs->ssa_frame_size =1 ; - secs->attributes =SGX_ATTR_MODE64BIT ; + secs->attributes =SGX_ATTR_MODE64BIT |SGX_ATTR_DEBUG ; secs->xfrm =3 ; secs->base = encl->encl_base; secs->size = encl->encl_size;Quoted 1 line
Regards, Vijay
I get also full pass with 2GB configuration (and also observed that kselftest runs much faster with this configuration).
But I looked at
sgx_alloc_epc_page()and saw this:if (list_empty(&sgx_active_page_list)) return ERR_PTR(-ENOMEM); if (!reclaim) { page = ERR_PTR(-EBUSY); break; }In
sgx_vma_fault(), when running completely out of reclaimable pages, this causesVM_FAULT_SIGBUSreturned instead ofVM_FAULT_NOPAGE:entry = sgx_encl_load_page(encl, addr, vma->vm_flags); if (IS_ERR(entry)) { mutex_unlock(&encl->lock); if (PTR_ERR(entry) == -EBUSY) return VM_FAULT_NOPAGE; return VM_FAULT_SIGBUS; }
Quoted 7 lines
Not sure if those should be re-ordered that would keep the process stuck up until there is something to reclaim. Now we use NOPAGE to address situation when there is actually something to reclaim but because of locking side of things we pass reclaim=false to
sgx_alloc_epc_page().So this is kind of OOM behaviour how it works now instead of stalling processes.
Right, I looked at the original email at was really a page fault that was catched. The above theory cannot possibly hold, as the process does not exit with a bus error.
I looked next to sgx_encl_eaug_page(), and found this:
encl_page = sgx_encl_page_alloc(encl, addr - encl->base, secinfo_flags);
if (IS_ERR(encl_page))
return VM_FAULT_OOM;This is AFAIK the only code path in sgx_vma_fault() flow that allocates non-EPC memory, and the code paths where EPC allocation fails the result would be SIGBUS.
So perhaps allocation is failing here. You could pretty easily trace allocations with bpftrace and kretprobe to see if this is what is happening, e.g. in one terminal:
sudo bpftrace -e 'kr:sgx_encl_page_alloc /retval != 0/ { printf("%d\n", retval); }'
And then run the kselftest in another.
BR, Jarkko
Jarkko Sakkinen
maintainerQuoted 184 lines · Jarkko Sakkinen
Quoted 161 lines · Jarkko Sakkinen
Quoted 132 lines · Vijay
Quoted 107 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 11:53 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 59 lines · Vijay
Quoted 59 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 9:10 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 15 lines · Jarkko Sakkinen
Quoted 14 lines · Jarkko Sakkinen
Quoted 13 lines · Vijay
Quoted 13 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Monday, August 8, 2022 5:18 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longOn Thu, Aug 04, 2022 at 01:14:56PM -0700,
[email protected]Quoted 22 lines · wrote:
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 5 lines
From: Vijay Dhanraj <[email protected]>
This commit adds a new test case which is same as
augment_via_eacceptbut adds more number of EPC pages to stress test
EAUGviaEACCEPT.Quoted 3 lines
Signed-off-by Vijay Dhanraj <[email protected]>Signed-off-by Haitao Huang <[email protected]>Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
Hey, to reproduce the original issue: does it reproduce on VM or should I run baremetal kernel?
BR, Jarkko
Hi Jarkko, The issue should be reproducible on baremetal kernel.
Thanks.
I need comment out other tests in order to make sane out of this :-)
Quoted 6 lines
Mentioning this because came into realization that stress tests should be IMHO moved each to a separate binary (so that they can be run separately). Just a note (TODO) to myself.
I'll work on this today again and *possibly* split your test to its own application to get a starting point for forementioned.
I got
Quoted 10 lines
Quoted 10 lines
# RUN enclave.augment_via_eaccept_long ...
# main.c:1241:augment_via_eaccept_long:test enclave:total_size= 8192,seg->size= 8192 # main.c:1241:augment_via_eaccept_long:test enclave:total_size= 12288,seg->size= 4096 # main.c:1241:augment_via_eaccept_long:test enclave:total_size= 36864,seg->size= 24576 # main.c:1241:augment_via_eaccept_long:test enclave:total_size= 40960,seg->size= 4096 # main.c:1259:augment_via_eaccept_long:mmaping pages at end ofenclave...
Quoted 4 lines
Quoted 4 lines
# main.c:1273:augment_via_eaccept_long:Entering enclave to run EACCEPT for each page of 8589934592 bytes may take a while ...
# OK enclave.augment_via_eaccept_longQuoted 8 lines
Quoted 7 lines
The CPU used for testing was according to
/proc/cpuinfo:model name : Intel(R) Xeon(R) Gold 6338 CPU @ 2.00GHz
I have couple of queries:
1. Is it possible to get dmesg output?
I did check the dmesg output but couldn't find anything related to the
failure. Just the general log messages.
Quoted 11 lines
Quoted 4 lines
2. Do I have to repeat the test multiple times, or does it occur unconditionaly?
I was able to repro every time but it was a bit sporadic for Haitao.
Quoted 1 line
BR, Jarkko
Also, did you set the PRMRR size to 2GB per socket in the BIOS? The issue is only reproduced for oversubscribed scenario. When I set my PRMRR to 64GB per socket, I wasn't able to repro the issue.
Quoted 8 lines
I need to revisit this.
Can you simply run
test_sgxwith gdb and see where it hits?HOST_CFLAGShas apparently "-g" already.Quoted 1 line
Regards, Vijay
BR, Jarkko
I am able to repro the issue when I reduce the PRMRR to 2B/socket but not but not able to break on the assertion failure with gdb. I also enabled debug attribute in the secs but still no avail. Anything I am missing here?
tools/testing/selftests/sgx/load.c +1
-1@@ -87,7 +87,7 @@ static bool encl_ioc_create(struct encl *encl)memset (secs,0 ,sizeof (*secs)); secs->ssa_frame_size =1 ; - secs->attributes =SGX_ATTR_MODE64BIT ; + secs->attributes =SGX_ATTR_MODE64BIT |SGX_ATTR_DEBUG ; secs->xfrm =3 ; secs->base = encl->encl_base; secs->size = encl->encl_size;Quoted 1 line
Regards, Vijay
I get also full pass with 2GB configuration (and also observed that kselftest runs much faster with this configuration).
But I looked at
sgx_alloc_epc_page()and saw this:if (list_empty(&sgx_active_page_list)) return ERR_PTR(-ENOMEM); if (!reclaim) { page = ERR_PTR(-EBUSY); break; }In
sgx_vma_fault(), when running completely out of reclaimable pages, this causesVM_FAULT_SIGBUSreturned instead ofVM_FAULT_NOPAGE:entry = sgx_encl_load_page(encl, addr, vma->vm_flags); if (IS_ERR(entry)) { mutex_unlock(&encl->lock); if (PTR_ERR(entry) == -EBUSY) return VM_FAULT_NOPAGE; return VM_FAULT_SIGBUS; }Quoted 7 lines
Not sure if those should be re-ordered that would keep the process stuck up until there is something to reclaim. Now we use NOPAGE to address situation when there is actually something to reclaim but because of locking side of things we pass reclaim=false to
sgx_alloc_epc_page().So this is kind of OOM behaviour how it works now instead of stalling processes.
Right, I looked at the original email at was really a page fault that was catched. The above theory cannot possibly hold, as the process does not exit with a bus error.
I looked next to
sgx_encl_eaug_page(), and found this:encl_page = sgx_encl_page_alloc(encl, addr - encl->base, secinfo_flags); if (IS_ERR(encl_page)) return VM_FAULT_OOM;This is AFAIK the only code path in
sgx_vma_fault()flow that allocates non-EPC memory, and the code paths where EPC allocation fails the result would be SIGBUS.
Quoted 5 lines
So perhaps allocation is failing here. You could pretty easily trace allocations with bpftrace and kretprobe to see if this is what is happening, e.g. in one terminal:
sudo bpftrace -e 'kr:sgx_encl_page_alloc /retval != 0/ {
printf("%d\n", retval); }'
Should be
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(long)retval < 0/ { printf("%d\n", retval); }'
BR, Jarkko
Quoted 221 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Wednesday, August 10, 2022 6:51 PM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 28 lines · Jarkko Sakkinen
Quoted 27 lines · Jarkko Sakkinen
Quoted 26 lines · Vijay
Quoted 26 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 11:53 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 14 lines · Vijay
Quoted 14 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 9:10 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 1 line · Jarkko Sakkinen
On Mon, Aug 08, 2022 at 06:29:13PM +0300, Jarkko Sakkinen
Quoted 1 line · wrote:
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
On Mon, Aug 08, 2022 at 01:00:54PM +0000, Dhanraj, Vijay
Quoted 27 lines · wrote:
Quoted 27 lines
Quoted 27 lines
Quoted 27 lines
Quoted 27 lines
Quoted 27 lines
Quoted 14 lines
Quoted 14 lines
Quoted 14 lines
Quoted 14 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Monday, August 8, 2022 5:18 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longOn Thu, Aug 04, 2022 at 01:14:56PM -0700, [email protected]
Quoted 12 lines · wrote:
Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
Quoted 5 lines
From: Vijay Dhanraj <[email protected]>
This commit adds a new test case which is same as
augment_via_eacceptbut adds more number of EPC
pages to stress test
EAUGviaEACCEPT.Quoted 5 lines
Signed-off-by: Vijay Dhanraj
<[email protected]>
Signed-off-by: Haitao Huang
<[email protected]>Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 4 lines
Hey, to reproduce the original issue: does it reproduce on VM or should I run baremetal kernel?
BR, Jarkko
Hi Jarkko, The issue should be reproducible on baremetal
kernel.
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 2 lines
Thanks.
I need comment out other tests in order to make sane out of this
:-)Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Mentioning this because came into realization that stress tests should be IMHO moved each to a separate binary (so that they can be run separately). Just a note (TODO) to myself.
I'll work on this today again and *possibly* split your test to its own application to get a starting point for
forementioned.
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
I got
# RUN enclave.augment_via_eaccept_long ...
# main.c:1241:augment_via_eaccept_long:test enclave:total_size= 8192,seg->size= 8192 # main.c:1241:augment_via_eaccept_long:testenclave:
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
total_size= 12288,seg->size= 4096 # main.c:1241:augment_via_eaccept_long:test enclave:total_size= 36864,seg->size= 24576 # main.c:1241:augment_via_eaccept_long:testenclave:
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 2 lines
Quoted 2 lines
total_size= 40960,seg->size= 4096 # main.c:1259:augment_via_eaccept_long:mmaping pages at end ofenclave...
Quoted 2 lines
Quoted 2 lines
# main.c:1273:augment_via_eaccept_long:Entering enclave to run EACCEPT for each page of 8589934592 bytes may take a while
...
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 11 lines
Quoted 9 lines
# OK enclave.augment_via_eaccept_long
The CPU used for testing was according to
/proc/cpuinfo:model name : Intel(R) Xeon(R) Gold 6338 CPU @ 2.00GHz
I have couple of queries:
1. Is it possible to get dmesg output?
I did check the dmesg output but couldn't find anything related to the
failure. Just the general log messages.
Quoted 4 lines
Quoted 4 lines
2. Do I have to repeat the test multiple times, or does it occur unconditionaly?
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
I was able to repro every time but it was a bit sporadic for Haitao.
Quoted 1 line
BR, Jarkko
Also, did you set the PRMRR size to 2GB per socket in the BIOS? The issue is only reproduced for oversubscribed scenario. When I set my PRMRR to 64GB per socket, I wasn't able to
repro the issue.
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 9 lines
I need to revisit this.
Can you simply run
test_sgxwith gdb and see where it hits?HOST_CFLAGShas apparently "-g" already.Quoted 1 line
Regards, Vijay
BR, Jarkko
I am able to repro the issue when I reduce the PRMRR to 2B/socket but
not but not able to break on the assertion failure with gdb. I also enabled debug attribute in the secs but still no avail. Anything I am missing here?
Quoted 26 lines
Quoted 26 lines
Quoted 16 lines
tools/testing/selftests/sgx/load.c +1
-1@@ -87,7 +87,7 @@ static bool encl_ioc_create(struct encl *encl)memset (secs,0 ,sizeof (*secs)); secs->ssa_frame_size =1 ; - secs->attributes =SGX_ATTR_MODE64BIT ; + secs->attributes =SGX_ATTR_MODE64BIT |SGX_ATTR_DEBUG ; secs->xfrm =3 ; secs->base = encl->encl_base; secs->size = encl->encl_size;Quoted 1 line
Regards, Vijay
I get also full pass with 2GB configuration (and also observed that kselftest runs much faster with this configuration).
But I looked at
sgx_alloc_epc_page()and saw this:if (list_empty(&sgx_active_page_list)) return ERR_PTR(-ENOMEM);Quoted 7 lines
Quoted 7 lines
if (!reclaim) { page = ERR_PTR(-EBUSY); break; }In
sgx_vma_fault(), when running completely out of reclaimable pages, this causesVM_FAULT_SIGBUSreturned instead of
VM_FAULT_NOPAGE:Quoted 8 lines
Quoted 8 lines
entry = sgx_encl_load_page(encl, addr, vma->vm_flags); if (IS_ERR(entry)) { mutex_unlock(&encl->lock); if (PTR_ERR(entry) == -EBUSY) return VM_FAULT_NOPAGE;Quoted 7 lines
Quoted 7 lines
return VM_FAULT_SIGBUS; }Not sure if those should be re-ordered that would keep the process stuck up until there is something to reclaim. Now we use NOPAGE to address situation when there is actually something to reclaim but because of locking side of things we pass reclaim=false to
sgx_alloc_epc_page().Quoted 11 lines
Quoted 3 lines
So this is kind of OOM behaviour how it works now instead of stalling processes.
Right, I looked at the original email at was really a page fault that was catched. The above theory cannot possibly hold, as the process does not exit with a bus error.
I looked next to
sgx_encl_eaug_page(), and found this:encl_page = sgx_encl_page_alloc(encl, addr - encl->base,
secinfo_flags);Quoted 7 lines
if (IS_ERR(encl_page)) return VM_FAULT_OOM;This is AFAIK the only code path in
sgx_vma_fault()flow that allocates non-EPC memory, and the code paths where EPC allocation fails the result would be SIGBUS.Quoted 5 lines
So perhaps allocation is failing here. You could pretty easily trace allocations with bpftrace and kretprobe to see if this is what is happening, e.g. in one terminal:
sudo bpftrace -e 'kr:sgx_encl_page_alloc /retval != 0/ { printf("%d\n",
Quoted 8 lines
retval); }'
Should be
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(long)retval < 0/ {
printf("%d\n", retval); }'BR, Jarkko
Thanks let me check and get back to you.
Regards, Vijay
Hi Jarkko
Quoted 221 lines · Jarkko Sakkinen
Quoted 220 lines · Jarkko Sakkinen
Quoted 9 lines · Jarkko Sakkinen
Quoted 8 lines · Vijay
Quoted 8 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 11:53 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang,Haitao
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
<[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 2 lines · Vijay
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 9:10 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected];Huang,
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftest
augment_via_eaccept_longQuoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
On Tue, Aug 09, 2022 at 01:45:35PM +0300, Jarkko Sakkinen
Quoted 1 line · wrote:
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
On Mon, Aug 08, 2022 at 06:29:13PM +0300, Jarkko Sakkinen
Quoted 1 line · wrote:
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
On Mon, Aug 08, 2022 at 01:00:54PM +0000, Dhanraj, Vijay
Quoted 7 lines · wrote:
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Monday, August 8, 2022 5:18 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>;Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longOn Thu, Aug 04, 2022 at 01:14:56PM -0700, [email protected]
Quoted 4 lines · wrote:
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
From: Vijay Dhanraj <[email protected]>
This commit adds a new test case which is same as
augment_via_eacceptbut adds more number of EPCpages to
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 1 line
stress test
EAUGviaEACCEPT.Quoted 2 lines
Signed-off-by Vijay DhanrajQuoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Signed-off-by Haitao HuangQuoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Hey, to reproduce the original issue: does it
reproduce on
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines
VM or should I run baremetal kernel?
BR, Jarkko
Hi Jarkko, The issue should be reproducible on
baremetal kernel.
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 2 lines
Thanks.
I need comment out other tests in order to make sane out of
this
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
:-)
Mentioning this because came into realization that stress
tests
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
should be IMHO moved each to a separate binary (so that
they can
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
be run separately). Just a note (TODO) to myself.
I'll work on this today again and *possibly* split your
test to
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
its own application to get a starting point for
forementioned.
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
I got
# RUN enclave.augment_via_eaccept_long ...
# main.c:1241:augment_via_eaccept_long:test enclave:
total_size=Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
8192,
seg->size= 8192 # main.c:1241:augment_via_eaccept_long:testenclave:
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
total_size= 12288,seg->size= 4096 # main.c:1241:augment_via_eaccept_long:test enclave:total_size=
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
36864,
seg->size= 24576 # main.c:1241:augment_via_eaccept_long:testenclave:
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 2 lines
Quoted 2 lines
total_size= 40960,seg->size= 4096 # main.c:1259:augment_via_eaccept_long:mmaping pages at end ofenclave...
Quoted 4 lines
Quoted 4 lines
# main.c:1273:augment_via_eaccept_long:Entering enclave to run EACCEPT for each page of 8589934592 bytes may take a while ...
# OK enclave.augment_via_eaccept_longQuoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 7 lines
The CPU used for testing was according to
/proc/cpuinfo:model name : Intel(R) Xeon(R) Gold 6338 CPU @ 2.00GHz
I have couple of queries:
1. Is it possible to get dmesg output?
I did check the dmesg output but couldn't find anything related
to the
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
failure. Just the general log messages.
Quoted 5 lines
Quoted 4 lines
2. Do I have to repeat the test multiple times, or does it occur unconditionaly?
I was able to repro every time but it was a bit sporadic for
Haitao.
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 2 lines
BR, Jarkko
Also, did you set the PRMRR size to 2GB per socket in the BIOS?
The
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
issue is only reproduced for oversubscribed scenario. When I
set my
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 1 line
PRMRR to 64GB per socket, I wasn't able to repro the issue.
I need to revisit this.
Quoted 8 lines
Quoted 8 lines
Quoted 6 lines
Can you simply run
test_sgxwith gdb and see where it hits?HOST_CFLAGShas apparently "-g" already.Quoted 1 line
Regards, Vijay
BR, Jarkko
I am able to repro the issue when I reduce the PRMRR to 2B/socket
but not but not able to break on the assertion failure with gdb. I also enabled debug attribute in the secs but still no avail. Anything I am missing here?
Quoted 2 lines
Review
Quoted 2 lines
(quoted diff) +0
-0
b/tools/testing/selftests/sgx/load.cQuoted 28 lines
Quoted 13 lines
index
7de1b15c90b1..c4bccd3f5f17100644tools/testing/selftests/sgx/load.c +1
-1@@ -87,7 +87,7 @@ static bool encl_ioc_create(struct encl *encl)memset (secs,0 ,sizeof (*secs)); secs->ssa_frame_size =1 ; - secs->attributes =SGX_ATTR_MODE64BIT ; + secs->attributes =SGX_ATTR_MODE64BIT |SGX_ATTR_DEBUG ; secs->xfrm =3 ; secs->base = encl->encl_base; secs->size = encl->encl_size;Quoted 1 line
Regards, Vijay
I get also full pass with 2GB configuration (and also observed that kselftest runs much faster with this configuration).
But I looked at
sgx_alloc_epc_page()and saw this:if (list_empty(&sgx_active_page_list)) return ERR_PTR(-ENOMEM); if (!reclaim) { page = ERR_PTR(-EBUSY); break; }Quoted 1 line
In
sgx_vma_fault(), when running completely out of reclaimable pages,this
Quoted 12 lines
causes
VM_FAULT_SIGBUSreturned instead ofVM_FAULT_NOPAGE:entry = sgx_encl_load_page(encl, addr, vma->vm_flags); if (IS_ERR(entry)) { mutex_unlock(&encl->lock); if (PTR_ERR(entry) == -EBUSY) return VM_FAULT_NOPAGE; return VM_FAULT_SIGBUS; }Quoted 1 line
Not sure if those should be re-ordered that would keep the process
stuck up
Quoted 1 line
until there is something to reclaim. Now we use NOPAGE to address
situation
Quoted 1 line
when there is actually something to reclaim but because of locking
side of
Quoted 4 lines
things we pass reclaim=false to
sgx_alloc_epc_page().So this is kind of OOM behaviour how it works now instead of stalling processes.
Right, I looked at the original email at was really a page fault that was catched. The above theory cannot possibly hold, as the process does not exit with a bus error.
I looked next to
sgx_encl_eaug_page(), and found this:encl_page = sgx_encl_page_alloc(encl, addr - encl->base, secinfo_flags); if (IS_ERR(encl_page)) return VM_FAULT_OOM;This is AFAIK the only code path in
sgx_vma_fault()flow that allocates non-EPC memory, and the code paths where EPC allocation fails the result would be SIGBUS.So perhaps allocation is failing here. You could pretty easily trace allocations with bpftrace and kretprobe to see if this is what is happening, e.g. in one terminal:
Quoted 7 lines
Quoted 1 line
sudo bpftrace -e 'kr:sgx_encl_page_alloc /retval != 0/ {
printf("%d\n", retval); }'Should be
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(long)retval < 0/ {
printf("%d\n", retval); }'BR, Jarkko
I tried these probs and got following results when failure happens (not always happen on my device).
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(int64)retval <0 / { printf("%X\n", retval); }'
--> lots of negative values, I believe they are valid addresses in unsigned long type. So I looked up IS_ERR_VALUE macro and translated it in following probes.
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(uint64)retval >= (uint64)(-4095)/ { printf("%X\n", retval); }'
--> none triggered
sudo bpftrace -e 'kr:sgx_alloc_epc_page /(uint64)retval >= (uint64)(-4095)/ { printf("%X\n", retval); }'
--> FFFFFFF0 printed, which I believe is -EBUSY.
BR
Haitao
Quoted 276 lines
-----Original Message-----
From: Haitao Huang <[email protected]>
Sent: Thursday, August 11, 2022 7:30 PM
To: Dhanraj, Vijay <[email protected]>; Jarkko Sakkinen <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longHi Jarkko
Quoted 116 lines · Jarkko Sakkinen
Quoted 115 lines · Jarkko Sakkinen
Quoted 10 lines · Jarkko Sakkinen
Quoted 9 lines · Vijay
Quoted 9 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 11:53 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang,Haitao
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
<[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 2 lines · Vijay
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 9:10 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected];Huang,
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftest
augment_via_eaccept_longQuoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
On Tue, Aug 09, 2022 at 01:45:35PM +0300, Jarkko Sakkinen
Quoted 1 line · wrote:
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
On Mon, Aug 08, 2022 at 06:29:13PM +0300, Jarkko Sakkinen
Quoted 2 lines · wrote:
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
On Mon, Aug 08, 2022 at 01:00:54PM +0000, Dhanraj, Vijay
Quoted 7 lines · wrote:
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Monday, August 8, 2022 5:18 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>;Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longOn Thu, Aug 04, 2022 at 01:14:56PM -0700, [email protected]
Quoted 4 lines · wrote:
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
From: Vijay Dhanraj <[email protected]>
This commit adds a new test case which is same as
`augment_via_eaccept` but adds more number of EPCpages to
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 1 line
stress test
EAUGviaEACCEPT.Quoted 2 lines
Signed-off-by: Vijay Dhanraj
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Signed-off-by: Haitao Huang
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Hey, to reproduce the original issue: does it
reproduce on
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines
VM or should I run baremetal kernel?
BR, Jarkko
Hi Jarkko, The issue should be reproducible on
baremetal kernel.
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 2 lines
Thanks.
I need comment out other tests in order to make sane out of
this
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
:-)
Mentioning this because came into realization that stress
tests
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
should be IMHO moved each to a separate binary (so that
they can
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
be run separately). Just a note (TODO) to myself.
I'll work on this today again and *possibly* split your
test to
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
its own application to get a starting point for
forementioned.
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
I got
# RUN enclave.augment_via_eaccept_long ...
# main.c:1241:augment_via_eaccept_long:test enclave:
total_size=Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
8192,
seg->size= 8192 # seg->main.c:1241:augment_via_eaccept_long:testenclave:
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
total_size= 12288,seg->size= 4096 # main.c:1241:augment_via_eaccept_long:test enclave:total_size=
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
36864,
seg->size= 24576 # seg->main.c:1241:augment_via_eaccept_long:testenclave:
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 3 lines
Quoted 3 lines
total_size= 40960,seg->size= 4096 # main.c:1259:augment_via_eaccept_long:mmaping pages at end ofenclave...
Quoted 2 lines
Quoted 2 lines
# main.c:1273:augment_via_eaccept_long:Entering enclave to run EACCEPT for each page of 8589934592 bytes may take a while
...
Quoted 120 lines
Quoted 113 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 9 lines
# OK enclave.augment_via_eaccept_long
The CPU used for testing was according to
/proc/cpuinfo:model name : Intel(R) Xeon(R) Gold 6338 CPU @ 2.00GHz
I have couple of queries:
1. Is it possible to get dmesg output?
I did check the dmesg output but couldn't find anything related
to the
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
failure. Just the general log messages.
Quoted 5 lines
Quoted 4 lines
2. Do I have to repeat the test multiple times, or does it occur unconditionaly?
I was able to repro every time but it was a bit sporadic for
Haitao.
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 2 lines
BR, Jarkko
Also, did you set the PRMRR size to 2GB per socket in the BIOS?
The
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
issue is only reproduced for oversubscribed scenario. When I
set my
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 1 line
PRMRR to 64GB per socket, I wasn't able to repro the issue.
I need to revisit this.
Quoted 8 lines
Quoted 8 lines
Quoted 6 lines
Can you simply run
test_sgxwith gdb and see where it hits?HOST_CFLAGShas apparently "-g" already.Quoted 1 line
Regards, Vijay
BR, Jarkko
I am able to repro the issue when I reduce the PRMRR to 2B/socket
but not but not able to break on the assertion failure with gdb. I also enabled debug attribute in the secs but still no avail. Anything I am missing here?
Quoted 2 lines
Review
Quoted 2 lines
(quoted diff) +0
-0
b/tools/testing/selftests/sgx/load.cQuoted 28 lines
Quoted 13 lines
index
7de1b15c90b1..c4bccd3f5f17100644tools/testing/selftests/sgx/load.c +1
-1@@ -87,7 +87,7 @@ static bool encl_ioc_create(struct encl *encl)memset (secs,0 ,sizeof (*secs)); secs->ssa_frame_size =1 ; - secs->attributes =SGX_ATTR_MODE64BIT ; + secs->attributes =SGX_ATTR_MODE64BIT |SGX_ATTR_DEBUG ; secs->xfrm =3 ; secs->base = encl->encl_base; secs->size = encl->encl_size;Quoted 1 line
Regards, Vijay
I get also full pass with 2GB configuration (and also observed that kselftest runs much faster with this configuration).
But I looked at
sgx_alloc_epc_page()and saw this:if (list_empty(&sgx_active_page_list)) return ERR_PTR(-ENOMEM); if (!reclaim) { page = ERR_PTR(-EBUSY); break; }Quoted 2 lines
In
sgx_vma_fault(), when running completely out of reclaimable pages,this
Quoted 12 lines
causes
VM_FAULT_SIGBUSreturned instead ofVM_FAULT_NOPAGE:entry = sgx_encl_load_page(encl, addr, vma->vm_flags); if (IS_ERR(entry)) { mutex_unlock(&encl->lock); if (PTR_ERR(entry) == -EBUSY) return VM_FAULT_NOPAGE; return VM_FAULT_SIGBUS; }Quoted 1 line
Not sure if those should be re-ordered that would keep the process
stuck up
Quoted 1 line
until there is something to reclaim. Now we use NOPAGE to address
situation
Quoted 1 line
when there is actually something to reclaim but because of locking
side of
Quoted 4 lines
things we pass reclaim=false to
sgx_alloc_epc_page().So this is kind of OOM behaviour how it works now instead of stalling processes.
Right, I looked at the original email at was really a page fault that was catched. The above theory cannot possibly hold, as the process does not exit with a bus error.
I looked next to
sgx_encl_eaug_page(), and found this:encl_page = sgx_encl_page_alloc(encl, addr - encl->base, secinfo_flags); if (IS_ERR(encl_page)) return VM_FAULT_OOM;This is AFAIK the only code path in
sgx_vma_fault()flow that allocates non-EPC memory, and the code paths where EPC allocation fails the result would be SIGBUS.Quoted 6 lines
So perhaps allocation is failing here. You could pretty easily trace allocations with bpftrace and kretprobe to see if this is what is happening, e.g. in one terminal:
sudo bpftrace -e 'kr:sgx_encl_page_alloc /retval != 0/ {
printf("%d\n", retval); }'Quoted 6 lines
Should be
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(long)retval < 0/ {
printf("%d\n", retval); }'BR, Jarkko
I tried these probs and got following results when failure happens (not always happen on my device).
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(int64)retval <0 / {
printf("%X\n", retval); }'--> lots of negative values, I believe they are valid addresses in unsigned long type. So I looked up
IS_ERR_VALUEmacro and translated it in following probes.sudo bpftrace -e 'kr:sgx_encl_page_alloc /(uint64)retval >= (uint64)(-4095)/ {
printf("%X\n", retval); }'--> none triggered
Quoted 7 lines
sudo bpftrace -e 'kr:sgx_alloc_epc_page /(uint64)retval >= (uint64)(-4095)/ {
printf("%X\n", retval); }'--> FFFFFFF0 printed, which I believe is
-EBUSY.BR
Haitao
I see the same behavior as Haitao.
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(long)retval < 0/ { printf("%d\n", retval); } -> This one gave an error stdin:1:24-31: ERROR: Unknown struct/union: 'long'
So switched to sudo bpftrace -e 'kr:sgx_encl_page_alloc /(int64)retval <0 / { printf("%X\n", retval); }' as suggested by Haitao and do see lot of positive and negative values.
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(uint64)retval >= (uint64)(-4095)/ { printf("%X\n", retval); }' -> No output.
sudo bpftrace -e 'kr:sgx_alloc_epc_page /(uint64)retval >= (uint64)(-4095)/ { printf("%X\n", retval); } -> FFFFFFF0 is printed.
Thanks, Vijay
Jarkko Sakkinen
maintainerQuoted 284 lines · Vijay
Quoted 276 lines
-----Original Message-----
From: Haitao Huang <[email protected]>
Sent: Thursday, August 11, 2022 7:30 PM
To: Dhanraj, Vijay <[email protected]>; Jarkko Sakkinen <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longHi Jarkko
Quoted 116 lines · Jarkko Sakkinen
Quoted 115 lines · Jarkko Sakkinen
Quoted 10 lines · Jarkko Sakkinen
Quoted 9 lines · Vijay
Quoted 9 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 11:53 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang,Haitao
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
<[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 2 lines · Vijay
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 9:10 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected];Huang,
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftest
augment_via_eaccept_longQuoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
On Tue, Aug 09, 2022 at 01:45:35PM +0300, Jarkko Sakkinen
Quoted 1 line · wrote:
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
On Mon, Aug 08, 2022 at 06:29:13PM +0300, Jarkko Sakkinen
Quoted 2 lines · wrote:
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
On Mon, Aug 08, 2022 at 01:00:54PM +0000, Dhanraj, Vijay
Quoted 7 lines · wrote:
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Monday, August 8, 2022 5:18 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette
<[email protected]>;Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftest
`augment_via_eaccept_long`On Thu, Aug 04, 2022 at 01:14:56PM -0700,
[email protected]Quoted 4 lines · wrote:
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
From: Vijay Dhanraj <[email protected]>
This commit adds a new test case which is same as
`augment_via_eaccept` but adds more number of EPCpages to
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
stress test
`EAUG` via `EACCEPT`.Signed-off-by: Vijay Dhanraj
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Signed-off-by: Haitao Huang
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Hey, to reproduce the original issue: does it
reproduce on
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines
VM or should I run baremetal kernel?
BR, Jarkko
Hi Jarkko, The issue should be reproducible on
baremetal kernel.
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 2 lines
Thanks.
I need comment out other tests in order to make sane out of
this
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
:-)
Mentioning this because came into realization that stress
tests
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
should be IMHO moved each to a separate binary (so that
they can
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
be run separately). Just a note (TODO) to myself.
I'll work on this today again and *possibly* split your
test to
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
its own application to get a starting point for
forementioned.
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
I got
# RUN enclave.augment_via_eaccept_long ...
# main.c:1241:augment_via_eaccept_long:test enclave:
total_size=Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
8192,
seg->size= 8192 # seg->main.c:1241:augment_via_eaccept_long:testenclave:
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
total_size= 12288,seg->size= 4096 # main.c:1241:augment_via_eaccept_long:test enclave:total_size=
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
36864,
seg->size= 24576 # seg->main.c:1241:augment_via_eaccept_long:testenclave:
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 3 lines
Quoted 3 lines
total_size= 40960,seg->size= 4096 # main.c:1259:augment_via_eaccept_long:mmaping pages at end ofenclave...
Quoted 2 lines
Quoted 2 lines
# main.c:1273:augment_via_eaccept_long:Entering enclave to run EACCEPT for each page of 8589934592 bytes may take a while
...
Quoted 120 lines
Quoted 113 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 9 lines
# OK enclave.augment_via_eaccept_long
The CPU used for testing was according to
/proc/cpuinfo:model name : Intel(R) Xeon(R) Gold 6338 CPU @ 2.00GHz
I have couple of queries:
1. Is it possible to get dmesg output?
I did check the dmesg output but couldn't find anything related
to the
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
failure. Just the general log messages.
Quoted 5 lines
Quoted 4 lines
2. Do I have to repeat the test multiple times, or does it occur unconditionaly?
I was able to repro every time but it was a bit sporadic for
Haitao.
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 2 lines
BR, Jarkko
Also, did you set the PRMRR size to 2GB per socket in the BIOS?
The
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
issue is only reproduced for oversubscribed scenario. When I
set my
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 1 line
PRMRR to 64GB per socket, I wasn't able to repro the issue.
I need to revisit this.
Quoted 8 lines
Quoted 8 lines
Quoted 6 lines
Can you simply run
test_sgxwith gdb and see where it hits?HOST_CFLAGShas apparently "-g" already.Quoted 1 line
Regards, Vijay
BR, Jarkko
I am able to repro the issue when I reduce the PRMRR to 2B/socket
but not but not able to break on the assertion failure with gdb. I also enabled debug attribute in the secs but still no avail. Anything I am missing here?
Quoted 2 lines
Review
Quoted 2 lines
(quoted diff) +0
-0
b/tools/testing/selftests/sgx/load.cQuoted 28 lines
Quoted 13 lines
index
7de1b15c90b1..c4bccd3f5f17100644tools/testing/selftests/sgx/load.c +1
-1@@ -87,7 +87,7 @@ static bool encl_ioc_create(struct encl *encl)memset (secs,0 ,sizeof (*secs)); secs->ssa_frame_size =1 ; - secs->attributes =SGX_ATTR_MODE64BIT ; + secs->attributes =SGX_ATTR_MODE64BIT |SGX_ATTR_DEBUG ; secs->xfrm =3 ; secs->base = encl->encl_base; secs->size = encl->encl_size;Quoted 1 line
Regards, Vijay
I get also full pass with 2GB configuration (and also observed that kselftest runs much faster with this configuration).
But I looked at
sgx_alloc_epc_page()and saw this:if (list_empty(&sgx_active_page_list)) return ERR_PTR(-ENOMEM); if (!reclaim) { page = ERR_PTR(-EBUSY); break; }Quoted 2 lines
In
sgx_vma_fault(), when running completely out of reclaimable pages,this
Quoted 12 lines
causes
VM_FAULT_SIGBUSreturned instead ofVM_FAULT_NOPAGE:entry = sgx_encl_load_page(encl, addr, vma->vm_flags); if (IS_ERR(entry)) { mutex_unlock(&encl->lock); if (PTR_ERR(entry) == -EBUSY) return VM_FAULT_NOPAGE; return VM_FAULT_SIGBUS; }Quoted 1 line
Not sure if those should be re-ordered that would keep the process
stuck up
Quoted 1 line
until there is something to reclaim. Now we use NOPAGE to address
situation
Quoted 1 line
when there is actually something to reclaim but because of locking
side of
Quoted 4 lines
things we pass reclaim=false to
sgx_alloc_epc_page().So this is kind of OOM behaviour how it works now instead of stalling processes.
Right, I looked at the original email at was really a page fault that was catched. The above theory cannot possibly hold, as the process does not exit with a bus error.
I looked next to
sgx_encl_eaug_page(), and found this:encl_page = sgx_encl_page_alloc(encl, addr - encl->base, secinfo_flags); if (IS_ERR(encl_page)) return VM_FAULT_OOM;This is AFAIK the only code path in
sgx_vma_fault()flow that allocates non-EPC memory, and the code paths where EPC allocation fails the result would be SIGBUS.Quoted 6 lines
So perhaps allocation is failing here. You could pretty easily trace allocations with bpftrace and kretprobe to see if this is what is happening, e.g. in one terminal:
sudo bpftrace -e 'kr:sgx_encl_page_alloc /retval != 0/ {
printf("%d\n", retval); }'Quoted 6 lines
Should be
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(long)retval < 0/ {
printf("%d\n", retval); }'BR, Jarkko
I tried these probs and got following results when failure happens (not always happen on my device).
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(int64)retval <0 / {
printf("%X\n", retval); }'--> lots of negative values, I believe they are valid addresses in unsigned long type. So I looked up
IS_ERR_VALUEmacro and translated it in following probes.sudo bpftrace -e 'kr:sgx_encl_page_alloc /(uint64)retval >= (uint64)(-4095)/ {
printf("%X\n", retval); }'--> none triggered
Quoted 7 lines
sudo bpftrace -e 'kr:sgx_alloc_epc_page /(uint64)retval >= (uint64)(-4095)/ {
printf("%X\n", retval); }'--> FFFFFFF0 printed, which I believe is
-EBUSY.BR
Haitao
Quoted 8 lines
I see the same behavior as Haitao.
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(long)retval < 0/ {printf("%d\n", retval); } -> This one gave an error stdin:1:24-31: ERROR: Unknown struct/union: 'long'So switched to sudo bpftrace -e 'kr:sgx_encl_page_alloc /(int64)retval <0 / {
printf("%X\n", retval); }' as suggested by Haitao and do see lot of positive and negative values.sudo bpftrace -e 'kr:sgx_encl_page_alloc /(uint64)retval >= (uint64)(-4095)/ {
printf("%X\n", retval); }' -> No output.sudo bpftrace -e 'kr:sgx_alloc_epc_page /(uint64)retval >= (uint64)(-4095)/ {
printf("%X\n", retval); } -> FFFFFFF0 is printed.
And you are still encountering segfaults? And does it happen when e.g. EBUSY is encountered, which should be fully legit. E.g. if there was segfault simultaneously perhaps that code path has something wrong.
BR, Jarkko
Quoted 327 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Sunday, August 14, 2022 11:09 AM
To: Dhanraj, Vijay <[email protected]>
Cc: Haitao Huang <[email protected]>; linux- [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 160 lines · Vijay
Quoted 160 lines
-----Original Message-----
From: Haitao Huang <[email protected]>
Sent: Thursday, August 11, 2022 7:30 PM
To: Dhanraj, Vijay <[email protected]>; Jarkko Sakkinen <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longHi Jarkko
Quoted 124 lines · Jarkko Sakkinen
Quoted 123 lines · Jarkko Sakkinen
Quoted 10 lines · Jarkko Sakkinen
Quoted 9 lines · Vijay
Quoted 9 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 11:53 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang,Haitao
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
<[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 2 lines · Vijay
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 9:10 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected];Huang,
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftest
augment_via_eaccept_longQuoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
On Tue, Aug 09, 2022 at 01:45:35PM +0300, Jarkko Sakkinen
Quoted 2 lines · wrote:
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
On Mon, Aug 08, 2022 at 06:29:13PM +0300, Jarkko
SakkinenQuoted 2 lines · wrote:
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
On Mon, Aug 08, 2022 at 01:00:54PM +0000, Dhanraj, Vijay
Quoted 7 lines · wrote:
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Monday, August 8, 2022 5:18 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette
<[email protected]>;Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftest
`augment_via_eaccept_long`Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
On Thu, Aug 04, 2022 at 01:14:56PM -0700,
[email protected]Quoted 5 lines · wrote:
Quoted 5 lines
Quoted 5 lines
From: Vijay Dhanraj <[email protected]>
This commit adds a new test case which is
same as `augment_via_eaccept` but adds more
number of EPCpages to
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
stress test
`EAUG` via `EACCEPT`.Signed-off-by: Vijay Dhanraj
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Signed-off-by: Haitao Huang
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Hey, to reproduce the original issue: does it
reproduce on
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
VM or should I run baremetal kernel?
BR, Jarkko
Hi Jarkko, The issue should be reproducible on
baremetal kernel.
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 2 lines
Thanks.
I need comment out other tests in order to make sane out of
this
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
:-)
Mentioning this because came into realization that stress
tests
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
should be IMHO moved each to a separate binary (so that
they can
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
be run separately). Just a note (TODO) to myself.
I'll work on this today again and *possibly* split your
test to
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
its own application to get a starting point for
forementioned.
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
I got
# RUN enclave.augment_via_eaccept_long ...
# main.c:1241:augment_via_eaccept_long:test enclave:
total_size=Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
8192,
seg->size= 8192 # seg->main.c:1241:augment_via_eaccept_long:testenclave:
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
total_size= 12288,seg->size= 4096 # main.c:1241:augment_via_eaccept_long:test enclave:total_size=
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
36864,
seg->size= 24576 # seg->main.c:1241:augment_via_eaccept_long:testenclave:
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 3 lines
Quoted 3 lines
total_size= 40960,seg->size= 4096 # main.c:1259:augment_via_eaccept_long:mmaping pages at end ofenclave...
Quoted 3 lines
Quoted 3 lines
# main.c:1273:augment_via_eaccept_long:Entering enclave to run EACCEPT for each page of 8589934592 bytes may take a while
...
Quoted 17 lines
Quoted 17 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 9 lines
# OK enclave.augment_via_eaccept_long
The CPU used for testing was according to
/proc/cpuinfo:model name : Intel(R) Xeon(R) Gold 6338 CPU @ 2.00GHz
I have couple of queries:
1. Is it possible to get dmesg output?
I did check the dmesg output but couldn't find anything related
to the
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
failure. Just the general log messages.
Quoted 4 lines
Quoted 4 lines
2. Do I have to repeat the test multiple times, or does it occur unconditionaly?
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
I was able to repro every time but it was a bit sporadic for
Haitao.
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 2 lines
BR, Jarkko
Also, did you set the PRMRR size to 2GB per socket in the
BIOS?
Quoted 53 lines
Quoted 53 lines
Quoted 53 lines
Quoted 53 lines
The
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
issue is only reproduced for oversubscribed scenario. When I
set my
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 1 line
PRMRR to 64GB per socket, I wasn't able to repro the issue.
I need to revisit this.
Can you simply run
test_sgxwith gdb and see where it hits?HOST_CFLAGShas apparently "-g" already.Quoted 6 lines
Quoted 6 lines
Quoted 3 lines
Quoted 1 line
Regards, Vijay
BR, Jarkko
I am able to repro the issue when I reduce the PRMRR to 2B/socket
but not but not able to break on the assertion failure with gdb. I also enabled debug attribute in the secs but still no avail. Anything I am missing here?
Quoted 2 lines
Review
Quoted 2 lines
(quoted diff) +0
-0
b/tools/testing/selftests/sgx/load.cQuoted 30 lines
Quoted 15 lines
index
7de1b15c90b1..c4bccd3f5f17100644tools/testing/selftests/sgx/load.c +2
-1@@ -87,7 +87,7 @@ static bool encl_ioc_create(struct encl *encl)memset (secs,0 ,sizeof (*secs)); secs->ssa_frame_size =1 ; - secs->attributes =SGX_ATTR_MODE64BIT ; + secs->attributes =SGX_ATTR_MODE64BIT | +SGX_ATTR_DEBUG ; secs->xfrm =3 ; secs->base = encl->encl_base; secs->size = encl->encl_size;Quoted 1 line
Regards, Vijay
I get also full pass with 2GB configuration (and also observed that kselftest runs much faster with this configuration).
But I looked at
sgx_alloc_epc_page()and saw this:if (list_empty(&sgx_active_page_list)) return ERR_PTR(-ENOMEM); if (!reclaim) { page = ERR_PTR(-EBUSY); break; }Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 2 lines
In
sgx_vma_fault(), when running completely out of reclaimable pages,this
Quoted 1 line
causes
VM_FAULT_SIGBUSreturned instead of
VM_FAULT_NOPAGE:Quoted 72 lines
Quoted 72 lines
Quoted 47 lines
Quoted 40 lines
Quoted 11 lines
entry = sgx_encl_load_page(encl, addr, vma->vm_flags); if (IS_ERR(entry)) { mutex_unlock(&encl->lock); if (PTR_ERR(entry) == -EBUSY) return VM_FAULT_NOPAGE; return VM_FAULT_SIGBUS; }Quoted 2 lines
Not sure if those should be re-ordered that would keep the process
stuck up
Quoted 2 lines
until there is something to reclaim. Now we use NOPAGE to address
situation
Quoted 2 lines
when there is actually something to reclaim but because of locking
side of
Quoted 4 lines
things we pass reclaim=false to
sgx_alloc_epc_page().So this is kind of OOM behaviour how it works now instead of stalling processes.
Right, I looked at the original email at was really a page fault that was catched. The above theory cannot possibly hold, as the process does not exit with a bus error.
I looked next to
sgx_encl_eaug_page(), and found this:encl_page = sgx_encl_page_alloc(encl, addr - encl->base, secinfo_flags); if (IS_ERR(encl_page)) return VM_FAULT_OOM;This is AFAIK the only code path in
sgx_vma_fault()flow that allocates non-EPC memory, and the code paths where EPC allocation fails the result would be SIGBUS.Quoted 6 lines
So perhaps allocation is failing here. You could pretty easily trace allocations with bpftrace and kretprobe to see if this is what is happening, e.g. in one terminal:
sudo bpftrace -e 'kr:sgx_encl_page_alloc /retval != 0/ {
printf("%d\n", retval); }'Quoted 6 lines
Should be
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(long)retval < 0/ {
printf("%d\n", retval); }'BR, Jarkko
I tried these probs and got following results when failure happens (not always happen on my device).
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(int64)retval <0 / {
printf("%X\n", retval); }'--> lots of negative values, I believe they are valid addresses in unsigned long type. So I looked up
IS_ERR_VALUEmacro and translated it in following probes.sudo bpftrace -e 'kr:sgx_encl_page_alloc /(uint64)retval >= (uint64)(-4095)/ {
printf("%X\n", retval); }'--> none triggered
sudo bpftrace -e 'kr:sgx_alloc_epc_page /(uint64)retval >= (uint64)(-4095)/ {
printf("%X\n", retval); }'Quoted 7 lines
Quoted 4 lines
--> FFFFFFF0 printed, which I believe is
-EBUSY.BR
HaitaoI see the same behavior as Haitao.
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(long)retval < 0/ { printf("%d\n",retval); } -> This one gave an error stdin:1:24-31: ERROR: Unknown struct/union: 'long'
Quoted 2 lines
So switched to sudo bpftrace -e 'kr:sgx_encl_page_alloc /(int64)retval <0 /
{
printf("%X\n", retval); }' as suggested by Haitao and do see lot of positive and negative values.Quoted 2 lines
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(uint64)retval >= (uint64)(-
4095)/ {
printf("%X\n", retval); }' -> No output.Quoted 1 line
Quoted 8 lines
Quoted 1 line
sudo bpftrace -e 'kr:sgx_alloc_epc_page /(uint64)retval >= (uint64)(-4095)/
{
printf("%X\n", retval); } -> FFFFFFF0 is printed.And you are still encountering segfaults? And does it happen when e.g. EBUSY is encountered, which should be fully legit. E.g. if there was segfault simultaneously perhaps that code path has something wrong.
BR, Jarkko
No, with the proposed VA page allocation fix from Haitao I no longer see the segfaults with Gramine tests.
Thanks, Vijay
Quoted 242 lines · Haitao Huang
Hi Jarkko
Quoted 223 lines · Jarkko Sakkinen
Quoted 222 lines · Jarkko Sakkinen
Quoted 9 lines · Jarkko Sakkinen
Quoted 8 lines · Vijay
Quoted 8 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 11:53 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected]; Huang,Haitao
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
<[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 2 lines · Vijay
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 9:10 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected];Huang,
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftest
augment_via_eaccept_longQuoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
On Tue, Aug 09, 2022 at 01:45:35PM +0300, Jarkko Sakkinen
Quoted 1 line · wrote:
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
On Mon, Aug 08, 2022 at 06:29:13PM +0300, Jarkko Sakkinen
Quoted 1 line · wrote:
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
On Mon, Aug 08, 2022 at 01:00:54PM +0000, Dhanraj, Vijay
Quoted 7 lines · wrote:
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Monday, August 8, 2022 5:18 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>;Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longOn Thu, Aug 04, 2022 at 01:14:56PM -0700, [email protected]
Quoted 4 lines · wrote:
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
From: Vijay Dhanraj <[email protected]>
This commit adds a new test case which is same as
`augment_via_eaccept` but adds more number of EPCpages to
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 1 line
stress test
EAUGviaEACCEPT.Quoted 2 lines
Signed-off-by: Vijay Dhanraj
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Signed-off-by: Haitao Huang
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Hey, to reproduce the original issue: does it
reproduce on
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines
VM or should I run baremetal kernel?
BR, Jarkko
Hi Jarkko, The issue should be reproducible on
baremetal kernel.
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 2 lines
Thanks.
I need comment out other tests in order to make sane out
of this
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
:-)
Mentioning this because came into realization that stress
tests
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
should be IMHO moved each to a separate binary (so that
they can
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
be run separately). Just a note (TODO) to myself.
I'll work on this today again and *possibly* split your
test to
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
its own application to get a starting point for
forementioned.
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
I got
# RUN enclave.augment_via_eaccept_long ...
# main.c:1241:augment_via_eaccept_long:test enclave:
total_size=Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
8192,
seg->size= 8192 # main.c:1241:augment_via_eaccept_long:testenclave:
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
total_size= 12288,seg->size= 4096 # main.c:1241:augment_via_eaccept_long:test enclave:
total_size=Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
36864,
seg->size= 24576 #main.c:1241:augment_via_eaccept_long:test enclave:
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 2 lines
Quoted 2 lines
total_size= 40960,seg->size= 4096 # main.c:1259:augment_via_eaccept_long:mmaping pages at end ofenclave...
Quoted 1 line
Quoted 1 line
# main.c:1273:augment_via_eaccept_long:Entering enclave to
run
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
EACCEPT for each page of 8589934592 bytes may take a while
...
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 9 lines
# OK enclave.augment_via_eaccept_long
The CPU used for testing was according to
/proc/cpuinfo:model name : Intel(R) Xeon(R) Gold 6338 CPU @ 2.00GHz
I have couple of queries:
1. Is it possible to get dmesg output?
I did check the dmesg output but couldn't find anything
related to the
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
failure. Just the general log messages.
Quoted 5 lines
Quoted 4 lines
2. Do I have to repeat the test multiple times, or does it occur unconditionaly?
I was able to repro every time but it was a bit sporadic for
Haitao.
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 2 lines
BR, Jarkko
Also, did you set the PRMRR size to 2GB per socket in the
BIOS? The
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
issue is only reproduced for oversubscribed scenario. When I
set my
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 1 line
PRMRR to 64GB per socket, I wasn't able to repro the issue.
I need to revisit this.
Quoted 8 lines
Quoted 8 lines
Quoted 6 lines
Can you simply run
test_sgxwith gdb and see where it hits?HOST_CFLAGShas apparently "-g" already.Quoted 1 line
Regards, Vijay
BR, Jarkko
I am able to repro the issue when I reduce the PRMRR to 2B/socket
but not but not able to break on the assertion failure with gdb. I also enabled debug attribute in the secs but still no avail. Anything I am missing here?
Quoted 2 lines
Review
Quoted 2 lines
(quoted diff) +0
-0
b/tools/testing/selftests/sgx/load.cQuoted 28 lines
Quoted 13 lines
index
7de1b15c90b1..c4bccd3f5f17100644tools/testing/selftests/sgx/load.c +1
-1@@ -87,7 +87,7 @@ static bool encl_ioc_create(struct encl *encl)memset (secs,0 ,sizeof (*secs)); secs->ssa_frame_size =1 ; - secs->attributes =SGX_ATTR_MODE64BIT ; + secs->attributes =SGX_ATTR_MODE64BIT |SGX_ATTR_DEBUG ; secs->xfrm =3 ; secs->base = encl->encl_base; secs->size = encl->encl_size;Quoted 1 line
Regards, Vijay
I get also full pass with 2GB configuration (and also observed that kselftest runs much faster with this configuration).
But I looked at
sgx_alloc_epc_page()and saw this:if (list_empty(&sgx_active_page_list)) return ERR_PTR(-ENOMEM); if (!reclaim) { page = ERR_PTR(-EBUSY); break; }Quoted 1 line
In
sgx_vma_fault(), when running completely out of reclaimablepages, this
Quoted 12 lines
causes
VM_FAULT_SIGBUSreturned instead ofVM_FAULT_NOPAGE:entry = sgx_encl_load_page(encl, addr, vma->vm_flags); if (IS_ERR(entry)) { mutex_unlock(&encl->lock); if (PTR_ERR(entry) == -EBUSY) return VM_FAULT_NOPAGE; return VM_FAULT_SIGBUS; }Quoted 1 line
Not sure if those should be re-ordered that would keep the process
stuck up
Quoted 1 line
until there is something to reclaim. Now we use NOPAGE to address
situation
Quoted 1 line
when there is actually something to reclaim but because of locking
side of
Quoted 4 lines
things we pass reclaim=false to
sgx_alloc_epc_page().So this is kind of OOM behaviour how it works now instead of stalling processes.
Right, I looked at the original email at was really a page fault that was catched. The above theory cannot possibly hold, as the process does not exit with a bus error.
I looked next to
sgx_encl_eaug_page(), and found this:encl_page = sgx_encl_page_alloc(encl, addr - encl->base, secinfo_flags); if (IS_ERR(encl_page)) return VM_FAULT_OOM;This is AFAIK the only code path in
sgx_vma_fault()flow that allocates non-EPC memory, and the code paths where EPC allocation fails the result would be SIGBUS.So perhaps allocation is failing here. You could pretty easily trace allocations with bpftrace and kretprobe to see if this is what is happening, e.g. in one terminal:
Quoted 7 lines
Quoted 1 line
sudo bpftrace -e 'kr:sgx_encl_page_alloc /retval != 0/ {
printf("%d\n", retval); }'Should be
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(long)retval < 0/ {
printf("%d\n", retval); }'BR, Jarkko
I tried these probs and got following results when failure happens (not always happen on my device).
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(int64)retval <0 / {
printf("%X\n", retval); }'--> lots of negative values, I believe they are valid addresses in
unsigned long type. So I looked upIS_ERR_VALUEmacro and translated it in following probes.
Quoted 8 lines
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(uint64)retval >= (uint64)(-4095)/ {
printf("%X\n", retval); }'--> none triggered
sudo bpftrace -e 'kr:sgx_alloc_epc_page /(uint64)retval >= (uint64)(-4095)/ {
printf("%X\n", retval); }'--> FFFFFFF0 printed, which I believe is
-EBUSY.
Using this probe:
sudo bpftrace -e 'kr:sgx_encl_grow /(uint64)retval >= (uint64)(-4095)/ {printf("%lX\n", retval); }'
I found sgx_encl_grow could also return -EBUSY when allocating the VA page for the page to be EAUG'd. But we did not handle the case in sgx_encl_eaug_page, and follow changes seem to fix the problem with limited tests:
arch/x86/kernel/cpu/sgx/encl.c +4 -0
@@ -345,7 +345,11 @@ static vm_fault_t sgx_encl_eaug_page(struct vm_area_struct *vma, va_page =sgx_encl_grow (encl, false);if (IS_ERR (va_page)) + { +if (PTR_ERR (va_page) == -EBUSY ) + vmret =VM_FAULT_NOPAGE ; goto err_out_epc; + }if (va_page)list_add (&va_page->list, &encl->va_pages);
Will do more testing to verify. Let me know if the changes make sense.
Thanks
Haitao
Jarkko Sakkinen
maintainerQuoted 291 lines · Haitao Huang
Quoted 253 lines · Haitao Huang
Hi Jarkko
Quoted 228 lines · Jarkko Sakkinen
Quoted 220 lines · Jarkko Sakkinen
Quoted 9 lines · Jarkko Sakkinen
Quoted 8 lines · Vijay
Quoted 8 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 11:53 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected];Huang, Haitao
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
<[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 2 lines · Vijay
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 9:10 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>;[email protected]; Huang,
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftest
augment_via_eaccept_longQuoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
On Tue, Aug 09, 2022 at 01:45:35PM +0300, Jarkko
Quoted 1 line · Sakkinen
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
On Mon, Aug 08, 2022 at 06:29:13PM +0300, Jarkko
Quoted 1 line · Sakkinen
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
On Mon, Aug 08, 2022 at 01:00:54PM +0000, Dhanraj,
Quoted 7 lines · Vijay
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Monday, August 8, 2022 5:18 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette
<[email protected]>;Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftest
`augment_via_eaccept_long`On Thu, Aug 04, 2022 at 01:14:56PM -0700,
[email protected]Quoted 4 lines · wrote:
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
From: Vijay Dhanraj <[email protected]>
This commit adds a new test case which is same as
`augment_via_eaccept` but adds more numberof EPC pages to
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
stress test
`EAUG` via `EACCEPT`.Signed-off-by: Vijay Dhanraj
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Signed-off-by: Haitao Huang
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Hey, to reproduce the original issue: does it
reproduce on
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines
VM or should I run baremetal kernel?
BR, Jarkko
Hi Jarkko, The issue should be reproducible on
baremetal kernel.
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 2 lines
Thanks.
I need comment out other tests in order to make sane
out of this
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
:-)
Mentioning this because came into realization that
stress tests
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
should be IMHO moved each to a separate binary (so
that they can
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
be run separately). Just a note (TODO) to myself.
I'll work on this today again and *possibly* split
your test to
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
its own application to get a starting point for
forementioned.
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
I got
# RUN enclave.augment_via_eaccept_long ...
# main.c:1241:augment_via_eaccept_long:test enclave:
total_size=Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
8192,
seg->size= 8192 #main.c:1241:augment_via_eaccept_long:test enclave:
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
total_size= 12288,seg->size= 4096 # main.c:1241:augment_via_eaccept_long:test enclave:
total_size=Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
36864,
seg->size= 24576 #main.c:1241:augment_via_eaccept_long:test enclave:
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 2 lines
Quoted 2 lines
total_size= 40960,seg->size= 4096 # main.c:1259:augment_via_eaccept_long:mmaping pages at end ofenclave...
Quoted 1 line
Quoted 1 line
# main.c:1273:augment_via_eaccept_long:Entering
enclave to run
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
EACCEPT for each page of 8589934592 bytes may take a
while ...
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 9 lines
# OK enclave.augment_via_eaccept_long
The CPU used for testing was according to
/proc/cpuinfo:model name : Intel(R) Xeon(R) Gold 6338 CPU @ 2.00GHz
I have couple of queries:
1. Is it possible to get dmesg output?
I did check the dmesg output but couldn't find anything
related to the
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
failure. Just the general log messages.
Quoted 5 lines
Quoted 4 lines
2. Do I have to repeat the test multiple times, or does it occur unconditionaly?
I was able to repro every time but it was a bit sporadic
for Haitao.
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 2 lines
BR, Jarkko
Also, did you set the PRMRR size to 2GB per socket in
the BIOS? The
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
issue is only reproduced for oversubscribed scenario.
When I set my
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 1 line
PRMRR to 64GB per socket, I wasn't able to repro the issue.
I need to revisit this.
Quoted 8 lines
Quoted 8 lines
Quoted 6 lines
Can you simply run
test_sgxwith gdb and see where it hits?HOST_CFLAGShas apparently "-g" already.Quoted 1 line
Regards, Vijay
BR, Jarkko
I am able to repro the issue when I reduce the PRMRR to
2B/socket but not but not able to break on the assertion failure with gdb. I also enabled debug attribute in the secs but still no avail. Anything I am missing here?
Quoted 2 lines
Review
Quoted 2 lines
(quoted diff) +0
-0
b/tools/testing/selftests/sgx/load.cQuoted 28 lines
Quoted 13 lines
index
7de1b15c90b1..c4bccd3f5f17100644tools/testing/selftests/sgx/load.c +1
-1@@ -87,7 +87,7 @@ static bool encl_ioc_create(struct encl *encl)memset (secs,0 ,sizeof (*secs)); secs->ssa_frame_size =1 ; - secs->attributes =SGX_ATTR_MODE64BIT ; + secs->attributes =SGX_ATTR_MODE64BIT |SGX_ATTR_DEBUG ; secs->xfrm =3 ; secs->base = encl->encl_base; secs->size = encl->encl_size;Quoted 1 line
Regards, Vijay
I get also full pass with 2GB configuration (and also observed that kselftest runs much faster with this configuration).
But I looked at
sgx_alloc_epc_page()and saw this:if (list_empty(&sgx_active_page_list)) return ERR_PTR(-ENOMEM); if (!reclaim) { page = ERR_PTR(-EBUSY); break; }Quoted 1 line
In
sgx_vma_fault(), when running completely out of reclaimablepages, this
Quoted 12 lines
causes
VM_FAULT_SIGBUSreturned instead ofVM_FAULT_NOPAGE:entry = sgx_encl_load_page(encl, addr, vma->vm_flags); if (IS_ERR(entry)) { mutex_unlock(&encl->lock); if (PTR_ERR(entry) == -EBUSY) return VM_FAULT_NOPAGE; return VM_FAULT_SIGBUS; }Quoted 1 line
Not sure if those should be re-ordered that would keep the
process stuck up
Quoted 1 line
until there is something to reclaim. Now we use NOPAGE to
address situation
Quoted 1 line
when there is actually something to reclaim but because of
locking side of
Quoted 4 lines
things we pass reclaim=false to
sgx_alloc_epc_page().So this is kind of OOM behaviour how it works now instead of stalling processes.
Right, I looked at the original email at was really a page fault that was catched. The above theory cannot possibly hold, as the process does not exit with a bus error.
I looked next to
sgx_encl_eaug_page(), and found this:encl_page = sgx_encl_page_alloc(encl, addr - encl->base, secinfo_flags); if (IS_ERR(encl_page)) return VM_FAULT_OOM;This is AFAIK the only code path in
sgx_vma_fault()flow that allocates non-EPC memory, and the code paths where EPC allocation fails the result would be SIGBUS.Quoted 6 lines
So perhaps allocation is failing here. You could pretty easily trace allocations with bpftrace and kretprobe to see if this is what is happening, e.g. in one terminal:
sudo bpftrace -e 'kr:sgx_encl_page_alloc /retval != 0/ {
printf("%d\n", retval); }'Quoted 6 lines
Should be
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(long)retval < 0/ {
printf("%d\n", retval); }'BR, Jarkko
I tried these probs and got following results when failure happens (not always happen on my device).
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(int64)retval <0 / {
printf("%X\n", retval); }'--> lots of negative values, I believe they are valid addresses in unsigned long type. So I looked up
IS_ERR_VALUEmacro and translated it in following probes.sudo bpftrace -e 'kr:sgx_encl_page_alloc /(uint64)retval >= (uint64)(-4095)/ {
printf("%X\n", retval); }'Quoted 7 lines
--> none triggered
sudo bpftrace -e 'kr:sgx_alloc_epc_page /(uint64)retval >= (uint64)(-4095)/ {
printf("%X\n", retval); }'--> FFFFFFF0 printed, which I believe is
-EBUSY.Using this probe:
sudo bpftrace -e 'kr:sgx_encl_grow /(uint64)retval >= (uint64)(-4095)/ {printf("%lX\n", retval); }'I found
sgx_encl_growcould also return-EBUSYwhen allocating the VA page for the page to be EAUG'd.
But we did not handle the case insgx_encl_eaug_page, and follow changes seem to fix the problem with limited tests:arch/x86/kernel/cpu/sgx/encl.c +4
-0@@ -345,7 +345,11 @@ static vm_fault_t sgx_encl_eaug_page(structvm_area_struct *vma, va_page =sgx_encl_grow (encl, false);if (IS_ERR (va_page)) + { +if (PTR_ERR (va_page) == -EBUSY ) + vmret =VM_FAULT_NOPAGE ; goto err_out_epc; + }if (va_page)list_add(&va_page->list, &encl->va_pages);
Quoted 1 line
Will do more testing to verify. Let me know if the changes make sense.
It does make sense to me. EBUSY should be always result NOPAGE in any situation.
The reason for this is locking. We need to round-trip through a #PF trigger when out-of-EPC when running out of EPC because otherwise the code ends up into ABBA deadlock. I.e. allocator just starts the reclaimer thread and returns, which gives chance to reclaim some pages.
BR, Jarkko
Jarkko Sakkinen
maintainerQuoted 244 lines · Haitao Huang
Hi Jarkko
Quoted 226 lines · Jarkko Sakkinen
Quoted 218 lines · Jarkko Sakkinen
Quoted 9 lines · Jarkko Sakkinen
Quoted 8 lines · Vijay
Quoted 8 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 11:53 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected];Huang, Haitao
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
<[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longQuoted 2 lines · Vijay
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 9:10 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected];Huang,
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftest
augment_via_eaccept_longQuoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
On Tue, Aug 09, 2022 at 01:45:35PM +0300, Jarkko Sakkinen
Quoted 1 line · wrote:
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
On Mon, Aug 08, 2022 at 06:29:13PM +0300, Jarkko
Quoted 1 line · Sakkinen
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
On Mon, Aug 08, 2022 at 01:00:54PM +0000, Dhanraj,
Quoted 7 lines · Vijay
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Monday, August 8, 2022 5:18 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>;Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftestaugment_via_eaccept_longOn Thu, Aug 04, 2022 at 01:14:56PM -0700, [email protected]
Quoted 4 lines · wrote:
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
From: Vijay Dhanraj <[email protected]>
This commit adds a new test case which is same as
`augment_via_eaccept` but adds more number ofEPC pages to
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 1 line
stress test
EAUGviaEACCEPT.Quoted 2 lines
Signed-off-by: Vijay Dhanraj
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Signed-off-by: Haitao Huang
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Hey, to reproduce the original issue: does it
reproduce on
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines
VM or should I run baremetal kernel?
BR, Jarkko
Hi Jarkko, The issue should be reproducible on
baremetal kernel.
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 2 lines
Thanks.
I need comment out other tests in order to make sane out
of this
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
:-)
Mentioning this because came into realization that
stress tests
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
should be IMHO moved each to a separate binary (so that
they can
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
be run separately). Just a note (TODO) to myself.
I'll work on this today again and *possibly* split your
test to
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
its own application to get a starting point for
forementioned.
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
I got
# RUN enclave.augment_via_eaccept_long ...
# main.c:1241:augment_via_eaccept_long:test enclave:
total_size=Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
8192,
seg->size= 8192 #main.c:1241:augment_via_eaccept_long:test enclave:
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
total_size= 12288,seg->size= 4096 # main.c:1241:augment_via_eaccept_long:test enclave:
total_size=Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
36864,
seg->size= 24576 #main.c:1241:augment_via_eaccept_long:test enclave:
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 2 lines
Quoted 2 lines
total_size= 40960,seg->size= 4096 # main.c:1259:augment_via_eaccept_long:mmaping pages at end ofenclave...
Quoted 4 lines
Quoted 4 lines
# main.c:1273:augment_via_eaccept_long:Entering enclave to run EACCEPT for each page of 8589934592 bytes may take a while ...
# OK enclave.augment_via_eaccept_longQuoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 7 lines
The CPU used for testing was according to
/proc/cpuinfo:model name : Intel(R) Xeon(R) Gold 6338 CPU @ 2.00GHz
I have couple of queries:
1. Is it possible to get dmesg output?
I did check the dmesg output but couldn't find anything
related to the
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
failure. Just the general log messages.
Quoted 5 lines
Quoted 4 lines
2. Do I have to repeat the test multiple times, or does it occur unconditionaly?
I was able to repro every time but it was a bit sporadic for
Haitao.
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 2 lines
BR, Jarkko
Also, did you set the PRMRR size to 2GB per socket in the
BIOS? The
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
issue is only reproduced for oversubscribed scenario. When I
set my
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 1 line
PRMRR to 64GB per socket, I wasn't able to repro the issue.
I need to revisit this.
Quoted 8 lines
Quoted 8 lines
Quoted 6 lines
Can you simply run
test_sgxwith gdb and see where it hits?HOST_CFLAGShas apparently "-g" already.Quoted 1 line
Regards, Vijay
BR, Jarkko
I am able to repro the issue when I reduce the PRMRR to
2B/socket but not but not able to break on the assertion failure with gdb. I also enabled debug attribute in the secs but still no avail. Anything I am missing here?
Quoted 2 lines
Review
Quoted 2 lines
(quoted diff) +0
-0
b/tools/testing/selftests/sgx/load.cQuoted 28 lines
Quoted 13 lines
index
7de1b15c90b1..c4bccd3f5f17100644tools/testing/selftests/sgx/load.c +1
-1@@ -87,7 +87,7 @@ static bool encl_ioc_create(struct encl *encl)memset (secs,0 ,sizeof (*secs)); secs->ssa_frame_size =1 ; - secs->attributes =SGX_ATTR_MODE64BIT ; + secs->attributes =SGX_ATTR_MODE64BIT |SGX_ATTR_DEBUG ; secs->xfrm =3 ; secs->base = encl->encl_base; secs->size = encl->encl_size;Quoted 1 line
Regards, Vijay
I get also full pass with 2GB configuration (and also observed that kselftest runs much faster with this configuration).
But I looked at
sgx_alloc_epc_page()and saw this:if (list_empty(&sgx_active_page_list)) return ERR_PTR(-ENOMEM); if (!reclaim) { page = ERR_PTR(-EBUSY); break; }Quoted 1 line
In
sgx_vma_fault(), when running completely out of reclaimablepages, this
Quoted 12 lines
causes
VM_FAULT_SIGBUSreturned instead ofVM_FAULT_NOPAGE:entry = sgx_encl_load_page(encl, addr, vma->vm_flags); if (IS_ERR(entry)) { mutex_unlock(&encl->lock); if (PTR_ERR(entry) == -EBUSY) return VM_FAULT_NOPAGE; return VM_FAULT_SIGBUS; }Quoted 1 line
Not sure if those should be re-ordered that would keep the process
stuck up
Quoted 1 line
until there is something to reclaim. Now we use NOPAGE to address
situation
Quoted 1 line
when there is actually something to reclaim but because of locking
side of
Quoted 4 lines
things we pass reclaim=false to
sgx_alloc_epc_page().So this is kind of OOM behaviour how it works now instead of stalling processes.
Right, I looked at the original email at was really a page fault that was catched. The above theory cannot possibly hold, as the process does not exit with a bus error.
I looked next to
sgx_encl_eaug_page(), and found this:encl_page = sgx_encl_page_alloc(encl, addr - encl->base, secinfo_flags); if (IS_ERR(encl_page)) return VM_FAULT_OOM;This is AFAIK the only code path in
sgx_vma_fault()flow that allocates non-EPC memory, and the code paths where EPC allocation fails the result would be SIGBUS.Quoted 6 lines
So perhaps allocation is failing here. You could pretty easily trace allocations with bpftrace and kretprobe to see if this is what is happening, e.g. in one terminal:
sudo bpftrace -e 'kr:sgx_encl_page_alloc /retval != 0/ {
printf("%d\n", retval); }'Quoted 6 lines
Should be
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(long)retval < 0/ {
printf("%d\n", retval); }'BR, Jarkko
I tried these probs and got following results when failure happens (not always happen on my device).
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(int64)retval <0 / {
printf("%X\n", retval); }'
Quoted 6 lines
--> lots of negative values, I believe they are valid addresses in unsigned long type. So I looked up
IS_ERR_VALUEmacro and translated it in following probes.sudo bpftrace -e 'kr:sgx_encl_page_alloc /(uint64)retval >= (uint64)(-4095)/ {
printf("%X\n", retval); }'
Thank you for refining the probe.
I'll add it to my collection:
https://github.com/jarkkojs/bpftrace-sgx
Despite the repository name it contains probes both for SGX and AMD-SNP, and also TDX and ARMv9 in future once I have ways to test them.
Quoted 7 lines
--> none triggered
sudo bpftrace -e 'kr:sgx_alloc_epc_page /(uint64)retval >= (uint64)(-4095)/ {
printf("%X\n", retval); }'--> FFFFFFF0 printed, which I believe is
-EBUSY.
EBUSY should be fine, it will be retrigger the #PF handler through round-trip.
Did you still encounter segfaults?
BR, Jarkko
Quoted 277 lines · Jarkko Sakkinen
Quoted 252 lines · Haitao Huang
Hi Jarkko
Quoted 11 lines · Jarkko Sakkinen
Quoted 10 lines · Jarkko Sakkinen
Quoted 9 lines · Jarkko Sakkinen
Quoted 8 lines · Vijay
Quoted 8 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 11:53 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected];Quoted 3 lines
Quoted 3 lines
Huang, Haitao
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
<[email protected]>
Subject: Re: [PATCH] Add SGX selftest
augment_via_eaccept_longQuoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
On Tue, Aug 09, 2022 at 05:08:21PM +0000, Dhanraj, Vijay
Quoted 31 lines · wrote:
Quoted 31 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Tuesday, August 9, 2022 9:10 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette <[email protected]>; [email protected];Huang,
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftest
augment_via_eaccept_longQuoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
On Tue, Aug 09, 2022 at 01:45:35PM +0300, Jarkko Sakkinen
Quoted 1 line · wrote:
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
On Mon, Aug 08, 2022 at 06:29:13PM +0300, Jarkko
Quoted 1 line · Sakkinen
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
On Mon, Aug 08, 2022 at 01:00:54PM +0000, Dhanraj,
Quoted 7 lines · Vijay
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
-----Original Message-----
From: Jarkko Sakkinen <[email protected]>
Sent: Monday, August 8, 2022 5:18 AM
To: Dhanraj, Vijay <[email protected]>
Cc: [email protected]; Chatre, Reinette
<[email protected]>;Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Huang, Haitao <[email protected]>
Subject: Re: [PATCH] Add SGX selftest
`augment_via_eaccept_long`Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
On Thu, Aug 04, 2022 at 01:14:56PM -0700,
[email protected]Quoted 3 lines · wrote:
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
From: Vijay Dhanraj <[email protected]>
This commit adds a new test case which is same
as
Quoted 44 lines
Quoted 44 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
`augment_via_eaccept` but adds more number of
EPC pages to
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
stress test
`EAUG` via `EACCEPT`.Signed-off-by: Vijay Dhanraj
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Signed-off-by: Haitao Huang
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Hey, to reproduce the original issue: does it
reproduce on
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines
VM or should I run baremetal kernel?
BR, Jarkko
Hi Jarkko, The issue should be reproducible on
baremetal kernel.
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 2 lines
Thanks.
I need comment out other tests in order to make sane out
of this
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
:-)
Mentioning this because came into realization that
stress tests
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
should be IMHO moved each to a separate binary (so that
they can
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
be run separately). Just a note (TODO) to myself.
I'll work on this today again and *possibly* split your
test to
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
its own application to get a starting point for
forementioned.
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
I got
# RUN enclave.augment_via_eaccept_long ...
# main.c:1241:augment_via_eaccept_long:test enclave:
total_size=Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
8192,
seg->size= 8192 #main.c:1241:augment_via_eaccept_long:test enclave:
Quoted 8 lines
Quoted 8 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
total_size= 12288,seg->size= 4096 # main.c:1241:augment_via_eaccept_long:test enclave:
total_size=Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
36864,
seg->size= 24576 #main.c:1241:augment_via_eaccept_long:test enclave:
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
total_size= 40960,seg->size= 4096 # main.c:1259:augment_via_eaccept_long:mmaping pages at endof
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
enclave...
Quoted 1 line
Quoted 1 line
# main.c:1273:augment_via_eaccept_long:Entering enclave
to run
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
EACCEPT for each page of 8589934592 bytes may take a
while ...
Quoted 56 lines
Quoted 56 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 9 lines
# OK enclave.augment_via_eaccept_long
The CPU used for testing was according to
/proc/cpuinfo:model name : Intel(R) Xeon(R) Gold 6338 CPU @ 2.00GHz
I have couple of queries:
1. Is it possible to get dmesg output?
I did check the dmesg output but couldn't find anything
related to the
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
failure. Just the general log messages.
Quoted 5 lines
Quoted 4 lines
2. Do I have to repeat the test multiple times, or does it occur unconditionaly?
I was able to repro every time but it was a bit sporadic for
Haitao.
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 2 lines
BR, Jarkko
Also, did you set the PRMRR size to 2GB per socket in the
BIOS? The
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
issue is only reproduced for oversubscribed scenario. When I
set my
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 1 line
PRMRR to 64GB per socket, I wasn't able to repro the issue.
I need to revisit this.
Quoted 8 lines
Quoted 8 lines
Quoted 6 lines
Can you simply run
test_sgxwith gdb and see where it hits?HOST_CFLAGShas apparently "-g" already.Quoted 1 line
Regards, Vijay
BR, Jarkko
I am able to repro the issue when I reduce the PRMRR to
2B/socket but not but not able to break on the assertion failure with gdb. I also enabled debug attribute in the secs but still no avail. Anything I am missing here?
Quoted 2 lines
Review
Quoted 2 lines
(quoted diff) +0
-0
b/tools/testing/selftests/sgx/load.cQuoted 13 lines
Quoted 13 lines
index
7de1b15c90b1..c4bccd3f5f17100644tools/testing/selftests/sgx/load.c +1
-1@@ -87,7 +87,7 @@ static bool encl_ioc_create(struct encl *encl)memset (secs,0 ,sizeof (*secs)); secs->ssa_frame_size =1 ; - secs->attributes =SGX_ATTR_MODE64BIT ; + secs->attributes =SGX_ATTR_MODE64BIT |SGX_ATTR_DEBUG ; secs->xfrm =3 ; secs->base = encl->encl_base; secs->size = encl->encl_size;Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
Regards, Vijay
I get also full pass with 2GB configuration (and also observed
that
Quoted 27 lines
Quoted 27 lines
Quoted 12 lines
kselftest runs much faster with this configuration).
But I looked at
sgx_alloc_epc_page()and saw this:if (list_empty(&sgx_active_page_list)) return ERR_PTR(-ENOMEM); if (!reclaim) { page = ERR_PTR(-EBUSY); break; }Quoted 1 line
In
sgx_vma_fault(), when running completely out of reclaimablepages, this
Quoted 13 lines
causes
VM_FAULT_SIGBUSreturned instead ofVM_FAULT_NOPAGE:entry = sgx_encl_load_page(encl, addr, vma->vm_flags); if (IS_ERR(entry)) { mutex_unlock(&encl->lock); if (PTR_ERR(entry) == -EBUSY) return VM_FAULT_NOPAGE; return VM_FAULT_SIGBUS; }Not sure if those should be re-ordered that would keep the process
Quoted 8 lines
Quoted 8 lines
stuck up
Quoted 1 line
until there is something to reclaim. Now we use NOPAGE to address
situation
Quoted 1 line
when there is actually something to reclaim but because of locking
side of
Quoted 3 lines
things we pass reclaim=false to
sgx_alloc_epc_page().So this is kind of OOM behaviour how it works now instead of
stalling
Quoted 24 lines
Quoted 17 lines
Quoted 1 line
processes.
Right, I looked at the original email at was really a page fault that was catched. The above theory cannot possibly hold, as the process does not exit with a bus error.
I looked next to
sgx_encl_eaug_page(), and found this:encl_page = sgx_encl_page_alloc(encl, addr - encl->base, secinfo_flags); if (IS_ERR(encl_page)) return VM_FAULT_OOM;This is AFAIK the only code path in
sgx_vma_fault()flow that allocates non-EPC memory, and the code paths where EPC allocation fails the result would be SIGBUS.Quoted 6 lines
So perhaps allocation is failing here. You could pretty easily trace allocations with bpftrace and kretprobe to see if this is what is happening, e.g. in one terminal:
sudo bpftrace -e 'kr:sgx_encl_page_alloc /retval != 0/ {
printf("%d\n", retval); }'Quoted 6 lines
Should be
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(long)retval < 0/ {
printf("%d\n", retval); }'BR, Jarkko
I tried these probs and got following results when failure happens (not always happen on my device).
sudo bpftrace -e 'kr:sgx_encl_page_alloc /(int64)retval <0 / {
printf("%X\n", retval); }'Quoted 7 lines
--> lots of negative values, I believe they are valid addresses in unsigned
long type. So I looked upIS_ERR_VALUEmacro and translated it in following probes.sudo bpftrace -e 'kr:sgx_encl_page_alloc /(uint64)retval >= (uint64)(-4095)/ {
printf("%X\n", retval); }'Thank you for refining the probe.
I'll add it to my collection:
https://github.com/jarkkojs/bpftrace-sgx
Despite the repository name it contains probes both for SGX and AMD-SNP, and also TDX and ARMv9 in future once I have ways to test them.
Quoted 6 lines
--> none triggered
sudo bpftrace -e 'kr:sgx_alloc_epc_page /(uint64)retval >= (uint64)(-4095)/ {
printf("%X\n", retval); }'
Quoted 7 lines
Quoted 1 line
--> FFFFFFF0 printed, which I believe is
-EBUSY.EBUSY should be fine, it will be retrigger the #PF handler through round-trip.
Did you still encounter segfaults?
Yes, but I think it was -EBUSY returned from sgx_encl_grow that was not handled and caused the segfault. See the patch.
Thanks
Haitao