selftests/sgx: Test a large dynamic heap
Jarkko Sakkinen
maintainercover selftests/sgx: Test a large dynamic heap
Implement functionality to define a dynamic heap for the test enclave. Then, implement a new augmentation test for a large dynamic heap.
1/5 is critical for v6.0. More information in the patch change log.
Haitao Huang (1):
selftests/sgx: Retry the ioctl()'s returned with EAGAIN
Jarkko Sakkinen (3):
selftests/sgx: Move ENCL_HEAP_SIZE_DEFAULT to main.c selftests/sgx: Use encl->encl_size in sigstruct.c selftests/sgx: Include the dynamic heap size to the ELRANGE
calculation
Vijay Dhanraj (1):
selftests/sgx: Add SGX selftest augment_via_eaccept_long
tools/testing/selftests/sgx/load.c | 8 +- tools/testing/selftests/sgx/main.c | 179 +++++++++++++++++++++--- tools/testing/selftests/sgx/main.h | 6 +- tools/testing/selftests/sgx/sigstruct.c | 8 +- 4 files changed, 172 insertions(+), 29 deletions(-)
Signature · 1 line
2.37.2
Jarkko Sakkinen
maintainerPATCH 1/5 selftests/sgx: Retry the ioctl()'s returned with EAGAIN
From: Haitao Huang <[email protected]>
For EMODT and EREMOVE ioctl()'s with a large range, kernel may not finish in one shot and return EAGAIN error code and count of bytes of EPC pages on that operations are finished successfully.
Change the unclobbered_vdso_oversubscribed_remove test to rerun the ioctl()'s in a loop, updating offset and length using the byte count returned in each iteration.
---
v3:
* Added a fixes tag. The bug is in v6.0 patches.
* Added my tested-by (the bug reproduced in my NUC often). v2:
* Changed branching in EAGAIN condition so that else branch is not required.
* Addressed Reinette's feedback:
--- tools/testing/selftests/sgx/main.c | 42 ++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 8 deletions(-)
tools/testing/selftests/sgx/main.c +34 -8
@@ -390,6 +390,7 @@ TEST_F_TIMEOUT(enclave, unclobbered_vdso_oversubscribed_remove, 900)struct encl_segment *heap;unsigned long total_mem;int ret, errno_save; +unsigned long count;unsigned long addr;unsigned long i;
@@ -453,16 +454,30 @@ TEST_F_TIMEOUT(enclave, unclobbered_vdso_oversubscribed_remove, 900) modt_ioc.offset = heap->offset; modt_ioc.length = heap->size; modt_ioc.page_type =SGX_PAGE_TYPE_TRIM ; - + count =0 ;TH_LOG ("Changing type of %zd bytes to trimmed may take a while ..." , heap->size); - ret =ioctl (self->encl.fd,SGX_IOC_ENCLAVE_MODIFY_TYPES , &modt_ioc); - errno_save = ret ==-1 ? errno :0 ; +do { + ret =ioctl (self->encl.fd,SGX_IOC_ENCLAVE_MODIFY_TYPES , &modt_ioc); + + errno_save = ret ==-1 ? errno :0 ; +if (errno_save !=EAGAIN ) +break ; + +EXPECT_EQ (modt_ioc.result,0 ); + + count += modt_ioc.count; + modt_ioc.offset += modt_ioc.count; + modt_ioc.length -= modt_ioc.count; + modt_ioc.result =0 ; + modt_ioc.count =0 ; + }while (modt_ioc.length !=0 );EXPECT_EQ (ret,0 );EXPECT_EQ (errno_save,0 );EXPECT_EQ (modt_ioc.result,0 ); -EXPECT_EQ (modt_ioc.count, heap->size); + count += modt_ioc.count; +EXPECT_EQ (count, heap->size);/* EACCEPT all removed pages. */ addr = self->encl.encl_base + heap->offset;
@@ -490,15 +505,26 @@ TEST_F_TIMEOUT(enclave, unclobbered_vdso_oversubscribed_remove, 900) remove_ioc.offset = heap->offset; remove_ioc.length = heap->size; - + count =0 ;TH_LOG ("Removing %zd bytes from enclave may take a while ..." , heap->size); - ret =ioctl (self->encl.fd,SGX_IOC_ENCLAVE_REMOVE_PAGES , &remove_ioc); - errno_save = ret ==-1 ? errno :0 ; +do { + ret =ioctl (self->encl.fd,SGX_IOC_ENCLAVE_REMOVE_PAGES , &remove_ioc); + + errno_save = ret ==-1 ? errno :0 ; +if (errno_save !=EAGAIN ) +break ; + + count += remove_ioc.count; + remove_ioc.offset += remove_ioc.count; + remove_ioc.length -= remove_ioc.count; + remove_ioc.count =0 ; + }while (remove_ioc.length !=0 );EXPECT_EQ (ret,0 );EXPECT_EQ (errno_save,0 ); -EXPECT_EQ (remove_ioc.count, heap->size); + count += remove_ioc.count; +EXPECT_EQ (count, heap->size); }TEST_F (enclave, clobbered_vdso)
Signature · 1 line
2.37.2
Jarkko Sakkinen
maintainerPATCH 2/5 selftests/sgx: Move ENCL_HEAP_SIZE_DEFAULT to main.c
In reply to Jarkko SakkinenMove ENCL_HEAP_SIZE_DEFAULT to main.c because all the other constants are also there, and it is only used locally there.
--- tools/testing/selftests/sgx/main.c | 1 + tools/testing/selftests/sgx/main.h | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-)
tools/testing/selftests/sgx/main.c +1 -0
@@ -21,6 +21,7 @@#include "../kselftest_harness.h" #include "main.h" +static const size_t ENCL_HEAP_SIZE_DEFAULT =PAGE_SIZE ;static const uint64_t MAGIC =0x1122334455667788ULL ;static const uint64_t MAGIC2 =0x8877665544332211ULL ;vdso_sgx_enter_enclave_t vdso_sgx_enter_enclave;
tools/testing/selftests/sgx/main.h +0 -2
@@ -6,8 +6,6 @@ #ifndef MAIN_H #define MAIN_H -#define ENCL_HEAP_SIZE_DEFAULT 4096 - struct encl_segment { void *src; off_t offset;
Signature · 1 line
2.37.2
Jarkko Sakkinen
maintainerPATCH 3/5 selftests/sgx: Use encl->encl_size in sigstruct.c
In reply to Jarkko SakkinenThe final enclave address range (referred as ELRANGE in Intel SDM) calculation is a reminiscent of signing tool being a separate command-line utility, and sigstruct being produced during the compilation. Given that nowadays the sigstruct is calculated on-fly, use the readily calculated encl->encl_size instead, in order to remove duplicate code.
--- tools/testing/selftests/sgx/load.c | 5 +++-- tools/testing/selftests/sgx/main.h | 1 - tools/testing/selftests/sgx/sigstruct.c | 8 ++------ 3 files changed, 5 insertions(+), 9 deletions(-)
tools/testing/selftests/sgx/load.c +3 -2
@@ -174,6 +174,7 @@ uint64_t encl_get_entry(struct encl *encl, const char *symbol)bool encl_load (const char *path,struct encl *encl,unsigned long heap_size) {const char device_path[] ="/dev/sgx_enclave" ; +unsigned long contents_size;struct encl_segment *seg;Elf64_Phdr *phdr_tbl;off_t src_offset;
@@ -298,9 +299,9 @@ bool encl_load(const char *path, struct encl *encl, unsigned long heap_size)if (seg->src ==MAP_FAILED ) goto err; - encl->src_size = encl->segment_tbl[j].offset + encl->segment_tbl[j].size; + contents_size = encl->segment_tbl[j].offset + encl->segment_tbl[j].size; -for (encl->encl_size =4096 ; encl->encl_size < encl->src_size; ) +for (encl->encl_size =4096 ; encl->encl_size < contents_size; ) encl->encl_size <<=1 ;return true;
tools/testing/selftests/sgx/main.h +0 -1
@@ -20,7 +20,6 @@ struct encl {void *bin;off_t bin_size;void *src; -size_t src_size;size_t encl_size;off_t encl_base;unsigned int nr_segments;
tools/testing/selftests/sgx/sigstruct.c +2 -6
@@ -218,13 +218,9 @@ struct mrecreate { } __attribute__((__packed__)); -static bool mrenclave_ecreate (EVP_MD_CTX *ctx,uint64_t blob_size) +static bool mrenclave_ecreate (EVP_MD_CTX *ctx,uint64_t encl_size) {struct mrecreate mrecreate; -uint64_t encl_size; - -for (encl_size =0x1000 ; encl_size < blob_size; ) - encl_size <<=1 ;memset (&mrecreate,0 ,sizeof (mrecreate)); mrecreate.tag =MRECREATE ;
@@ -349,7 +345,7 @@ bool encl_measure(struct encl *encl)if (!ctx) goto err; -if (!mrenclave_ecreate (ctx, encl->src_size)) +if (!mrenclave_ecreate (ctx, encl->encl_size)) goto err;for (i =0 ; i < encl->nr_segments; i++) {
Signature · 1 line
2.37.2
Jarkko Sakkinen
maintainerPATCH 5/5 selftests/sgx: Add SGX selftest augment_via_eaccept_long
In reply to Jarkko SakkinenFrom: Vijay Dhanraj <[email protected]>
Add a new test case which is same as augment_via_eaccept but adds a larger number of EPC pages to stress test EAUG via EACCEPT.
---
v7:
- Contains now only the test case. Support for dynamic heap is prepared in prepending patches.
v6:
- Address Reinette's feedback:
https://lore.kernel.org/linux-sgx/Yw6%2FiTzSdSw%2FY%[email protected]/
v5:
- Add the klog dump and sysctl option to the commit message.
v4:
- Explain expectations for dirty_page_list in the function header, instead of an inline comment.
- Improve commit message to explain the conditions better.
- Return the number of pages left dirty to ksgxd() and print warning after the 2nd call, if there are any.
v3:
- Remove WARN_ON().
- Tuned comments and the commit message a bit.
v2:
- Replaced WARN_ON() with optional pr_info() inside __sgx_sanitize_pages().
- Rewrote the commit message.
- Added the fixes tag.
--- tools/testing/selftests/sgx/main.c | 112 ++++++++++++++++++++++++++++- 1 file changed, 111 insertions(+), 1 deletion(-)
tools/testing/selftests/sgx/main.c +111 -1
@@ -23,8 +23,15 @@static const size_t ENCL_HEAP_SIZE_DEFAULT =PAGE_SIZE ;static const size_t ENCL_DYNAMIC_SIZE_DEFAULT =PAGE_SIZE ; +/* +* The size was chosen based on a bug report: +* Message-ID: <DM8PR11MB55912A7F47A84EC9913A6352F6999@DM8PR11MB5591.namprd11.prod.outlook.com> +*/ +static const size_t ENCL_DYNAMIC_SIZE_LONG =8L *1024L *1024L *1024L ; +static const unsigned long TIMEOUT_DEFAULT =900 ;static const uint64_t MAGIC =0x1122334455667788ULL ;static const uint64_t MAGIC2 =0x8877665544332211ULL ; +vdso_sgx_enter_enclave_t vdso_sgx_enter_enclave;/*
@@ -388,7 +395,7 @@ TEST_F(enclave, unclobbered_vdso_oversubscribed)EXPECT_EQ (self->run.user_data,0 ); } -TEST_F_TIMEOUT(enclave, unclobbered_vdso_oversubscribed_remove, 900) +TEST_F_TIMEOUT(enclave, unclobbered_vdso_oversubscribed_remove, TIMEOUT_DEFAULT) {struct sgx_enclave_remove_pages remove_ioc;struct sgx_enclave_modify_types modt_ioc;
@@ -1248,6 +1255,109 @@ TEST_F(enclave, augment_via_eaccept)munmap (addr,PAGE_SIZE ); } +/* +* Test for the addition of large number of pages to an initialized enclave via +* a pre-emptive run of EACCEPT on every page to be added. +*/ +TEST_F_TIMEOUT(enclave, augment_via_eaccept_long, TIMEOUT_DEFAULT) +{ +struct encl_op_get_from_addr get_addr_op; +struct encl_op_put_to_addr put_addr_op; +struct encl_op_eaccept eaccept_op; +size_t total_size =0 ; +unsigned long i; +void *addr; + +if (!sgx2_supported ()) +SKIP (return,"SGX2 not supported" ); + +ASSERT_TRUE (setup_test_encl_dynamic (ENCL_HEAP_SIZE_DEFAULT , +ENCL_DYNAMIC_SIZE_LONG , + &self->encl, _metadata)); + +memset (&self->run,0 ,sizeof (self->run)); + self->run.tcs = self->encl.encl_base; + +for (i =0 ; i < self->encl.nr_segments; i++) { +struct encl_segment *seg = &self->encl.segment_tbl[i]; + + total_size += seg->size; + } + +/* +* mmap() every page at end of existing enclave to be used for +* EDMM. +*/ + addr =mmap ((void *)self->encl.encl_base + total_size,ENCL_DYNAMIC_SIZE_LONG , +PROT_READ |PROT_WRITE |PROT_EXEC ,MAP_SHARED |MAP_FIXED , + self->encl.fd,0 ); +EXPECT_NE (addr,MAP_FAILED ); + + self->run.exception_vector =0 ; + self->run.exception_error_code =0 ; + self->run.exception_addr =0 ; + +/* +* Run EACCEPT on every page to trigger the #PF->EAUG->EACCEPT(again +* without a #PF). All should be transparent to userspace. +*/ + eaccept_op.flags =SGX_SECINFO_R |SGX_SECINFO_W |SGX_SECINFO_REG |SGX_SECINFO_PENDING ; + eaccept_op.ret =0 ; + eaccept_op.header.type =ENCL_OP_EACCEPT ; + +for (i =0 ; i <ENCL_DYNAMIC_SIZE_LONG ; i +=4096 ) { + eaccept_op.epc_addr = (uint64_t )(addr + i); + +EXPECT_EQ (ENCL_CALL (&eaccept_op, &self->run, true),0 ); +if (self->run.exception_vector ==14 && + self->run.exception_error_code ==4 && + self->run.exception_addr == self->encl.encl_base) { +munmap (addr,ENCL_DYNAMIC_SIZE_LONG ); +SKIP (return,"Kernel does not support adding pages to initialized enclave" ); + } + +EXPECT_EQ (self->run.exception_vector,0 ); +EXPECT_EQ (self->run.exception_error_code,0 ); +EXPECT_EQ (self->run.exception_addr,0 ); +ASSERT_EQ (eaccept_op.ret,0 ); +ASSERT_EQ (self->run.function,EEXIT ); + } + +/* +* Pool of pages were successfully added to enclave. Perform sanity +* check on first page of the pool only to ensure data can be written +* to and read from a dynamically added enclave page. +*/ + put_addr_op.value =MAGIC ; + put_addr_op.addr = (unsigned long )addr; + put_addr_op.header.type =ENCL_OP_PUT_TO_ADDRESS ; + +EXPECT_EQ (ENCL_CALL (&put_addr_op, &self->run, true),0 ); + +EXPECT_EEXIT (&self->run); +EXPECT_EQ (self->run.exception_vector,0 ); +EXPECT_EQ (self->run.exception_error_code,0 ); +EXPECT_EQ (self->run.exception_addr,0 ); + +/* +* Read memory from newly added page that was just written to, +* confirming that data previously written (MAGIC) is present. +*/ + get_addr_op.value =0 ; + get_addr_op.addr = (unsigned long )addr; + get_addr_op.header.type =ENCL_OP_GET_FROM_ADDRESS ; + +EXPECT_EQ (ENCL_CALL (&get_addr_op, &self->run, true),0 ); + +EXPECT_EQ (get_addr_op.value,MAGIC ); +EXPECT_EEXIT (&self->run); +EXPECT_EQ (self->run.exception_vector,0 ); +EXPECT_EQ (self->run.exception_error_code,0 ); +EXPECT_EQ (self->run.exception_addr,0 ); + +munmap (addr,ENCL_DYNAMIC_SIZE_LONG ); +} +/* * SGX2 page type modification test in two phases: * Phase 1:
Signature · 1 line
2.37.2
Jarkko Sakkinen
maintainerPATCH 4/5 selftests/sgx: Include the dynamic heap size to the ELRANGE calculation
In reply to Jarkko SakkinenWhen calculating ELRANGE, i.e. the address range defined for an enclave, and represented by encl->encl_size, also dynamic memory should be taken into account. Implement setup_test_encl_dynamic() with dynamic_size parameter for the dynamic heap size, and use it in 'augment_via_eaccept' and 'augment' tests.
--- tools/testing/selftests/sgx/load.c | 5 +++-- tools/testing/selftests/sgx/main.c | 24 +++++++++++++++++------- tools/testing/selftests/sgx/main.h | 3 ++- 3 files changed, 22 insertions(+), 10 deletions(-)
tools/testing/selftests/sgx/load.c +3 -2
@@ -171,7 +171,8 @@ uint64_t encl_get_entry(struct encl *encl, const char *symbol)return 0 ; } -bool encl_load (const char *path,struct encl *encl,unsigned long heap_size) +bool encl_load (const char *path,struct encl *encl,unsigned long heap_size, +unsigned long dynamic_size) {const char device_path[] ="/dev/sgx_enclave" ;unsigned long contents_size;
@@ -299,7 +300,7 @@ bool encl_load(const char *path, struct encl *encl, unsigned long heap_size)if (seg->src ==MAP_FAILED ) goto err; - contents_size = encl->segment_tbl[j].offset + encl->segment_tbl[j].size; + contents_size = encl->segment_tbl[j].offset + encl->segment_tbl[j].size + dynamic_size;for (encl->encl_size =4096 ; encl->encl_size < contents_size; ) encl->encl_size <<=1 ;
tools/testing/selftests/sgx/main.c +17 -7
@@ -22,6 +22,7 @@#include "main.h" static const size_t ENCL_HEAP_SIZE_DEFAULT =PAGE_SIZE ; +static const size_t ENCL_DYNAMIC_SIZE_DEFAULT =PAGE_SIZE ;static const uint64_t MAGIC =0x1122334455667788ULL ;static const uint64_t MAGIC2 =0x8877665544332211ULL ;vdso_sgx_enter_enclave_t vdso_sgx_enter_enclave;
@@ -173,8 +174,8 @@ FIXTURE(enclave) {struct sgx_enclave_run run; }; -static bool setup_test_encl (unsigned long heap_size,struct encl *encl, -struct __test_metadata *_metadata) +static bool setup_test_encl_dynamic (unsigned long heap_size,unsigned long dynamic_size, +struct encl *encl,struct __test_metadata *_metadata) {Elf64_Sym *sgx_enter_enclave_sym =NULL ;struct vdso_symtab symtab;
@@ -184,7 +185,7 @@ static bool setup_test_encl(unsigned long heap_size, struct encl *encl,unsigned int i;void *addr; -if (!encl_load ("test_encl.elf" , encl, heap_size)) { +if (!encl_load ("test_encl.elf" , encl, heap_size, dynamic_size)) {encl_delete (encl);TH_LOG ("Failed to load the test enclave." );return false;
@@ -251,6 +252,12 @@ static bool setup_test_encl(unsigned long heap_size, struct encl *encl,return false; } +static bool setup_test_encl (unsigned long heap_size,struct encl *encl, +struct __test_metadata *_metadata) +{ +return setup_test_encl_dynamic (heap_size,0 , encl, _metadata); +} +FIXTURE_SETUP (enclave) { }
@@ -1013,7 +1020,9 @@ TEST_F(enclave, augment)if (!sgx2_supported ())SKIP (return,"SGX2 not supported" ); -ASSERT_TRUE (setup_test_encl (ENCL_HEAP_SIZE_DEFAULT , &self->encl, _metadata)); +ASSERT_TRUE (setup_test_encl_dynamic (ENCL_HEAP_SIZE_DEFAULT , +ENCL_DYNAMIC_SIZE_DEFAULT , + &self->encl, _metadata));memset (&self->run,0 ,sizeof (self->run)); self->run.tcs = self->encl.encl_base;
@@ -1143,7 +1152,9 @@ TEST_F(enclave, augment_via_eaccept)if (!sgx2_supported ())SKIP (return,"SGX2 not supported" ); -ASSERT_TRUE (setup_test_encl (ENCL_HEAP_SIZE_DEFAULT , &self->encl, _metadata)); +ASSERT_TRUE (setup_test_encl_dynamic (ENCL_HEAP_SIZE_DEFAULT , +ENCL_DYNAMIC_SIZE_DEFAULT , + &self->encl, _metadata));memset (&self->run,0 ,sizeof (self->run)); self->run.tcs = self->encl.encl_base;
@@ -1264,8 +1275,7 @@ TEST_F(enclave, tcs_create)int errno_save;int ret, i; -ASSERT_TRUE (setup_test_encl (ENCL_HEAP_SIZE_DEFAULT , &self->encl, - _metadata)); +ASSERT_TRUE (setup_test_encl (ENCL_HEAP_SIZE_DEFAULT , &self->encl, _metadata));memset (&self->run,0 ,sizeof (self->run)); self->run.tcs = self->encl.encl_base;
tools/testing/selftests/sgx/main.h +2 -1
@@ -32,7 +32,8 @@ extern unsigned char sign_key[];extern unsigned char sign_key_end[];void encl_delete (struct encl *ctx); -bool encl_load (const char *path,struct encl *encl,unsigned long heap_size); +bool encl_load (const char *path,struct encl *encl,unsigned long heap_size, +unsigned long dynamic_size);bool encl_measure (struct encl *encl);bool encl_build (struct encl *encl);uint64_t encl_get_entry (struct encl *encl,const char *symbol);
Signature · 1 line
2.37.2