Skip to contentSkip to conversation

xen: Arch Teardown

PATCH In review Andrew Cooper 4 participants 8 messages Jun 5, 2023 – Jun 7, 2023

lore
Share & tools
lore
#0 lore raw

cover xen: Arch Teardown

Sorry this is late... Patch 1 ought to be ready in this form.

I'll leave it up to the ARM maintainers as to whether they want to take patch 2 in that form (with TEE in patch 3), or whether to merge the two.

Compile tested on all architectures, functionally tested on x86 only.

Branch available from:

https://xenbits.xen.org/gitweb/?p=people/andrewcoop/xen.git;a=shortlog;h=refs/heads/arch-teardown

Full Gitlab CI run in progress:

https://gitlab.com/xen-project/people/andyhhp/xen/-/pipelines/889759838

Andrew Cooper (2):
x86: Introduce arch_domain_teardown()
arm: Boilerpate arch_domain_teardown()

 xen/arch/arm/domain.c    | 32 ++++++++++++++++++++++++++++++++
 xen/arch/x86/domain.c    |  5 +++++
 xen/common/domain.c      |  6 ++++++
 xen/include/xen/domain.h |  1 +
 xen/include/xen/sched.h  |  1 +
 5 files changed, 45 insertions(+)

base-commit: b3880c365db89051728e1de6b6889c750cbdd915

Signature · 1 line
2.30.2
#1 lore raw

PATCH 1/2 x86: Introduce arch_domain_teardown()

Plumb it into domain_teardown(). Provide arch_val in the teardown continuation information for use by arch_domain_teardown().

No practical change.

---
CC: Jan Beulich <[email protected]>
CC: Roger Pau Monné <[email protected]>
CC: Wei Liu <[email protected]>
CC: Stefano Stabellini <[email protected]>
CC: Julien Grall <[email protected]>
CC: Volodymyr Babchuk <[email protected]>
CC: Bertrand Marquis <[email protected]>
CC: Jens Wiklander <[email protected]>

---
 xen/arch/arm/domain.c    | 5 +++++
 xen/arch/x86/domain.c    | 5 +++++
 xen/common/domain.c      | 6 ++++++
 xen/include/xen/domain.h | 1 +
 xen/include/xen/sched.h  | 1 +
 5 files changed, 18 insertions(+)
xen/arch/arm/domain.c +5 -0
index d8ef6501ff8e..b3981d70a442 100644
@@ -750,6 +750,11 @@ int arch_domain_create(struct domain *d,
     return rc;
 }
 
+int arch_domain_teardown(struct domain *d)
+{
+    return 0;
+}
+
 void arch_domain_destroy(struct domain *d)
 {
     /* IOMMU page table is shared with P2M, always call
xen/arch/x86/domain.c +5 -0
index 39c215316546..5f66c2ae33d7 100644
@@ -888,6 +888,11 @@ int arch_domain_create(struct domain *d,
     return rc;
 }
 
+int arch_domain_teardown(struct domain *d)
+{
+    return 0;
+}
+
 void arch_domain_destroy(struct domain *d)
 {
     if ( is_hvm_domain(d) )
xen/common/domain.c +6 -0
index 6a440590fe2a..b0d850e3595b 100644
@@ -416,6 +416,7 @@ static int domain_teardown(struct domain *d)
             PROG_none,
             PROG_gnttab_mappings,
             PROG_vcpu_teardown,
+            PROG_arch_teardown,
             PROG_done,
         };
 
@@ -436,6 +437,11 @@ static int domain_teardown(struct domain *d)
                 return rc;
         }
 
+    PROGRESS(arch_teardown):
+        rc = arch_domain_teardown(d);
+        if ( rc )
+            return rc;
+
     PROGRESS(done):
         break;
 
xen/include/xen/domain.h +1 -0
index 26f9c4f6dd5b..c3348c90748f 100644
@@ -80,6 +80,7 @@ int arch_domain_create(struct domain *d,
                        struct xen_domctl_createdomain *config,
                        unsigned int flags);
 
+int arch_domain_teardown(struct domain *d);
 void arch_domain_destroy(struct domain *d);
 
 void arch_domain_shutdown(struct domain *d);
xen/include/xen/sched.h +1 -0
index 85242a73d374..854f3e32c00e 100644
@@ -589,6 +589,7 @@ struct domain
      */
     struct {
         unsigned int val;
+        unsigned int arch_val;
         struct vcpu *vcpu;
     } teardown;
 
Signature · 1 line
2.30.2
#2 lore raw

Re: [PATCH 1/2] x86: Introduce arch_domain_teardown()

Quoted 96 lines · Andrew Cooper

Plumb it into domain_teardown(). Provide arch_val in the teardown continuation information for use by arch_domain_teardown().

No practical change.

---
CC: Jan Beulich <[email protected]>
CC: Roger Pau Monné <[email protected]>
CC: Wei Liu <[email protected]>
CC: Stefano Stabellini <[email protected]>
CC: Julien Grall <[email protected]>
CC: Volodymyr Babchuk <[email protected]>
CC: Bertrand Marquis <[email protected]>
CC: Jens Wiklander <[email protected]>

---
 xen/arch/arm/domain.c    | 5 +++++
 xen/arch/x86/domain.c    | 5 +++++
 xen/common/domain.c      | 6 ++++++
 xen/include/xen/domain.h | 1 +
 xen/include/xen/sched.h  | 1 +
 5 files changed, 18 insertions(+)
xen/arch/arm/domain.c +5 -0
index d8ef6501ff8e..b3981d70a442 100644
@@ -750,6 +750,11 @@ int arch_domain_create(struct domain *d,
     return rc;
 }
 
+int arch_domain_teardown(struct domain *d)
+{
+    return 0;
+}
+
 void arch_domain_destroy(struct domain *d)
 {
     /* IOMMU page table is shared with P2M, always call
xen/arch/x86/domain.c +5 -0
index 39c215316546..5f66c2ae33d7 100644
@@ -888,6 +888,11 @@ int arch_domain_create(struct domain *d,
     return rc;
 }
 
+int arch_domain_teardown(struct domain *d)
+{
+    return 0;
+}
+
 void arch_domain_destroy(struct domain *d)
 {
     if ( is_hvm_domain(d) )
xen/common/domain.c +6 -0
index 6a440590fe2a..b0d850e3595b 100644
@@ -416,6 +416,7 @@ static int domain_teardown(struct domain *d)
             PROG_none,
             PROG_gnttab_mappings,
             PROG_vcpu_teardown,
+            PROG_arch_teardown,
             PROG_done,
         };
 
@@ -436,6 +437,11 @@ static int domain_teardown(struct domain *d)
                 return rc;
         }
 
+    PROGRESS(arch_teardown):
+        rc = arch_domain_teardown(d);
+        if ( rc )
+            return rc;
+
     PROGRESS(done):
         break;
 
xen/include/xen/domain.h +1 -0
index 26f9c4f6dd5b..c3348c90748f 100644
@@ -80,6 +80,7 @@ int arch_domain_create(struct domain *d,
                        struct xen_domctl_createdomain *config,
                        unsigned int flags);
 
+int arch_domain_teardown(struct domain *d);
 void arch_domain_destroy(struct domain *d);
 
 void arch_domain_shutdown(struct domain *d);
xen/include/xen/sched.h +0 -0
index 85242a73d374..854f3e32c00e 100644

xen/include/xen/sched.h struct domain · line 592

Quoted 4 lines
xen/include/xen/sched.h +1 -0
@@ -589,6 +589,7 @@ struct domain
      */
     struct {
         unsigned int val;
+        unsigned int arch_val;

While I haven't looked at patch 2, wouldn't such continuation information better be encoded in arch_domain in whatever way is more suitable for each architecture?

Thanks, Roger.

#3 lore raw

Re: [PATCH 1/2] x86: Introduce arch_domain_teardown()

Quoted 13 lines · Roger Pau Monné

xen/include/xen/sched.h struct domain · line 592

Quoted 9 lines · Andrew Cooper
xen/include/xen/sched.h +1 -0
index 85242a73d374..854f3e32c00e 100644
@@ -589,6 +589,7 @@ struct domain
      */
     struct {
         unsigned int val;
+        unsigned int arch_val;

While I haven't looked at patch 2, wouldn't such continuation information better be encoded in arch_domain in whatever way is more suitable for each architecture?

Well, it's filling a hole here on 64bit builds, which it couldn't do in arch_domain.

And it's contained inside teardown{} which already signals it as fairly magic.

~Andrew

Acked-by
#4 lore raw

Re: [PATCH 1/2] x86: Introduce arch_domain_teardown()

Quoted 15 lines · Andrew Cooper
Quoted 13 lines · Roger Pau Monné

xen/include/xen/sched.h struct domain · line 592

Quoted 9 lines · Andrew Cooper
xen/include/xen/sched.h +1 -0
index 85242a73d374..854f3e32c00e 100644
@@ -589,6 +589,7 @@ struct domain
      */
     struct {
         unsigned int val;
+        unsigned int arch_val;

While I haven't looked at patch 2, wouldn't such continuation information better be encoded in arch_domain in whatever way is more suitable for each architecture?

Quoted 5 lines

Well, it's filling a hole here on 64bit builds, which it couldn't do in arch_domain.

And it's contained inside teardown{} which already signals it as fairly magic.

Plus why have each architecture duplicate the field? I expect none of the arch_domain_teardown() instances will remain without an actual use of the new field, mid to long term.

I don't want to override Roger's concern, but from my pov

Jan

#5 lore raw

Re: [PATCH 1/2] x86: Introduce arch_domain_teardown()

Quoted 22 lines · Jan Beulich
Quoted 15 lines · Andrew Cooper
Quoted 13 lines · Roger Pau Monné

xen/include/xen/sched.h struct domain · line 592

Quoted 9 lines · Andrew Cooper
xen/include/xen/sched.h +1 -0
index 85242a73d374..854f3e32c00e 100644
@@ -589,6 +589,7 @@ struct domain
      */
     struct {
         unsigned int val;
+        unsigned int arch_val;

While I haven't looked at patch 2, wouldn't such continuation information better be encoded in arch_domain in whatever way is more suitable for each architecture?

Quoted 5 lines

Well, it's filling a hole here on 64bit builds, which it couldn't do in arch_domain.

And it's contained inside teardown{} which already signals it as fairly magic.

Quoted 6 lines

Plus why have each architecture duplicate the field? I expect none of the arch_domain_teardown() instances will remain without an actual use of the new field, mid to long term.

I don't want to override Roger's concern, but from my pov

Oh, no worries, I was meaning to reply I was fine with Andrews justification, but forgot.

Thanks, Roger

#6 lore raw

PATCH 2/2 arm: Boilerpate arch_domain_teardown()

In reply to Andrew Cooper

XXX to be filled in with TEE teardown.

---
CC: Stefano Stabellini <[email protected]>
CC: Julien Grall <[email protected]>
CC: Volodymyr Babchuk <[email protected]>
CC: Bertrand Marquis <[email protected]>
CC: Jens Wiklander <[email protected]>

Jens: In the same was as the previous patch in the common path, you want to add a PROG_tee(?) here, and rearrange the right function(s).

---
 xen/arch/arm/domain.c | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
xen/arch/arm/domain.c +27 -0
index b3981d70a442..b00d0e4f30b7 100644
@@ -752,6 +752,33 @@ int arch_domain_create(struct domain *d,
 
 int arch_domain_teardown(struct domain *d)
 {
+    BUG_ON(!d->is_dying);
+
+    /* See domain_teardown() for an explanation of all of this magic. */
+    switch ( d->teardown.arch_val )
+    {
+#define PROGRESS(x)                             \
+        d->teardown.arch_val = PROG_ ## x;      \
+        fallthrough;                            \
+    case PROG_ ## x
+
+        enum {
+            PROG_none,
+            PROG_done,
+        };
+
+    case PROG_none:
+        BUILD_BUG_ON(PROG_none != 0);
+
+    PROGRESS(done):
+        break;
+
+#undef PROGRESS
+
+    default:
+        BUG();
+    }
+
     return 0;
 }
 
Signature · 1 line
2.30.2
#7 lore raw
In reply to Andrew Cooper

Hi Andrew,

Quoted 9 lines · Andrew Cooper

Sorry this is late... Patch 1 ought to be ready in this form.

I'll leave it up to the ARM maintainers as to whether they want to take patch 2 in that form (with TEE in patch 3), or whether to merge the two.

Compile tested on all architectures, functionally tested on x86 only.

Quoted 7 lines

Branch available from:

https://xenbits.xen.org/gitweb/?p=people/andrewcoop/xen.git;a=shortlog;h=refs/heads/arch-teardown

Full Gitlab CI run in progress:

https://gitlab.com/xen-project/people/andyhhp/xen/-/pipelines/889759838

Thanks a lot for that.

I will sync up with Jens and we will handle the rest :-)

Cheers
Bertrand

Quoted 17 lines

Andrew Cooper (2):
x86: Introduce arch_domain_teardown()
arm: Boilerpate arch_domain_teardown()

xen/arch/arm/domain.c    | 32 ++++++++++++++++++++++++++++++++
xen/arch/x86/domain.c    |  5 +++++
xen/common/domain.c      |  6 ++++++
xen/include/xen/domain.h |  1 +
xen/include/xen/sched.h  |  1 +
5 files changed, 45 insertions(+)

base-commit: b3880c365db89051728e1de6b6889c750cbdd915

Signature · 1 line
2.30.2