Skip to contentSkip to conversation

x86: Work around Shstk fracturing

PATCH In review Andrew Cooper 2 participants 10 messages Jan 4, 2023 – Mar 6, 2023

lore
Share & tools
lore
#0 lore raw

cover x86: Work around Shstk fracturing

See patch 2 for details.

Andrew Cooper (2):
x86/cpuid: Infrastructure for leaves 7:1{ecx,edx} x86/shskt: Disable CET-SS on parts susceptible to fractured updates

 docs/misc/xen-command-line.pandoc           |  7 ++++-
 tools/libs/light/libxl_cpuid.c              |  2 ++
 tools/misc/xen-cpuid.c                      | 11 +++++++
 xen/arch/x86/cpu/common.c                   | 14 +++++++--
 xen/arch/x86/setup.c                        | 46 ++++++++++++++++++++++++-----
 xen/include/public/arch-x86/cpufeatureset.h |  4 +++
 xen/include/xen/lib/x86/cpuid.h             | 15 +++++++++-
 7 files changed, 86 insertions(+), 13 deletions(-)
Signature · 1 line
2.11.0
#1 lore raw

PATCH 1/2 x86/cpuid: Infrastructure for leaves 7:1{ecx,edx}

We don't actually need ecx yet, but adding it in now will reduce the amount to which leaf 7 is out of order in a featureset.

cpufeatureset.h remains in leaf architectrual order for the sanity of anyone trying to locate where to insert new rows.

---
CC: Jan Beulich <[email protected]>
CC: Roger Pau Monné <[email protected]>
CC: Wei Liu <[email protected]>

v2:
* Fix decodes[] short string

---
 tools/misc/xen-cpuid.c                      | 10 ++++++++++
 xen/arch/x86/cpu/common.c                   |  3 ++-
 xen/include/public/arch-x86/cpufeatureset.h |  3 +++
 xen/include/xen/lib/x86/cpuid.h             | 15 ++++++++++++++-
 4 files changed, 29 insertions(+), 2 deletions(-)
tools/misc/xen-cpuid.c +10 -0
index d5833e9ce879..addb3a39a11a 100644
@@ -202,6 +202,14 @@ static const char *const str_7b1[32] =
     [ 0] = "ppin",
 };
 
+static const char *const str_7c1[32] =
+{
+};
+
+static const char *const str_7d1[32] =
+{
+};
+
 static const char *const str_7d2[32] =
 {
     [ 0] = "intel-psfd",
@@ -229,6 +237,8 @@ static const struct {
     { "0x80000021.eax",  "e21a", str_e21a },
     { "0x00000007:1.ebx", "7b1", str_7b1 },
     { "0x00000007:2.edx", "7d2", str_7d2 },
+    { "0x00000007:1.ecx", "7c1", str_7c1 },
+    { "0x00000007:1.edx", "7d1", str_7d1 },
 };
 
 #define COL_ALIGN "18"
xen/arch/x86/cpu/common.c +2 -1
index 0412dbc915e5..b3fcf4680f3a 100644
@@ -450,7 +450,8 @@ static void generic_identify(struct cpuinfo_x86 *c)
 			cpuid_count(7, 1,
 				    &c->x86_capability[FEATURESET_7a1],
 				    &c->x86_capability[FEATURESET_7b1],
-				    &tmp, &tmp);
+				    &c->x86_capability[FEATURESET_7c1],
+				    &c->x86_capability[FEATURESET_7d1]);
 		if (max_subleaf >= 2)
 			cpuid_count(7, 2,
 				    &tmp, &tmp, &tmp,
xen/include/public/arch-x86/cpufeatureset.h +3 -0
index 7915f5826f57..7a896f0e2d92 100644
@@ -288,6 +288,9 @@ XEN_CPUFEATURE(NSCB,               11*32+ 6) /*A  Null Selector Clears Base (and
 /* Intel-defined CPU features, CPUID level 0x00000007:1.ebx, word 12 */
 XEN_CPUFEATURE(INTEL_PPIN,         12*32+ 0) /*   Protected Processor Inventory Number */
 
+/* Intel-defined CPU features, CPUID level 0x00000007:1.ecx, word 14 */
+/* Intel-defined CPU features, CPUID level 0x00000007:1.edx, word 15 */
+
 /* Intel-defined CPU features, CPUID level 0x00000007:2.edx, word 13 */
 XEN_CPUFEATURE(INTEL_PSFD,         13*32+ 0) /*A  MSR_SPEC_CTRL.PSFD */
 XEN_CPUFEATURE(IPRED_CTRL,         13*32+ 1) /*   MSR_SPEC_CTRL.IPRED_DIS_* */
xen/include/xen/lib/x86/cpuid.h +14 -1
index 73a5c330365e..fa98b371eef4 100644
@@ -18,6 +18,8 @@
 #define FEATURESET_e21a  11 /* 0x80000021.eax      */
 #define FEATURESET_7b1   12 /* 0x00000007:1.ebx    */
 #define FEATURESET_7d2   13 /* 0x00000007:2.edx    */
+#define FEATURESET_7c1   14 /* 0x00000007:1.ecx    */
+#define FEATURESET_7d1   15 /* 0x00000007:1.edx    */
 
 struct cpuid_leaf
 {
@@ -194,7 +196,14 @@ struct cpuid_policy
                 uint32_t _7b1;
                 struct { DECL_BITFIELD(7b1); };
             };
-            uint32_t /* c */:32, /* d */:32;
+            union {
+                uint32_t _7c1;
+                struct { DECL_BITFIELD(7c1); };
+            };
+            union {
+                uint32_t _7d1;
+                struct { DECL_BITFIELD(7d1); };
+            };
 
             /* Subleaf 2. */
             uint32_t /* a */:32, /* b */:32, /* c */:32;
@@ -343,6 +352,8 @@ static inline void cpuid_policy_to_featureset(
     fs[FEATURESET_e21a] = p->extd.e21a;
     fs[FEATURESET_7b1] = p->feat._7b1;
     fs[FEATURESET_7d2] = p->feat._7d2;
+    fs[FEATURESET_7c1] = p->feat._7c1;
+    fs[FEATURESET_7d1] = p->feat._7d1;
 }
 
 /* Fill in a CPUID policy from a featureset bitmap. */
@@ -363,6 +374,8 @@ static inline void cpuid_featureset_to_policy(
     p->extd.e21a  = fs[FEATURESET_e21a];
     p->feat._7b1  = fs[FEATURESET_7b1];
     p->feat._7d2  = fs[FEATURESET_7d2];
+    p->feat._7c1  = fs[FEATURESET_7c1];
+    p->feat._7d1  = fs[FEATURESET_7d1];
 }
 
 static inline uint64_t cpuid_policy_xcr0_max(const struct cpuid_policy *p)
Signature · 1 line
2.11.0
Reviewed-by on v1 1/2
#2 lore raw

Re: [PATCH 1/2] x86/cpuid: Infrastructure for leaves 7:1{ecx,edx}

Quoted 7 lines · Andrew Cooper

We don't actually need ecx yet, but adding it in now will reduce the amount to which leaf 7 is out of order in a featureset.

cpufeatureset.h remains in leaf architectrual order for the sanity of anyone trying to locate where to insert new rows.

albeit ...

xen/include/public/arch-x86/cpufeatureset.h XEN_CPUFEATURE(NSCB, 11*32+ 6) /*A Null Selector Clears Base (and · line 293

Quoted 9 lines
xen/include/public/arch-x86/cpufeatureset.h +3 -0
@@ -288,6 +288,9 @@ XEN_CPUFEATURE(NSCB,               11*32+ 6) /*A  Null Selector Clears Base (and
 /* Intel-defined CPU features, CPUID level 0x00000007:1.ebx, word 12 */
 XEN_CPUFEATURE(INTEL_PPIN,         12*32+ 0) /*   Protected Processor Inventory Number */
 
+/* Intel-defined CPU features, CPUID level 0x00000007:1.ecx, word 14 */
+/* Intel-defined CPU features, CPUID level 0x00000007:1.edx, word 15 */
+

... I'm not convinced getting these ordered by other than their word indexes is quite reasonable. We can't really predict in what order elements / leaves get populated, as can best be seen from ...

xen/include/public/arch-x86/cpufeatureset.h XEN_CPUFEATURE(NSCB, 11*32+ 6) /*A Null Selector Clears Base (and · line 296

Quoted 3 lines
xen/include/public/arch-x86/cpufeatureset.h +0 -0
 /* Intel-defined CPU features, CPUID level 0x00000007:2.edx, word 13 */
 XEN_CPUFEATURE(INTEL_PSFD,         13*32+ 0) /*A  MSR_SPEC_CTRL.PSFD */
 XEN_CPUFEATURE(IPRED_CTRL,         13*32+ 1) /*   MSR_SPEC_CTRL.IPRED_DIS_* */

... subleaf 2 already having one entry, and that one not being eax, ebx, nor ecx, but edx. AMD (extended) leaves would also always be sprinkled into the middle of any such sequence. To me it ends up more confusing (perhaps not right away, but in a couple of years time) if one needs to go hunt for what the next free index value would be. Similarly the need to re-base stuff using non-upstream index values (like is the case for the KeyLocker leaves that I have pending locally) is easier to notice when new sets are put at the end of the existing list.

In any event, what I'd like to ask for as a minimum is that you insert a blank line between the two new comments.

Jan

#3 lore raw

Re: [PATCH 1/2] x86/cpuid: Infrastructure for leaves 7:1{ecx,edx}

In reply to Andrew Cooper

xen/include/public/arch-x86/cpufeatureset.h XEN_CPUFEATURE(NSCB, 11*32+ 6) /*A Null Selector Clears Base (and · line 292

Quoted 8 lines · Andrew Cooper
xen/include/public/arch-x86/cpufeatureset.h +2 -0
@@ -288,6 +288,9 @@ XEN_CPUFEATURE(NSCB,               11*32+ 6) /*A  Null Selector Clears Base (and
 /* Intel-defined CPU features, CPUID level 0x00000007:1.ebx, word 12 */
 XEN_CPUFEATURE(INTEL_PPIN,         12*32+ 0) /*   Protected Processor Inventory Number */
 
+/* Intel-defined CPU features, CPUID level 0x00000007:1.ecx, word 14 */
+/* Intel-defined CPU features, CPUID level 0x00000007:1.edx, word 15 */

While committing the backports of this (where I normally test-build every commit individually) I came to notice that this introduces a transient (until the next commit) build breakage: FEATURESET_NR_ENTRIES is calculated from the highest entry found; the comments here don't matter at all. Therefore ...

Quoted 10 lines
xen/include/public/arch-x86/cpufeatureset.h +2 -0
@@ -343,6 +352,8 @@ static inline void cpuid_policy_to_featureset(
     fs[FEATURESET_e21a] = p->extd.e21a;
     fs[FEATURESET_7b1] = p->feat._7b1;
     fs[FEATURESET_7d2] = p->feat._7d2;
+    fs[FEATURESET_7c1] = p->feat._7c1;
+    fs[FEATURESET_7d1] = p->feat._7d1;
 }
 
 /* Fill in a CPUID policy from a featureset bitmap. */

xen/include/public/arch-x86/cpufeatureset.h static inline void cpuid_featureset_to_policy( · line 379

Quoted 6 lines
xen/include/public/arch-x86/cpufeatureset.h +2 -0
@@ -363,6 +374,8 @@ static inline void cpuid_featureset_to_policy(
     p->extd.e21a  = fs[FEATURESET_e21a];
     p->feat._7b1  = fs[FEATURESET_7b1];
     p->feat._7d2  = fs[FEATURESET_7d2];
+    p->feat._7c1  = fs[FEATURESET_7c1];
+    p->feat._7d1  = fs[FEATURESET_7d1];
 }

... the compiler legitimately complains about out-of-bounds array accesses here. This is just fyi for the future (to arrange patch splitting differently); I've left the backports as they were.

Jan

#4 lore raw

Re: [PATCH 1/2] x86/cpuid: Infrastructure for leaves 7:1{ecx,edx}

Quoted 23 lines · Jan Beulich

xen/include/public/arch-x86/cpufeatureset.h XEN_CPUFEATURE(NSCB, 11*32+ 6) /*A Null Selector Clears Base (and · line 292

Quoted 8 lines · Andrew Cooper
xen/include/public/arch-x86/cpufeatureset.h +2 -0
@@ -288,6 +288,9 @@ XEN_CPUFEATURE(NSCB,               11*32+ 6) /*A  Null Selector Clears Base (and
 /* Intel-defined CPU features, CPUID level 0x00000007:1.ebx, word 12 */
 XEN_CPUFEATURE(INTEL_PPIN,         12*32+ 0) /*   Protected Processor Inventory Number */
 
+/* Intel-defined CPU features, CPUID level 0x00000007:1.ecx, word 14 */
+/* Intel-defined CPU features, CPUID level 0x00000007:1.edx, word 15 */

While committing the backports of this (where I normally test-build every commit individually) I came to notice that this introduces a transient (until the next commit) build breakage: FEATURESET_NR_ENTRIES is calculated from the highest entry found; the comments here don't matter at all. Therefore ...

Quoted 8 lines
xen/include/public/arch-x86/cpufeatureset.h +2 -0
@@ -343,6 +352,8 @@ static inline void cpuid_policy_to_featureset(
     fs[FEATURESET_e21a] = p->extd.e21a;
     fs[FEATURESET_7b1] = p->feat._7b1;
     fs[FEATURESET_7d2] = p->feat._7d2;
+    fs[FEATURESET_7c1] = p->feat._7c1;
+    fs[FEATURESET_7d1] = p->feat._7d1;
 }
 
Quoted 11 lines

xen/include/public/arch-x86/cpufeatureset.h static inline void cpuid_featureset_to_policy( · line 379

Quoted 8 lines
xen/include/public/arch-x86/cpufeatureset.h +2 -0
 /* Fill in a CPUID policy from a featureset bitmap. */
@@ -363,6 +374,8 @@ static inline void cpuid_featureset_to_policy(
     p->extd.e21a  = fs[FEATURESET_e21a];
     p->feat._7b1  = fs[FEATURESET_7b1];
     p->feat._7d2  = fs[FEATURESET_7d2];
+    p->feat._7c1  = fs[FEATURESET_7c1];
+    p->feat._7d1  = fs[FEATURESET_7d1];
 }

... the compiler legitimately complains about out-of-bounds array accesses here. This is just fyi for the future (to arrange patch splitting differently); I've left the backports as they were.

Hmm.  c/s e3662437eb43 was designed to specifically allow CPUID patches to be split like this.

Which compiler?  I think I agree with your analysis, but I've never seen a complaint, hence not noticing.

I suspect we actually want FEATURESET_NR_ENTRIES defined in C, next to the FEATURESET_* defines, and we want to BUILD_BUG_ON() if the autogen length is larger than the C length.

~Andrew

#5 lore raw

Re: [PATCH 1/2] x86/cpuid: Infrastructure for leaves 7:1{ecx,edx}

Quoted 36 lines · Andrew Cooper
Quoted 23 lines · Jan Beulich

xen/include/public/arch-x86/cpufeatureset.h XEN_CPUFEATURE(NSCB, 11*32+ 6) /*A Null Selector Clears Base (and · line 292

Quoted 8 lines · Andrew Cooper
xen/include/public/arch-x86/cpufeatureset.h +2 -0
@@ -288,6 +288,9 @@ XEN_CPUFEATURE(NSCB,               11*32+ 6) /*A  Null Selector Clears Base (and
 /* Intel-defined CPU features, CPUID level 0x00000007:1.ebx, word 12 */
 XEN_CPUFEATURE(INTEL_PPIN,         12*32+ 0) /*   Protected Processor Inventory Number */
 
+/* Intel-defined CPU features, CPUID level 0x00000007:1.ecx, word 14 */
+/* Intel-defined CPU features, CPUID level 0x00000007:1.edx, word 15 */

While committing the backports of this (where I normally test-build every commit individually) I came to notice that this introduces a transient (until the next commit) build breakage: FEATURESET_NR_ENTRIES is calculated from the highest entry found; the comments here don't matter at all. Therefore ...

Quoted 8 lines
xen/include/public/arch-x86/cpufeatureset.h +2 -0
@@ -343,6 +352,8 @@ static inline void cpuid_policy_to_featureset(
     fs[FEATURESET_e21a] = p->extd.e21a;
     fs[FEATURESET_7b1] = p->feat._7b1;
     fs[FEATURESET_7d2] = p->feat._7d2;
+    fs[FEATURESET_7c1] = p->feat._7c1;
+    fs[FEATURESET_7d1] = p->feat._7d1;
 }
 
Quoted 11 lines

xen/include/public/arch-x86/cpufeatureset.h static inline void cpuid_featureset_to_policy( · line 379

Quoted 8 lines
xen/include/public/arch-x86/cpufeatureset.h +2 -0
 /* Fill in a CPUID policy from a featureset bitmap. */
@@ -363,6 +374,8 @@ static inline void cpuid_featureset_to_policy(
     p->extd.e21a  = fs[FEATURESET_e21a];
     p->feat._7b1  = fs[FEATURESET_7b1];
     p->feat._7d2  = fs[FEATURESET_7d2];
+    p->feat._7c1  = fs[FEATURESET_7c1];
+    p->feat._7d1  = fs[FEATURESET_7d1];
 }

... the compiler legitimately complains about out-of-bounds array accesses here. This is just fyi for the future (to arrange patch splitting differently); I've left the backports as they were.

Quoted 5 lines

Hmm.  c/s e3662437eb43 was designed to specifically allow CPUID patches to be split like this.

Which compiler?  I think I agree with your analysis, but I've never seen a complaint, hence not noticing.

gcc 12

Quoted 3 lines

I suspect we actually want FEATURESET_NR_ENTRIES defined in C, next to the FEATURESET_* defines, and we want to BUILD_BUG_ON() if the autogen length is larger than the C length.

Hmm, yes, this may be the best we can do.

Jan

#6 lore raw

PATCH 2/2 x86/shskt: Disable CET-SS on parts susceptible to fractured updates

In reply to Andrew Cooper

Refer to Intel SDM Rev 70 (Dec 2022), Vol3 17.2.3 "Supervisor Shadow Stack Token".

Architecturally, an event delivery which starts in CPL<3 and switches shadow stack will first validate the Supervisor Shadow Stack Token (setting the busy bit), then pushes CS/LIP/SSP. One example of this is an NMI interrupting Xen.

Some CPUs suffer from an issue called fracturing, whereby a fault/vmexit/etc between setting the busy bit and completing the event injection renders the action non-restartable, because when it comes time to restart, the busy bit is found to be already set.

This is far more easily encountered under virt, yet it is not the fault of the hypervisor, nor the fault of the guest kernel. The fault lies somewhere between the architectural specification, and the uarch behaviour.

Intel have allocated CPUID.7[1].ecx[18] CET_SSS to enumerate that supervisor shadow stacks are safe to use. Because of how Xen lays out its shadow stacks, fracturing is not expected to be a problem on native.

Detect this case on boot and default to not using shstk if virtualised. Specifying cet=shstk on the command line will override this heuristic and enable shadow stacks irrespective.

---
CC: Jan Beulich <[email protected]>
CC: Roger Pau Monné <[email protected]>
CC: Wei Liu <[email protected]>

v2:
* Confirmation from AMD that their parts are not impacted.
* Fix pv-shim build. The "define false" trick doesn't work with tristates.
* Tweak wording in several places.
* Fix tabs vs spaces.

This ideally wants backporting to Xen 4.14. I have no idea how likely it is to need to backport the prerequisite patch for new feature words, but we've already had to do that once for security patches. OTOH, I have no idea how easy it is to trigger in non-synthetic cases.

---
 docs/misc/xen-command-line.pandoc           |  7 ++++-
 tools/libs/light/libxl_cpuid.c              |  2 ++
 tools/misc/xen-cpuid.c                      |  1 +
 xen/arch/x86/cpu/common.c                   | 11 +++++--
 xen/arch/x86/setup.c                        | 46 ++++++++++++++++++++++++-----
 xen/include/public/arch-x86/cpufeatureset.h |  1 +
 6 files changed, 57 insertions(+), 11 deletions(-)
docs/misc/xen-command-line.pandoc +6 -1
index 923910f553c5..19d4d815bdee 100644
@@ -287,10 +287,15 @@ can be maintained with the pv-shim mechanism.
     protection.
 
     The option is available when `CONFIG_XEN_SHSTK` is compiled in, and
-    defaults to `true` on hardware supporting CET-SS.  Specifying
+    generally defaults to `true` on hardware supporting CET-SS.  Specifying
     `cet=no-shstk` will cause Xen not to use Shadow Stacks even when support
     is available in hardware.
 
+    Some hardware suffers from an issue known as Supervisor Shadow Stack
+    Fracturing.  On such hardware, Xen will default to not using Shadow Stacks
+    when virtualised.  Specifying `cet=shstk` will override this heuristic and
+    enable Shadow Stacks unilaterally.
+
 *   The `ibt=` boolean controls whether Xen uses Indirect Branch Tracking for
     its own protection.
 
tools/libs/light/libxl_cpuid.c +2 -0
index 2aa23225f42c..d97a2f3338bc 100644
@@ -235,6 +235,8 @@ int libxl_cpuid_parse_config(libxl_cpuid_policy_list *cpuid, const char* str)
         {"fsrs",         0x00000007,  1, CPUID_REG_EAX, 11,  1},
         {"fsrcs",        0x00000007,  1, CPUID_REG_EAX, 12,  1},
 
+        {"cet-sss",      0x00000007,  1, CPUID_REG_EDX, 18,  1},
+
         {"intel-psfd",   0x00000007,  2, CPUID_REG_EDX,  0,  1},
         {"mcdt-no",      0x00000007,  2, CPUID_REG_EDX,  5,  1},
 
tools/misc/xen-cpuid.c +1 -0
index addb3a39a11a..0248eaef44c1 100644
@@ -208,6 +208,7 @@ static const char *const str_7c1[32] =
 
 static const char *const str_7d1[32] =
 {
+    [18] = "cet-sss",
 };
 
 static const char *const str_7d2[32] =
xen/arch/x86/cpu/common.c +9 -2
index b3fcf4680f3a..27f73d3bbe31 100644
@@ -346,11 +346,18 @@ void __init early_cpu_init(void)
 	       x86_cpuid_vendor_to_str(c->x86_vendor), c->x86, c->x86,
 	       c->x86_model, c->x86_model, c->x86_mask, eax);
 
-	if (c->cpuid_level >= 7)
-		cpuid_count(7, 0, &eax, &ebx,
+	if (c->cpuid_level >= 7) {
+		uint32_t max_subleaf;
+
+		cpuid_count(7, 0, &max_subleaf, &ebx,
 			    &c->x86_capability[FEATURESET_7c0],
 			    &c->x86_capability[FEATURESET_7d0]);
 
+		if (max_subleaf >= 1)
+			cpuid_count(7, 1, &eax, &ebx, &ecx,
+				    &c->x86_capability[FEATURESET_7d1]);
+	}
+
 	eax = cpuid_eax(0x80000000);
 	if ((eax >> 16) == 0x8000 && eax >= 0x80000008) {
 		ebx = eax >= 0x8000001f ? cpuid_ebx(0x8000001f) : 0;
xen/arch/x86/setup.c +38 -8

60 lines not on this page · raw message

xen/include/public/arch-x86/cpufeatureset.h +1 -0
index 7a896f0e2d92..f6a46f62a549 100644
@@ -290,6 +290,7 @@ XEN_CPUFEATURE(INTEL_PPIN,         12*32+ 0) /*   Protected Processor Inventory
 
 /* Intel-defined CPU features, CPUID level 0x00000007:1.ecx, word 14 */
 /* Intel-defined CPU features, CPUID level 0x00000007:1.edx, word 15 */
+XEN_CPUFEATURE(CET_SSS,            15*32+18) /*   CET Supervisor Shadow Stacks safe to use */
 
 /* Intel-defined CPU features, CPUID level 0x00000007:2.edx, word 13 */
 XEN_CPUFEATURE(INTEL_PSFD,         13*32+ 0) /*A  MSR_SPEC_CTRL.PSFD */
Signature · 1 line
2.11.0
Reviewed-by on v1 2/2
#7 lore raw

Re: [PATCH 2/2] x86/shskt: Disable CET-SS on parts susceptible to fractured updates

Quoted 12 lines · Andrew Cooper

Refer to Intel SDM Rev 70 (Dec 2022), Vol3 17.2.3 "Supervisor Shadow Stack Token".

Architecturally, an event delivery which starts in CPL<3 and switches shadow stack will first validate the Supervisor Shadow Stack Token (setting the busy bit), then pushes CS/LIP/SSP. One example of this is an NMI interrupting Xen.

Some CPUs suffer from an issue called fracturing, whereby a fault/vmexit/etc between setting the busy bit and completing the event injection renders the action non-restartable, because when it comes time to restart, the busy bit is found to be already set.

Quoted 7 lines

This is far more easily encountered under virt, yet it is not the fault of the hypervisor, nor the fault of the guest kernel. The fault lies somewhere between the architectural specification, and the uarch behaviour.

Intel have allocated CPUID.7[1].ecx[18] CET_SSS to enumerate that supervisor shadow stacks are safe to use. Because of how Xen lays out its shadow stacks, fracturing is not expected to be a problem on native.

IOW that's the "contained in an aligned 32-byte region" constraint which we meet, aiui.

Quoted 5 lines

Detect this case on boot and default to not using shstk if virtualised. Specifying cet=shstk on the command line will override this heuristic and enable shadow stacks irrespective.

with one nit (below).

Quoted 4 lines

This ideally wants backporting to Xen 4.14. I have no idea how likely it is to need to backport the prerequisite patch for new feature words, but we've already had to do that once for security patches. OTOH, I have no idea how easy it is to trigger in non-synthetic cases.

Plus: How likely is it that Xen actually is used virtualized in production? For the moment I don't see any reason to backport to branches in security- only maintenance mode. I'm not even sure it needs backporting at all.

Quoted 9 lines
xen/arch/x86/setup.c +4 -1
@@ -1099,11 +1095,45 @@ void __init noreturn __start_xen(unsigned long mbi_p)
     early_cpu_init();
 
     /* Choose shadow stack early, to set infrastructure up appropriately. */
-    if ( opt_xen_shstk && boot_cpu_has(X86_FEATURE_CET_SS) )
+    if ( !boot_cpu_has(X86_FEATURE_CET_SS) )
+        opt_xen_shstk = 0;
+
+    if ( opt_xen_shstk )

xen/arch/x86/setup.c void __init noreturn __start_xen(unsigned long mbi_p) · line 1110

Quoted 10 lines
xen/arch/x86/setup.c +8 -1
     {
-        printk("Enabling Supervisor Shadow Stacks\n");
+        /*
+         * Some CPUs suffer from Shadow Stack Fracturing, an issue whereby a
+         * fault/VMExit/etc between setting a Supervisor Busy bit and the
+         * event delivery completing renders the operation non-restartable.
+         * On restart, event delivery will find the Busy bit already set.
+         *
+         * This is a problem on bare metal, but outside of synthetic cases or
+         * a very badly timed #MC, it's not believed to problem.  It is a much

Nit: "... to be a problem."

Jan

#8 lore raw

Re: [PATCH 2/2] x86/shskt: Disable CET-SS on parts susceptible to fractured updates

Quoted 17 lines · Jan Beulich
Quoted 16 lines · Andrew Cooper

Refer to Intel SDM Rev 70 (Dec 2022), Vol3 17.2.3 "Supervisor Shadow Stack Token".

Architecturally, an event delivery which starts in CPL<3 and switches shadow stack will first validate the Supervisor Shadow Stack Token (setting the busy bit), then pushes CS/LIP/SSP. One example of this is an NMI interrupting Xen.

Some CPUs suffer from an issue called fracturing, whereby a fault/vmexit/etc between setting the busy bit and completing the event injection renders the action non-restartable, because when it comes time to restart, the busy bit is found to be already set.

This is far more easily encountered under virt, yet it is not the fault of the hypervisor, nor the fault of the guest kernel. The fault lies somewhere between the architectural specification, and the uarch behaviour.

Quoted 5 lines
Quoted 3 lines

Intel have allocated CPUID.7[1].ecx[18] CET_SSS to enumerate that supervisor shadow stacks are safe to use. Because of how Xen lays out its shadow stacks, fracturing is not expected to be a problem on native.

IOW that's the "contained in an aligned 32-byte region" constraint which we meet, aiui.

For practical purposes, it is "contained within a single cache line".

AMD's position is "if the OS doesn't have suitable alignment, or doesn't place the shstk on WB memory, it gets to keep any resulting pieces". They have confirmed that if there is suitable alignment and it is on a WB mapping, then no vmexit can occur which would fracture the the shadow stack.

Intel retroactively tightened the constraints in microcode on TGL/ADL so that MSR_PL?_SSP (or ISST pointers from memory) would suffer #GP if they didn't refer to the top word in an aligned 32-byte region.

But this provide to be insufficient to fix the problem, hence the new CPUID bit, and recommendation to disable supervisor shadow stacks by default under virt.

Quoted 5 lines
Quoted 5 lines

Detect this case on boot and default to not using shstk if virtualised. Specifying cet=shstk on the command line will override this heuristic and enable shadow stacks irrespective.

Quoted 8 lines

with one nit (below).

Quoted 4 lines

This ideally wants backporting to Xen 4.14. I have no idea how likely it is to need to backport the prerequisite patch for new feature words, but we've already had to do that once for security patches. OTOH, I have no idea how easy it is to trigger in non-synthetic cases.

Plus: How likely is it that Xen actually is used virtualized in production?

All of our gitlab smoke tests, a larger part of QubesOS's CI, and non-default PV-shim configurations.

As soon as we get guest CET working, then part of OSStest, and a portion of XenServer's general testing too.

It does need backporting to all general support trees.  (When I first started working on this problem with Intel, that would have included 4.14 too...)

Quoted 12 lines
Quoted 12 lines
xen/arch/x86/setup.c +5 -2
@@ -1099,11 +1095,45 @@ void __init noreturn __start_xen(unsigned long mbi_p)
     early_cpu_init();
 
     /* Choose shadow stack early, to set infrastructure up appropriately. */
-    if ( opt_xen_shstk && boot_cpu_has(X86_FEATURE_CET_SS) )
+    if ( !boot_cpu_has(X86_FEATURE_CET_SS) )
+        opt_xen_shstk = 0;
+
+    if ( opt_xen_shstk )
     {
-        printk("Enabling Supervisor Shadow Stacks\n");
+        /*
Quoted 8 lines

xen/arch/x86/setup.c void __init noreturn __start_xen(unsigned long mbi_p) · line 1110

Quoted 7 lines
xen/arch/x86/setup.c +7 -0
+         * Some CPUs suffer from Shadow Stack Fracturing, an issue whereby a
+         * fault/VMExit/etc between setting a Supervisor Busy bit and the
+         * event delivery completing renders the operation non-restartable.
+         * On restart, event delivery will find the Busy bit already set.
+         *
+         * This is a problem on bare metal, but outside of synthetic cases or
+         * a very badly timed #MC, it's not believed to problem.  It is a much

Nit: "... to be a problem."

Fixed, thanks.

~Andrew