x86: host-phys-bits-limit option

Some downstream distributions of QEMU set host-phys-bits=on by default. This worked very well for most use cases, because phys-bits really didn't have huge consequences. The only difference was on the CPUID data seen by guests, and on the handling of reserved bits.

This changed in KVM commit 855feb673640 ("KVM: MMU: Add 5 level EPT & Shadow page table support"). Now choosing a large phys-bits value for a VM has bigger impact: it will make KVM use 5-level EPT even when it's not really necessary. This means using the host phys-bits value may not be the best choice.

Management software could address this problem by manually configuring phys-bits depending on the size of the VM and the amount of MMIO address space required for hotplug. But this is not trivial to implement.

However, there's another workaround that would work for most
cases: keep using the host phys-bits value, but only if it's smaller than 48. This patch makes this possible by introducing a new "-cpu" option: "host-phys-bits-limit". Management software or users can make sure they will always use 4-level EPT using:
"host-phys-bits=on,host-phys-bits-limit=48".

This behavior is still not enabled by default because QEMU doesn't enable host-phys-bits=on by default. But users, management software, or downstream distributions may choose to change their defaults using the new option.

Signed-off-by Eduardo Habkost <[email protected]>

---
 target/i386/cpu.h                 |  3 ++
 target/i386/cpu.c                 |  5 ++
 tests/acceptance/x86-phys-bits.py | 81 +++++++++++++++++++++++++++++++
 3 files changed, 89 insertions(+)
 create mode 100644 tests/acceptance/x86-phys-bits.py

target/i386/cpu.h +3 -0

index 9c52d0cbeb..a545b77c2c 100644
@@ -1457,6 +1457,9 @@ struct X86CPU {
     /* if true override the phys_bits value with a value read from the host */
     bool host_phys_bits;
 
+    /* if set, limit maximum value for phys_bits when host_phys_bits is true */
+    uint8_t host_phys_bits_limit;
+
     /* Stop SMI delivery for migration compatibility with old machines */
     bool kvm_no_smi_migration;
 

target/i386/cpu.c +5 -0

index f81d35e1f9..df200754a2 100644
@@ -5152,6 +5152,10 @@ static void x86_cpu_realizefn(DeviceState *dev, Error **errp)
             if (cpu->host_phys_bits) {
                 /* The user asked for us to use the host physical bits */
                 cpu->phys_bits = host_phys_bits;
+                if (cpu->host_phys_bits_limit &&
+                    cpu->phys_bits > cpu->host_phys_bits_limit) {
+                    cpu->phys_bits = cpu->host_phys_bits_limit;
+                }
             }
 
             /* Print a warning if the user set it to a value that's not the

@@ -5739,6 +5743,7 @@ static Property x86_cpu_properties[] = {
     DEFINE_PROP_BOOL("kvm", X86CPU, expose_kvm, true),
     DEFINE_PROP_UINT32("phys-bits", X86CPU, phys_bits, 0),
     DEFINE_PROP_BOOL("host-phys-bits", X86CPU, host_phys_bits, false),
+    DEFINE_PROP_UINT8("host-phys-bits-limit", X86CPU, host_phys_bits_limit, 0),
     DEFINE_PROP_BOOL("fill-mtrr-mask", X86CPU, fill_mtrr_mask, true),
     DEFINE_PROP_UINT32("level", X86CPU, env.cpuid_level, UINT32_MAX),
     DEFINE_PROP_UINT32("xlevel", X86CPU, env.cpuid_xlevel, UINT32_MAX),

tests/acceptance/x86-phys-bits.py new file +81 -0

new file mode 100644
index 0000000000..ae85d7e4e7
@@ -0,0 +1,81 @@
+# Test for host-phys-bits-limit option
+#
+# Copyright (c) 2018 Red Hat, Inc.
+#
+# Author:
+#  Eduardo Habkost <[email protected]>
+#
+# This work is licensed under the terms of the GNU GPL, version 2 or
+# later.  See the COPYING file in the top-level directory.
+import re
+
+from avocado_qemu import Test
+
+class HostPhysbits(Test):
+    """
+    Check if `host-phys-bits` and `host-phys-bits` options work.
+
+    :avocado: enable
+    :avocado: tags=x86_64
+    """
+
+    def cpu_qom_get(self, prop):
+        qom_path = self.vm.command('query-cpus')[0]['qom_path']
+        return self.vm.command('qom-get', path=qom_path, property=prop)
+
+    def cpu_phys_bits(self):
+        return self.cpu_qom_get('phys-bits')
+
+    def host_phys_bits(self):
+        cpuinfo = open('/proc/cpuinfo', 'rb').read()
+        m = re.search(b'([0-9]+) bits physical', cpuinfo)
+        if m is None:
+            self.cancel("Couldn't read phys-bits from /proc/cpuinfo")
+        return int(m.group(1))
+
+    def setUp(self):
+        super(HostPhysbits, self).setUp()
+        self.vm.add_args('-S', '-machine', 'accel=kvm:tcg')
+        self.vm.launch()
+        if not self.vm.command('query-kvm')['enabled']:
+            self.cancel("Test case requires KVM")
+        self.vm.shutdown()
+
+
+    def test_no_host_phys_bits(self):
+        # default should be phys-bits=40 if host-phys-bits=off
+        self.vm.add_args('-cpu', 'qemu64,host-phys-bits=off')
+        self.vm.launch()
+        self.assertEqual(self.cpu_phys_bits(), 40)
+
+    def test_manual_phys_bits(self):
+        self.vm.add_args('-cpu', 'qemu64,host-phys-bits=off,phys-bits=35')
+        self.vm.launch()
+        self.assertEqual(self.cpu_phys_bits(), 35)
+
+    def test_host_phys_bits(self):
+        host_phys_bits = self.host_phys_bits()
+        self.vm.add_args('-cpu', 'qemu64,host-phys-bits=on')
+        self.vm.launch()
+        self.assertEqual(self.cpu_phys_bits(), host_phys_bits)
+
+    def test_host_phys_bits_limit_high(self):
+        hbits = self.host_phys_bits()
+        self.vm.add_args('-cpu', 'qemu64,host-phys-bits=on,'
+                                 'host-phys-bits-limit=%d' % (hbits + 1))
+        self.vm.launch()
+        self.assertEqual(self.cpu_phys_bits(), hbits)
+
+    def test_host_phys_bits_limit_equal(self):
+        hbits = self.host_phys_bits()
+        self.vm.add_args('-cpu', 'qemu64,host-phys-bits=on,'
+                                 'host-phys-bits-limit=%d' % (hbits))
+        self.vm.launch()
+        self.assertEqual(self.cpu_phys_bits(), hbits)
+
+    def test_host_phys_bits_limit_low(self):
+        hbits = self.host_phys_bits()
+        self.vm.add_args('-cpu', 'qemu64,host-phys-bits=on,'
+                                 'host-phys-bits-limit=%d' % (hbits - 1))
+        self.vm.launch()
+        self.assertEqual(self.cpu_phys_bits(), hbits - 1)

Signature · 1 line

2.18.0.rc1.1.g3f1ff2140

Quoted 29 lines · Eduardo Habkost

Some downstream distributions of QEMU set host-phys-bits=on by default. This worked very well for most use cases, because phys-bits really didn't have huge consequences. The only difference was on the CPUID data seen by guests, and on the handling of reserved bits.

This changed in KVM commit 855feb673640 ("KVM: MMU: Add 5 level EPT & Shadow page table support"). Now choosing a large phys-bits value for a VM has bigger impact: it will make KVM use 5-level EPT even when it's not really necessary. This means using the host phys-bits value may not be the best choice.

Management software could address this problem by manually configuring phys-bits depending on the size of the VM and the amount of MMIO address space required for hotplug. But this is not trivial to implement.

However, there's another workaround that would work for most
cases: keep using the host phys-bits value, but only if it's smaller than 48. This patch makes this possible by introducing a new "-cpu" option: "host-phys-bits-limit". Management software or users can make sure they will always use 4-level EPT using:
"host-phys-bits=on,host-phys-bits-limit=48".

This behavior is still not enabled by default because QEMU doesn't enable host-phys-bits=on by default. But users, management software, or downstream distributions may choose to change their defaults using the new option.

Quoted 1 line

Signed-off-by Eduardo Habkost <[email protected]>

Thanks, Eduardo. One question is, should we check host-phys-bits-limit against 48? If not, how about we just say in the commit message, that the suggested value of host-phys-bits-limit is no bigger than 48 to ensure a 4-level EPT? :-)

Quoted 30 lines · Yu Zhang

Quoted 29 lines · Eduardo Habkost

Some downstream distributions of QEMU set host-phys-bits=on by default. This worked very well for most use cases, because phys-bits really didn't have huge consequences. The only difference was on the CPUID data seen by guests, and on the handling of reserved bits.

This changed in KVM commit 855feb673640 ("KVM: MMU: Add 5 level EPT & Shadow page table support"). Now choosing a large phys-bits value for a VM has bigger impact: it will make KVM use 5-level EPT even when it's not really necessary. This means using the host phys-bits value may not be the best choice.

Management software could address this problem by manually configuring phys-bits depending on the size of the VM and the amount of MMIO address space required for hotplug. But this is not trivial to implement.

However, there's another workaround that would work for most
cases: keep using the host phys-bits value, but only if it's smaller than 48. This patch makes this possible by introducing a new "-cpu" option: "host-phys-bits-limit". Management software or users can make sure they will always use 4-level EPT using:
"host-phys-bits=on,host-phys-bits-limit=48".

This behavior is still not enabled by default because QEMU doesn't enable host-phys-bits=on by default. But users, management software, or downstream distributions may choose to change their defaults using the new option.

Quoted 6 lines

Quoted 1 line

Signed-off-by Eduardo Habkost <[email protected]>

Thanks, Eduardo. One question is, should we check host-phys-bits-limit against 48? If not, how about we just say in the commit message, that the suggested value of host-phys-bits-limit is no bigger than 48 to ensure a 4-level EPT? :-)

I'm not sure I understood the question. I tried to document this at:

|                                      [...]  Management software
| or users can make sure they will always use 4-level EPT using:
| "host-phys-bits=on,host-phys-bits-limit=48".

Signature · 1 line

Eduardo

Quoted 38 lines · Eduardo Habkost

Quoted 30 lines · Yu Zhang

Quoted 29 lines · Eduardo Habkost

Some downstream distributions of QEMU set host-phys-bits=on by default. This worked very well for most use cases, because phys-bits really didn't have huge consequences. The only difference was on the CPUID data seen by guests, and on the handling of reserved bits.

This changed in KVM commit 855feb673640 ("KVM: MMU: Add 5 level EPT & Shadow page table support"). Now choosing a large phys-bits value for a VM has bigger impact: it will make KVM use 5-level EPT even when it's not really necessary. This means using the host phys-bits value may not be the best choice.

Management software could address this problem by manually configuring phys-bits depending on the size of the VM and the amount of MMIO address space required for hotplug. But this is not trivial to implement.

However, there's another workaround that would work for most
cases: keep using the host phys-bits value, but only if it's smaller than 48. This patch makes this possible by introducing a new "-cpu" option: "host-phys-bits-limit". Management software or users can make sure they will always use 4-level EPT using:
"host-phys-bits=on,host-phys-bits-limit=48".

This behavior is still not enabled by default because QEMU doesn't enable host-phys-bits=on by default. But users, management software, or downstream distributions may choose to change their defaults using the new option.

Quoted 6 lines

Quoted 1 line

Signed-off-by Eduardo Habkost <[email protected]>

Thanks, Eduardo. One question is, should we check host-phys-bits-limit against 48? If not, how about we just say in the commit message, that the suggested value of host-phys-bits-limit is no bigger than 48 to ensure a 4-level EPT? :-)

Quoted 7 lines

I'm not sure I understood the question. I tried to document this at:

|                                      [...]  Management software
| or users can make sure they will always use 4-level EPT using:
| "host-phys-bits=on,host-phys-bits-limit=48".

Oh, I was just saying that host-phys-bits-limit can be of any value less than 48, instead of just 48, to ensure a 4-level EPT. Not a big deal. I guess readers can get the point in your commit message.

Another question is, should we check the value of host-phys-bits? Shall we accept values greater than 48(may be a useless configuration, but still acceptible), or less than 32?

Some downstream distributions of QEMU set host-phys-bits=on by default. This worked very well for most use cases, because phys-bits really didn't have huge consequences. The only difference was on the CPUID data seen by guests, and on the handling of reserved bits.

This changed in KVM commit 855feb673640 ("KVM: MMU: Add 5 level EPT & Shadow page table support"). Now choosing a large phys-bits value for a VM has bigger impact: it will make KVM use 5-level EPT even when it's not really necessary. This means using the host phys-bits value may not be the best choice.

Management software could address this problem by manually configuring phys-bits depending on the size of the VM and the amount of MMIO address space required for hotplug. But this is not trivial to implement.

However, there's another workaround that would work for most
cases: keep using the host phys-bits value, but only if it's smaller than 48. This patch makes this possible by introducing a new "-cpu" option: "host-phys-bits-limit". Management software or users can make sure they will always use 4-level EPT using:
"host-phys-bits=on,host-phys-bits-limit=48".

This behavior is still not enabled by default because QEMU doesn't enable host-phys-bits=on by default. But users, management software, or downstream distributions may choose to change their defaults using the new option.

Signed-off-by Eduardo Habkost <[email protected]>

---
  target/i386/cpu.h                 |  3 ++
  target/i386/cpu.c                 |  5 ++
  tests/acceptance/x86-phys-bits.py | 81 +++++++++++++++++++++++++++++++
  3 files changed, 89 insertions(+)

create mode 100644 tests/acceptance/x86-phys-bits.py

target/i386/cpu.h +3 -0

index 9c52d0cbeb..a545b77c2c 100644
@@ -1457,6 +1457,9 @@ struct X86CPU {
      /* if true override the phys_bits value with a value read from the host */
      bool host_phys_bits;
 
+    /* if set, limit maximum value for phys_bits when host_phys_bits is true */
+    uint8_t host_phys_bits_limit;
+
      /* Stop SMI delivery for migration compatibility with old machines */
      bool kvm_no_smi_migration;
 

target/i386/cpu.c +4 -0

index f81d35e1f9..df200754a2 100644
@@ -5152,6 +5152,10 @@ static void x86_cpu_realizefn(DeviceState *dev, Error **errp)
              if (cpu->host_phys_bits) {
                  /* The user asked for us to use the host physical bits */
                  cpu->phys_bits = host_phys_bits;
+                if (cpu->host_phys_bits_limit &&
+                    cpu->phys_bits > cpu->host_phys_bits_limit) {
+                    cpu->phys_bits = cpu->host_phys_bits_limit;
+                }
              }
 
              /* Print a warning if the user set it to a value that's not the

tests/acceptance/x86-phys-bits.py new file +6 -0

@@ -0,0 +1,81 @@
+# Test for host-phys-bits-limit option
+#
+# Copyright (c) 2018 Red Hat, Inc.
+#
+# Author:
+#  Eduardo Habkost <[email protected]>

tests/acceptance/x86-phys-bits.py new file +5 -0

+
+    :avocado: enable
+    :avocado: tags=x86_64
+    """
+

tests/acceptance/x86-phys-bits.py new file +25 -0

+        self.vm.add_args('-cpu', 'qemu64,host-phys-bits=off')
+        self.vm.launch()
+        self.assertEqual(self.cpu_phys_bits(), 40)
+
+    def test_manual_phys_bits(self):
+        self.vm.add_args('-cpu', 'qemu64,host-phys-bits=off,phys-bits=35')
+        self.vm.launch()
+        self.assertEqual(self.cpu_phys_bits(), 35)
+
+    def test_host_phys_bits(self):
+        host_phys_bits = self.host_phys_bits()
+        self.vm.add_args('-cpu', 'qemu64,host-phys-bits=on')
+        self.vm.launch()
+        self.assertEqual(self.cpu_phys_bits(), host_phys_bits)
+
+    def test_host_phys_bits_limit_high(self):
+        hbits = self.host_phys_bits()
+        self.vm.add_args('-cpu', 'qemu64,host-phys-bits=on,'
+                                 'host-phys-bits-limit=%d' % (hbits + 1))
+        self.vm.launch()
+        self.assertEqual(self.cpu_phys_bits(), hbits)
+
+    def test_host_phys_bits_limit_equal(self):
+        hbits = self.host_phys_bits()
+        self.vm.add_args('-cpu', 'qemu64,host-phys-bits=on,'

Quoted 64 lines · Eduardo Habkost

Some downstream distributions of QEMU set host-phys-bits=on by default. This worked very well for most use cases, because phys-bits really didn't have huge consequences. The only difference was on the CPUID data seen by guests, and on the handling of reserved bits.

This changed in KVM commit 855feb673640 ("KVM: MMU: Add 5 level EPT & Shadow page table support"). Now choosing a large phys-bits value for a VM has bigger impact: it will make KVM use 5-level EPT even when it's not really necessary. This means using the host phys-bits value may not be the best choice.

Management software could address this problem by manually configuring phys-bits depending on the size of the VM and the amount of MMIO address space required for hotplug. But this is not trivial to implement.

However, there's another workaround that would work for most
cases: keep using the host phys-bits value, but only if it's smaller than 48. This patch makes this possible by introducing a new "-cpu" option: "host-phys-bits-limit". Management software or users can make sure they will always use 4-level EPT using:
"host-phys-bits=on,host-phys-bits-limit=48".

This behavior is still not enabled by default because QEMU doesn't enable host-phys-bits=on by default. But users, management software, or downstream distributions may choose to change their defaults using the new option.

Signed-off-by Eduardo Habkost <[email protected]>

---
  target/i386/cpu.h                 |  3 ++
  target/i386/cpu.c                 |  5 ++
  tests/acceptance/x86-phys-bits.py | 81 +++++++++++++++++++++++++++++++
  3 files changed, 89 insertions(+)

create mode 100644 tests/acceptance/x86-phys-bits.py

target/i386/cpu.h +3 -0

index 9c52d0cbeb..a545b77c2c 100644
@@ -1457,6 +1457,9 @@ struct X86CPU {
      /* if true override the phys_bits value with a value read from the host */
      bool host_phys_bits;
+    /* if set, limit maximum value for phys_bits when host_phys_bits is true */
+    uint8_t host_phys_bits_limit;
+
      /* Stop SMI delivery for migration compatibility with old machines */
      bool kvm_no_smi_migration;

target/i386/cpu.c +4 -0

index f81d35e1f9..df200754a2 100644
@@ -5152,6 +5152,10 @@ static void x86_cpu_realizefn(DeviceState *dev, Error **errp)
              if (cpu->host_phys_bits) {
                  /* The user asked for us to use the host physical bits */
                  cpu->phys_bits = host_phys_bits;
+                if (cpu->host_phys_bits_limit &&
+                    cpu->phys_bits > cpu->host_phys_bits_limit) {
+                    cpu->phys_bits = cpu->host_phys_bits_limit;
+                }
              }
              /* Print a warning if the user set it to a value that's not the

target/i386/cpu.c static Property x86_cpu_properties[] = { · line 5749

Quoted 7 lines

target/i386/cpu.c +1 -0

@@ -5739,6 +5743,7 @@ static Property x86_cpu_properties[] = {
      DEFINE_PROP_BOOL("kvm", X86CPU, expose_kvm, true),
      DEFINE_PROP_UINT32("phys-bits", X86CPU, phys_bits, 0),
      DEFINE_PROP_BOOL("host-phys-bits", X86CPU, host_phys_bits, false),
+    DEFINE_PROP_UINT8("host-phys-bits-limit", X86CPU, host_phys_bits_limit, 0),
      DEFINE_PROP_BOOL("fill-mtrr-mask", X86CPU, fill_mtrr_mask, true),
      DEFINE_PROP_UINT32("level", X86CPU, env.cpuid_level, UINT32_MAX),
      DEFINE_PROP_UINT32("xlevel", X86CPU, env.cpuid_xlevel, UINT32_MAX),

The changes to introduce the host-phys-bits-limit option looks good to me, although I haven't much knowledge on this area anyway...

tests/acceptance/x86-phys-bits.py

Quoted 5 lines

tests/acceptance/x86-phys-bits.py new file +0 -0

new file mode 100644
index 0000000000..ae85d7e4e7

PEP8 does not have a convention for Python files naming, although it suggests the use of underscore on module and package names. We already have the tests/acceptance/boot_linux_console.py with underscore, so I think we should make it a convention.

Quoted 13 lines

tests/acceptance/x86-phys-bits.py new file +10 -0

@@ -0,0 +1,81 @@
+# Test for host-phys-bits-limit option
+#
+# Copyright (c) 2018 Red Hat, Inc.
+#
+# Author:
+#  Eduardo Habkost <[email protected]>
+#
+# This work is licensed under the terms of the GNU GPL, version 2 or
+# later.  See the COPYING file in the top-level directory.
+import re

tests/acceptance/x86-phys-bits.py line 16

Quoted 6 lines

tests/acceptance/x86-phys-bits.py new file +6 -0

+
+from avocado_qemu import Test
+
+class HostPhysbits(Test):
+    """
+    Check if `host-phys-bits` and `host-phys-bits` options work.

Did you mean "host-phys-bits and host-phys-bits-limit?

Quoted 6 lines

tests/acceptance/x86-phys-bits.py new file +5 -0

+
+    :avocado: enable
+    :avocado: tags=x86_64
+    """
+

tests/acceptance/x86-phys-bits.py line 31

Quoted 10 lines

tests/acceptance/x86-phys-bits.py new file +10 -0

+    def cpu_qom_get(self, prop):
+        qom_path = self.vm.command('query-cpus')[0]['qom_path']
+        return self.vm.command('qom-get', path=qom_path, property=prop)
+
+    def cpu_phys_bits(self):
+        return self.cpu_qom_get('phys-bits')
+
+    def host_phys_bits(self):
+        cpuinfo = open('/proc/cpuinfo', 'rb').read()
+        m = re.search(b'([0-9]+) bits physical', cpuinfo)

pylint says:

"tests/acceptance/x86-phys-bits.py:31:8: C0103: Variable name "m" doesn't conform to snake_case naming style (invalid-name)"

It also warns about "Missing method docstring" for each test method. I don't know whether we should convention that every method must have a docstring explaining the test, or just ignore that warn.

tests/acceptance/x86-phys-bits.py line 33

Quoted 3 lines

tests/acceptance/x86-phys-bits.py new file +2 -0

+        if m is None:
+            self.cancel("Couldn't read phys-bits from /proc/cpuinfo")

I suggest something like 'bits physical size' instead of 'phys-bits'.

tests/acceptance/x86-phys-bits.py line 38

Quoted 6 lines

tests/acceptance/x86-phys-bits.py new file +5 -0

+        return int(m.group(1))
+
+    def setUp(self):
+        super(HostPhysbits, self).setUp()
+        self.vm.add_args('-S', '-machine', 'accel=kvm:tcg')

I'm curious to know why you freeze the CPU at start (-S).

tests/acceptance/x86-phys-bits.py line 46

Quoted 9 lines

tests/acceptance/x86-phys-bits.py new file +8 -0

+        self.vm.launch()
+        if not self.vm.command('query-kvm')['enabled']:
+            self.cancel("Test case requires KVM")
+        self.vm.shutdown()
+
+
+    def test_no_host_phys_bits(self):
+        # default should be phys-bits=40 if host-phys-bits=off

Perhaps put that comment in a docstring block.

Quoted 4 lines

tests/acceptance/x86-phys-bits.py new file +3 -0

+        self.vm.add_args('-cpu', 'qemu64,host-phys-bits=off')
+        self.vm.launch()
+        self.assertEqual(self.cpu_phys_bits(), 40)

Suppose an user sets both host-phys-bits and phys-bits by mistake. What is the expected outcome? If this case is relevant...below test case fails (phys-bits is set to host's):

    def test_manual_phys_bits_mistake(self):
        """
        By mistake set host-phys-bits=on and phys-bits
        """
        self.vm.add_args('-cpu', 'qemu64,host-phys-bits=on,phys-bits=35') self.vm.launch() self.assertEqual(self.cpu_phys_bits(), 35)