Skip to contentSkip to conversation

s390x: initial support for virtio-mem

RFC PATCH Superseded David Hildenbrand 4 participants 12 messages Jul 10, 2020 – Jul 15, 2020

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

v2 cover s390x: initial support for virtio-mem

This wires up the initial, basic version of virito-mem for s390x. General information about virtio-mem can be found at [1] and in QEMU commit [2]. Patch #5 contains a short example for s390x.

virtio-mem for x86-64 Linux is part of v5.8-rc1. A branch with a s390x prototype can be found at:

    [email protected]:davidhildenbrand/linux.git virtio-mem-s390x

Note that the kernel should either be compiled via
CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE, or "memhp_default_state=online" should be passed on the kernel cmdline.

This series can be found at:

    [email protected]:davidhildenbrand/qemu.git virtio-mem-s390x-rfcv2

Related to s390x, we'll have to tackle migration of storage keys and storage attributes (especially, skipping unplugged parts). Not sure if I am missing something else (any ideas?). For virtio-mem in general, there are a couple of TODOs, e.g., documented in [1] and [2], both in QEMU and Linux. However, the basics are around.

I only tested this with fairly small amount of RAM in a z/VM environemnt and under TCG ...

[1] https://virtio-mem.gitlab.io/
[2] 910b25766b33 ("virtio-mem: Paravirtualized memory hot(un)plug")

RFCv1 -> RFCv2:
- "s390x/diag: no need to check for PGM_PRIVILEGED in diag308" -- Added
- "s390x/diag: implement diag260"
-- Implement according to doc (fix error cases)
-- Implement subcode 0xc.
-- Enable the new diag unconditionally
- "s390x: prepare device memory address space"
-- Expose maxram size now via diag260 (0xc), not via SCLP. Unfmodified
guests can now boot without any issues. This needed kernel changes (updated the Linux kernel branch)
- "s390x: implement virtio-mem-ccw"
-- Force virtio revision 1

David Hildenbrand (6):
s390x: move setting of maximum ram size to machine init s390x/diag: no need to check for PGM_PRIVILEGED in diag308 s390x/diag: implement diag260
s390x: prepare device memory address space
s390x: implement virtio-mem-ccw
s390x: initial support for virtio-mem

 hw/s390x/Kconfig                   |   1 +
 hw/s390x/Makefile.objs             |   1 +
 hw/s390x/s390-virtio-ccw.c         | 178 ++++++++++++++++++++++++++++-
 hw/s390x/sclp.c                    |  23 +---
 hw/s390x/virtio-ccw-mem.c          | 167 +++++++++++++++++++++++++++
 hw/s390x/virtio-ccw.h              |  13 +++
 hw/virtio/virtio-mem.c             |   2 +
 include/hw/s390x/s390-virtio-ccw.h |   3 +
 target/s390x/diag.c                |  62 +++++++++-
 target/s390x/internal.h            |   2 +
 target/s390x/kvm.c                 |  11 ++
 target/s390x/misc_helper.c         |   6 +
 target/s390x/translate.c           |   7 ++
 13 files changed, 446 insertions(+), 30 deletions(-)
 create mode 100644 hw/s390x/virtio-ccw-mem.c
Signature · 1 line
2.26.2
#1 lore raw

RFC PATCH v2 1/6 s390x: move setting of maximum ram size to machine init

As we no longer fixup the maximum ram size in sclp code, let's move setting the maximum ram size to ccw_init()->s390_memory_init(), which now looks like a better fit.

---
 hw/s390x/s390-virtio-ccw.c | 19 ++++++++++++++++---
 hw/s390x/sclp.c            | 20 +-------------------
 2 files changed, 17 insertions(+), 22 deletions(-)
hw/s390x/s390-virtio-ccw.c +16 -3
index 023fd25f2b..2e6d292c23 100644
@@ -160,13 +160,26 @@ static void virtio_ccw_register_hcalls(void)
                                    virtio_ccw_hcall_early_printk);
 }
 
-static void s390_memory_init(MemoryRegion *ram)
+static void s390_memory_init(MachineState *machine)
 {
     MemoryRegion *sysmem = get_system_memory();
     Error *local_err = NULL;
+    uint64_t hw_limit;
+    int ret;
+
+    /* We have to set the memory limit before adding any regions to sysmem. */
+    ret = s390_set_memory_limit(machine->maxram_size, &hw_limit);
+    if (ret == -E2BIG) {
+        error_report("host supports a maximum of %" PRIu64 " GB",
+                     hw_limit / GiB);
+        exit(EXIT_FAILURE);
+    } else if (ret) {
+        error_report("setting the guest size failed");
+        exit(EXIT_FAILURE);
+    }
 
     /* allocate RAM for core */
-    memory_region_add_subregion(sysmem, 0, ram);
+    memory_region_add_subregion(sysmem, 0, machine->ram);
 
     /*
      * Configure the maximum page size. As no memory devices were created
@@ -249,7 +262,7 @@ static void ccw_init(MachineState *machine)
 
     s390_sclp_init();
     /* init memory + setup max page size. Required for the CPU model */
-    s390_memory_init(machine->ram);
+    s390_memory_init(machine);
 
     /* init CPUs (incl. CPU model) early so s390_has_feature() works */
     s390_init_cpus(machine);
hw/s390x/sclp.c +1 -19
index d39f6d7785..f59195e15a 100644
@@ -327,32 +327,14 @@ void s390_sclp_init(void)
 
 static void sclp_realize(DeviceState *dev, Error **errp)
 {
-    MachineState *machine = MACHINE(qdev_get_machine());
     SCLPDevice *sclp = SCLP(dev);
-    Error *err = NULL;
-    uint64_t hw_limit;
-    int ret;
 
     /*
      * qdev_device_add searches the sysbus for TYPE_SCLP_EVENTS_BUS. As long
      * as we can't find a fitting bus via the qom tree, we have to add the
      * event facility to the sysbus, so e.g. a sclp console can be created.
      */
-    sysbus_realize(SYS_BUS_DEVICE(sclp->event_facility), &err);
-    if (err) {
-        goto out;
-    }
-
-    ret = s390_set_memory_limit(machine->maxram_size, &hw_limit);
-    if (ret == -E2BIG) {
-        error_setg(&err, "host supports a maximum of %" PRIu64 " GB",
-                   hw_limit / GiB);
-    } else if (ret) {
-        error_setg(&err, "setting the guest size failed");
-    }
-
-out:
-    error_propagate(errp, err);
+    sysbus_realize(SYS_BUS_DEVICE(sclp->event_facility), errp);
 }
 
 static void sclp_memory_init(SCLPDevice *sclp)
Signature · 1 line
2.26.2
#2 lore raw

RFC PATCH v2 2/6 s390x/diag: no need to check for PGM_PRIVILEGED in diag308

In reply to David Hildenbrand

Whenever we reach this point via KVM or TCG, we already verified that we are running in the supervisor state.

TCG checks this via IF_PRIV, KVM checks this directly in the diag instruction handler, before exiting to userspace.

---
 target/s390x/diag.c | 5 -----
 1 file changed, 5 deletions(-)
target/s390x/diag.c +0 -5
index 1a48429564..be70aecd72 100644
@@ -80,11 +80,6 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
     uint64_t subcode = env->regs[r3];
     IplParameterBlock *iplb;
 
-    if (env->psw.mask & PSW_MASK_PSTATE) {
-        s390_program_interrupt(env, PGM_PRIVILEGED, ra);
-        return;
-    }
-
     if (subcode & ~0x0ffffULL) {
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return;
Signature · 1 line
2.26.2
Acked-by on v2 2/6
#3 lore raw

Re: [PATCH RFCv2 2/6] s390x/diag: no need to check for PGM_PRIVILEGED in diag308

Quoted 7 lines · David Hildenbrand

Whenever we reach this point via KVM or TCG, we already verified that we are running in the supervisor state.

TCG checks this via IF_PRIV, KVM checks this directly in the diag instruction handler, before exiting to userspace.

I'm trusting you on the tcg part.

Quoted 21 lines
---
 target/s390x/diag.c | 5 -----
 1 file changed, 5 deletions(-)
target/s390x/diag.c +0 -5
index 1a48429564..be70aecd72 100644
@@ -80,11 +80,6 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
     uint64_t subcode = env->regs[r3];
     IplParameterBlock *iplb;
 
-    if (env->psw.mask & PSW_MASK_PSTATE) {
-        s390_program_interrupt(env, PGM_PRIVILEGED, ra);
-        return;
-    }
-
     if (subcode & ~0x0ffffULL) {
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return;
#4 lore raw

RFC PATCH v2 3/6 s390x/diag: implement diag260

In reply to David Hildenbrand

Let's implement diag260 - "Access Certain Virtual Machine Information", used under z/VM to expose the storage configuration (especially, layout of storage extends and thereby holes). For now, the returned information is completely redundant to the information exposed via SCLP.

We want to reuse diag260 in QEMU to implement memory devices - to have a mechanism to indicate to the guest OS that the initial ram size and the maximum possible physical address differ.

The Linux kernel supports diag260 (0x10) to query the available memory since v4.20. Ancient Linux versions used diag 260 (0xc), but stopped doing so a while ago.

Let's unconditionally implement the new diag, without any migration checks (e.g., compatibility machine, CPU model). Although a guest OS could observe this when migrating between QEMU evrsions, it's somewhat unlikely to ever trigger due to the way diag260 is used within a guest OS - called only once or twice during boot.

---
 target/s390x/diag.c        | 51 ++++++++++++++++++++++++++++++++++++++
 target/s390x/internal.h    |  2 ++
 target/s390x/kvm.c         | 11 ++++++++
 target/s390x/misc_helper.c |  6 +++++
 target/s390x/translate.c   |  7 ++++++
 5 files changed, 77 insertions(+)
target/s390x/diag.c +51 -0
index be70aecd72..5378fcf582 100644
@@ -23,6 +23,57 @@
 #include "hw/s390x/pv.h"
 #include "kvm_s390x.h"
 
+void handle_diag_260(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
+{
+    MachineState *ms = MACHINE(qdev_get_machine());
+    const ram_addr_t initial_ram_size = ms->ram_size;
+    const uint64_t subcode = env->regs[r3];
+
+    switch (subcode) {
+    case 0xc:
+        /* The first storage extent maps to our initial ram. */
+        env->regs[r1] = initial_ram_size - 1;
+        /* The highest addressable byte maps to the initial ram size for now. */
+        env->regs[r3] = initial_ram_size - 1;
+        break;
+    case 0x10: {
+        ram_addr_t addr, length;
+        uint64_t tmp;
+
+        if (r1 & 1) {
+            s390_program_interrupt(env, PGM_SPECIFICATION, ra);
+            return;
+        }
+
+        addr = env->regs[r1];
+        length = env->regs[r1 + 1];
+        if (!QEMU_IS_ALIGNED(addr, 16) || !QEMU_IS_ALIGNED(length, 16) ||
+            !length) {
+            s390_program_interrupt(env, PGM_SPECIFICATION, ra);
+            return;
+        }
+        if (!address_space_access_valid(&address_space_memory, addr, length,
+                                        true, MEMTXATTRS_UNSPECIFIED)) {
+            s390_program_interrupt(env, PGM_ADDRESSING, ra);
+            return;
+        }
+
+        /* Indicate our initial memory ([0 .. ram_size - 1]) */
+        tmp = cpu_to_be64(0);
+        cpu_physical_memory_write(addr, &tmp, sizeof(tmp));
+        tmp = cpu_to_be64(initial_ram_size - 1);
+        cpu_physical_memory_write(addr + sizeof(tmp), &tmp, sizeof(tmp));
+
+        /* Exactly one entry was stored, it always fits into the area. */
+        env->regs[r3] = 1;
+        setcc(env_archcpu(env), 0);
+        break;
+    }
+    default:
+        s390_program_interrupt(env, PGM_SPECIFICATION, ra);
+    }
+}
+
 int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
 {
     uint64_t func = env->regs[r1];
target/s390x/internal.h +2 -0
index b1e0ebf67f..a7a3df9a3b 100644
@@ -372,6 +372,8 @@ int mmu_translate_real(CPUS390XState *env, target_ulong raddr, int rw,
 
 
 /* misc_helper.c */
+void handle_diag_260(CPUS390XState *env, uint64_t r1, uint64_t r3,
+                     uintptr_t ra);
 int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3);
 void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3,
                      uintptr_t ra);
target/s390x/kvm.c +11 -0
index f2f75d2a57..d6de3ad86c 100644
@@ -1565,6 +1565,14 @@ static int handle_hypercall(S390CPU *cpu, struct kvm_run *run)
     return ret;
 }
 
+static void kvm_handle_diag_260(S390CPU *cpu, struct kvm_run *run)
+{
+    const uint64_t r1 = (run->s390_sieic.ipa & 0x00f0) >> 4;
+    const uint64_t r3 = run->s390_sieic.ipa & 0x000f;
+
+    handle_diag_260(&cpu->env, r1, r3, 0);
+}
+
 static void kvm_handle_diag_288(S390CPU *cpu, struct kvm_run *run)
 {
     uint64_t r1, r3;
@@ -1614,6 +1622,9 @@ static int handle_diag(S390CPU *cpu, struct kvm_run *run, uint32_t ipb)
      */
     func_code = decode_basedisp_rs(&cpu->env, ipb, NULL) & DIAG_KVM_CODE_MASK;
     switch (func_code) {
+    case 0x260:
+        kvm_handle_diag_260(cpu, run);
+        break;
     case DIAG_TIMEREVENT:
         kvm_handle_diag_288(cpu, run);
         break;
target/s390x/misc_helper.c +6 -0
index 58dbc023eb..d7274eb320 100644
@@ -116,6 +116,12 @@ void HELPER(diag)(CPUS390XState *env, uint32_t r1, uint32_t r3, uint32_t num)
     uint64_t r;
 
     switch (num) {
+    case 0x260:
+        qemu_mutex_lock_iothread();
+        handle_diag_260(env, r1, r3, GETPC());
+        qemu_mutex_unlock_iothread();
+        r = 0;
+        break;
     case 0x500:
         /* KVM hypercall */
         qemu_mutex_lock_iothread();
target/s390x/translate.c +7 -0
index 4f6f1e31cd..e3358395f0 100644
@@ -2397,6 +2397,13 @@ static DisasJumpType op_diag(DisasContext *s, DisasOps *o)
     TCGv_i32 r3 = tcg_const_i32(get_field(s, r3));
     TCGv_i32 func_code = tcg_const_i32(get_field(s, i2));
 
+    /*
+     * Diag 0x260 updates the CC - only for some subcodes. Calculate the
+     * current cc, such that the helper can simply overwrite it conditionally.
+     */
+    if (get_field(s, i2) == 0x260) {
+        gen_op_calc_cc(s);
+    }
     gen_helper_diag(cpu_env, r1, r3, func_code);
 
     tcg_temp_free_i32(func_code);
Signature · 1 line
2.26.2
#5 lore raw

Re: [PATCH RFCv2 3/6] s390x/diag: implement diag260

Quoted 81 lines · On Fri, 10 Jul 2020 17:12:36 +0200 David Hildenbrand <[email protected]> wrote:

Let's implement diag260 - "Access Certain Virtual Machine Information", used under z/VM to expose the storage configuration (especially, layout of storage extends and thereby holes). For now, the returned information is completely redundant to the information exposed via SCLP.

We want to reuse diag260 in QEMU to implement memory devices - to have a mechanism to indicate to the guest OS that the initial ram size and the maximum possible physical address differ.

The Linux kernel supports diag260 (0x10) to query the available memory since v4.20. Ancient Linux versions used diag 260 (0xc), but stopped doing so a while ago.

Let's unconditionally implement the new diag, without any migration checks (e.g., compatibility machine, CPU model). Although a guest OS could observe this when migrating between QEMU evrsions, it's somewhat unlikely to ever trigger due to the way diag260 is used within a guest OS - called only once or twice during boot.

---
 target/s390x/diag.c        | 51
++++++++++++++++++++++++++++++++++++++ target/s390x/internal.h    |
2 ++ target/s390x/kvm.c         | 11 ++++++++
 target/s390x/misc_helper.c |  6 +++++
 target/s390x/translate.c   |  7 ++++++
 5 files changed, 77 insertions(+)
target/s390x/diag.c +10 -0
index be70aecd72..5378fcf582 100644
@@ -23,6 +23,57 @@
 #include "hw/s390x/pv.h"
 #include "kvm_s390x.h"
 
+void handle_diag_260(CPUS390XState *env, uint64_t r1, uint64_t r3,
 uintptr_t ra) +{
+    MachineState *ms = MACHINE(qdev_get_machine());
+    const ram_addr_t initial_ram_size = ms->ram_size;
+    const uint64_t subcode = env->regs[r3];
+
+    switch (subcode) {
+    case 0xc:
+        /* The first storage extent maps to our initial ram. */
+        env->regs[r1] = initial_ram_size - 1;
+        /* The highest addressable byte maps to the initial ram size

for now. */

target/s390x/diag.c +14 -0
+        env->regs[r3] = initial_ram_size - 1;
+        break;
+    case 0x10: {
+        ram_addr_t addr, length;
+        uint64_t tmp;
+
+        if (r1 & 1) {
+            s390_program_interrupt(env, PGM_SPECIFICATION, ra);
+            return;
+        }
+
+        addr = env->regs[r1];
+        length = env->regs[r1 + 1];
+        if (!QEMU_IS_ALIGNED(addr, 16) || !QEMU_IS_ALIGNED(length,

16) ||

target/s390x/diag.c +5 -0
+            !length) {
+            s390_program_interrupt(env, PGM_SPECIFICATION, ra);
+            return;
+        }
+        if (!address_space_access_valid(&address_space_memory, addr,

length,
+ true,
MEMTXATTRS_UNSPECIFIED)) {

target/s390x/diag.c +9 -0
+            s390_program_interrupt(env, PGM_ADDRESSING, ra);
+            return;
+        }
+
+        /* Indicate our initial memory ([0 .. ram_size - 1]) */
+        tmp = cpu_to_be64(0);
+        cpu_physical_memory_write(addr, &tmp, sizeof(tmp));
+        tmp = cpu_to_be64(initial_ram_size - 1);
+        cpu_physical_memory_write(addr + sizeof(tmp), &tmp,
Quoted 3 lines

sizeof(tmp)); +
+ /* Exactly one entry was stored, it always fits into the
area. */

maybe I missed something, but I have the impression that your implementation of DIAG 260 always only returns the first extent?

shouldn't it return all the hotplugged areas once hotplugging is enabled?

Quoted 91 lines
target/s390x/diag.c +9 -0
+        env->regs[r3] = 1;
+        setcc(env_archcpu(env), 0);
+        break;
+    }
+    default:
+        s390_program_interrupt(env, PGM_SPECIFICATION, ra);
+    }
+}
+
 int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
 {
     uint64_t func = env->regs[r1];
target/s390x/internal.h +2 -0
index b1e0ebf67f..a7a3df9a3b 100644
@@ -372,6 +372,8 @@ int mmu_translate_real(CPUS390XState *env,
 target_ulong raddr, int rw, 
 
 /* misc_helper.c */
+void handle_diag_260(CPUS390XState *env, uint64_t r1, uint64_t r3,
+                     uintptr_t ra);
 int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3);
 void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3,
                      uintptr_t ra);
target/s390x/kvm.c +8 -0
index f2f75d2a57..d6de3ad86c 100644
@@ -1565,6 +1565,14 @@ static int handle_hypercall(S390CPU *cpu,
 struct kvm_run *run) return ret;
 }
 
+static void kvm_handle_diag_260(S390CPU *cpu, struct kvm_run *run)
+{
+    const uint64_t r1 = (run->s390_sieic.ipa & 0x00f0) >> 4;
+    const uint64_t r3 = run->s390_sieic.ipa & 0x000f;
+
+    handle_diag_260(&cpu->env, r1, r3, 0);
+}
+
 static void kvm_handle_diag_288(S390CPU *cpu, struct kvm_run *run)
 {
     uint64_t r1, r3;
@@ -1614,6 +1622,9 @@ static int handle_diag(S390CPU *cpu, struct

kvm_run *run, uint32_t ipb) */

     func_code = decode_basedisp_rs(&cpu->env, ipb, NULL) &

DIAG_KVM_CODE_MASK; switch (func_code) {

target/s390x/kvm.c +3 -0
+    case 0x260:
+        kvm_handle_diag_260(cpu, run);
+        break;
     case DIAG_TIMEREVENT:
         kvm_handle_diag_288(cpu, run);
         break;
target/s390x/misc_helper.c +6 -0
index 58dbc023eb..d7274eb320 100644
@@ -116,6 +116,12 @@ void HELPER(diag)(CPUS390XState *env, uint32_t
 r1, uint32_t r3, uint32_t num) uint64_t r;
 
     switch (num) {
+    case 0x260:
+        qemu_mutex_lock_iothread();
+        handle_diag_260(env, r1, r3, GETPC());
+        qemu_mutex_unlock_iothread();
+        r = 0;
+        break;
     case 0x500:
         /* KVM hypercall */
         qemu_mutex_lock_iothread();
target/s390x/translate.c +2 -0
index 4f6f1e31cd..e3358395f0 100644
@@ -2397,6 +2397,13 @@ static DisasJumpType op_diag(DisasContext *s,
 DisasOps *o) TCGv_i32 r3 = tcg_const_i32(get_field(s, r3));
     TCGv_i32 func_code = tcg_const_i32(get_field(s, i2));
 
+    /*
+     * Diag 0x260 updates the CC - only for some subcodes. Calculate

the
+ * current cc, such that the helper can simply overwrite it
conditionally.

target/s390x/translate.c +4 -0
+     */
+    if (get_field(s, i2) == 0x260) {
+        gen_op_calc_cc(s);
+    }
     gen_helper_diag(cpu_env, r1, r3, func_code);
 
     tcg_temp_free_i32(func_code);
#6 lore raw

Re: [PATCH RFCv2 3/6] s390x/diag: implement diag260

Quoted 82 lines · Claudio Imbrenda
Quoted 79 lines · On Fri, 10 Jul 2020 17:12:36 +0200 David Hildenbrand <[email protected]> wrote:

Let's implement diag260 - "Access Certain Virtual Machine Information", used under z/VM to expose the storage configuration (especially, layout of storage extends and thereby holes). For now, the returned information is completely redundant to the information exposed via SCLP.

We want to reuse diag260 in QEMU to implement memory devices - to have a mechanism to indicate to the guest OS that the initial ram size and the maximum possible physical address differ.

The Linux kernel supports diag260 (0x10) to query the available memory since v4.20. Ancient Linux versions used diag 260 (0xc), but stopped doing so a while ago.

Let's unconditionally implement the new diag, without any migration checks (e.g., compatibility machine, CPU model). Although a guest OS could observe this when migrating between QEMU evrsions, it's somewhat unlikely to ever trigger due to the way diag260 is used within a guest OS - called only once or twice during boot.

---
 target/s390x/diag.c        | 51
++++++++++++++++++++++++++++++++++++++ target/s390x/internal.h    |
2 ++ target/s390x/kvm.c         | 11 ++++++++
 target/s390x/misc_helper.c |  6 +++++
 target/s390x/translate.c   |  7 ++++++
 5 files changed, 77 insertions(+)
target/s390x/diag.c +10 -0
index be70aecd72..5378fcf582 100644
@@ -23,6 +23,57 @@
 #include "hw/s390x/pv.h"
 #include "kvm_s390x.h"
 
+void handle_diag_260(CPUS390XState *env, uint64_t r1, uint64_t r3,
 uintptr_t ra) +{
+    MachineState *ms = MACHINE(qdev_get_machine());
+    const ram_addr_t initial_ram_size = ms->ram_size;
+    const uint64_t subcode = env->regs[r3];
+
+    switch (subcode) {
+    case 0xc:
+        /* The first storage extent maps to our initial ram. */
+        env->regs[r1] = initial_ram_size - 1;
+        /* The highest addressable byte maps to the initial ram size

for now. */

target/s390x/diag.c +14 -0
+        env->regs[r3] = initial_ram_size - 1;
+        break;
+    case 0x10: {
+        ram_addr_t addr, length;
+        uint64_t tmp;
+
+        if (r1 & 1) {
+            s390_program_interrupt(env, PGM_SPECIFICATION, ra);
+            return;
+        }
+
+        addr = env->regs[r1];
+        length = env->regs[r1 + 1];
+        if (!QEMU_IS_ALIGNED(addr, 16) || !QEMU_IS_ALIGNED(length,

16) ||

target/s390x/diag.c +5 -0
+            !length) {
+            s390_program_interrupt(env, PGM_SPECIFICATION, ra);
+            return;
+        }
+        if (!address_space_access_valid(&address_space_memory, addr,

length,
+ true,
MEMTXATTRS_UNSPECIFIED)) {

target/s390x/diag.c +7 -0
+            s390_program_interrupt(env, PGM_ADDRESSING, ra);
+            return;
+        }
+
+        /* Indicate our initial memory ([0 .. ram_size - 1]) */
+        tmp = cpu_to_be64(0);
+        cpu_physical_memory_write(addr, &tmp, sizeof(tmp));
Quoted 8 lines
Quoted 5 lines
target/s390x/diag.c +2 -0
+        tmp = cpu_to_be64(initial_ram_size - 1);
+        cpu_physical_memory_write(addr + sizeof(tmp), &tmp,

sizeof(tmp)); +
+ /* Exactly one entry was stored, it always fits into the
area. */

maybe I missed something, but I have the impression that your implementation of DIAG 260 always only returns the first extent?

We only indicate boot memory (e.g., -m 2G), never any memory part of the device memory address space, because such memory has different semantics.

Quoted 3 lines

shouldn't it return all the hotplugged areas once hotplugging is enabled?

No, that would be dangerous and wrong. Memory ranges part of memory devices never must be indicated as part of hw/firmware interfaces to indicate valid boot memory. Memory provided via memory devices (virtio-mem, virtio-pmem, ...) has different semantics than ordinary hotplugged memory, and unmodified OSs (esp., older Linux versions) should not silently try to make use of any such memory. It's not just some hotplugged memory a guest OS should detect+use during boot as system ram. Thanks!

Signature · 3 lines
Thanks,

David / dhildenb
#7 lore raw

Re: [PATCH RFCv2 3/6] s390x/diag: implement diag260

Quoted 4 lines · David Hildenbrand
Quoted 2 lines · Claudio Imbrenda

shouldn't it return all the hotplugged areas once hotplugging is enabled?

Quoted 8 lines

No, that would be dangerous and wrong. Memory ranges part of memory devices never must be indicated as part of hw/firmware interfaces to indicate valid boot memory. Memory provided via memory devices (virtio-mem, virtio-pmem, ...) has different semantics than ordinary hotplugged memory, and unmodified OSs (esp., older Linux versions) should not silently try to make use of any such memory. It's not just some hotplugged memory a guest OS should detect+use during boot as system ram. Thanks!

How is kdump supposed to work, if there is no mechanism to figure out which memory ranges have been added dynamically to the system?

#8 lore raw

Re: [PATCH RFCv2 3/6] s390x/diag: implement diag260

Quoted 14 lines · Heiko Carstens
Quoted 4 lines · David Hildenbrand
Quoted 2 lines · Claudio Imbrenda

shouldn't it return all the hotplugged areas once hotplugging is enabled?

Quoted 8 lines

No, that would be dangerous and wrong. Memory ranges part of memory devices never must be indicated as part of hw/firmware interfaces to indicate valid boot memory. Memory provided via memory devices (virtio-mem, virtio-pmem, ...) has different semantics than ordinary hotplugged memory, and unmodified OSs (esp., older Linux versions) should not silently try to make use of any such memory. It's not just some hotplugged memory a guest OS should detect+use during boot as system ram. Thanks!

Quoted 2 lines

How is kdump supposed to work, if there is no mechanism to figure out which memory ranges have been added dynamically to the system?

Like other archs (esp x86-64), we would want to revive letting user space (kexec-tools) prepare the dump header, specifying the memory ranges. Would be necessary under KVM only, if virtio-mem is detected.

(Note: I am not sure if we would currently dump standby memory that has been onlined from a kdump kernel.)

Signature · 3 lines
Thanks,

David / dhildenb
#9 lore raw

RFC PATCH v2 4/6 s390x: prepare device memory address space

In reply to David Hildenbrand

Let's allocate the device memory information and setup the device memory address space. Expose the maximum ramsize via diag260. The RAM size returned via SCLP is not modified. Guest OSs which support virtio-mem are expected to consult diag260.

---
 hw/s390x/s390-virtio-ccw.c         | 43 ++++++++++++++++++++++++++++++
 hw/s390x/sclp.c                    |  3 ++-
 include/hw/s390x/s390-virtio-ccw.h |  3 +++
 target/s390x/diag.c                | 10 +++++--
 4 files changed, 56 insertions(+), 3 deletions(-)
hw/s390x/s390-virtio-ccw.c +43 -0
index 2e6d292c23..577590e623 100644
@@ -160,6 +160,35 @@ static void virtio_ccw_register_hcalls(void)
                                    virtio_ccw_hcall_early_printk);
 }
 
+static void s390_device_memory_init(MachineState *machine)
+{
+    MemoryRegion *sysmem = get_system_memory();
+
+    machine->device_memory = g_malloc0(sizeof(*machine->device_memory));
+
+    /* initialize device memory address space */
+    if (machine->ram_size < machine->maxram_size) {
+        ram_addr_t device_mem_size = machine->maxram_size - machine->ram_size;
+
+        if (QEMU_ALIGN_UP(machine->maxram_size, MiB) != machine->maxram_size) {
+            error_report("maximum memory size must by aligned to 1 MB");
+            exit(EXIT_FAILURE);
+        }
+
+        machine->device_memory->base = machine->ram_size;
+        if (machine->device_memory->base + device_mem_size < device_mem_size) {
+            error_report("unsupported amount of maximum memory: " RAM_ADDR_FMT,
+                         machine->maxram_size);
+            exit(EXIT_FAILURE);
+        }
+
+        memory_region_init(&machine->device_memory->mr, OBJECT(machine),
+                           "device-memory", device_mem_size);
+        memory_region_add_subregion(sysmem, machine->device_memory->base,
+                                    &machine->device_memory->mr);
+    }
+}
+
 static void s390_memory_init(MachineState *machine)
 {
     MemoryRegion *sysmem = get_system_memory();
@@ -194,6 +223,11 @@ static void s390_memory_init(MachineState *machine)
     s390_skeys_init();
     /* Initialize storage attributes device */
     s390_stattrib_init();
+
+    /* Support for memory devices is glued to compat machines. */
+    if (memory_devices_allowed()) {
+        s390_device_memory_init(machine);
+    }
 }
 
 static void s390_init_ipl_dev(const char *kernel_filename,
@@ -617,6 +651,7 @@ static void ccw_machine_class_init(ObjectClass *oc, void *data)
     s390mc->cpu_model_allowed = true;
     s390mc->css_migration_enabled = true;
     s390mc->hpage_1m_allowed = true;
+    s390mc->memory_devices_allowed = true;
     mc->init = ccw_init;
     mc->reset = s390_machine_reset;
     mc->hot_add_cpu = s390_hot_add_cpu;
@@ -713,6 +748,11 @@ bool hpage_1m_allowed(void)
     return get_machine_class()->hpage_1m_allowed;
 }
 
+bool memory_devices_allowed(void)
+{
+    return get_machine_class()->memory_devices_allowed;
+}
+
 static char *machine_get_loadparm(Object *obj, Error **errp)
 {
     S390CcwMachineState *ms = S390_CCW_MACHINE(obj);
@@ -831,8 +871,11 @@ static void ccw_machine_5_0_instance_options(MachineState *machine)
 
 static void ccw_machine_5_0_class_options(MachineClass *mc)
 {
+    S390CcwMachineClass *s390mc = S390_MACHINE_CLASS(mc);
+
     ccw_machine_5_1_class_options(mc);
     compat_props_add(mc->compat_props, hw_compat_5_0, hw_compat_5_0_len);
+    s390mc->memory_devices_allowed = false;
 }
 DEFINE_CCW_MACHINE(5_0, "5.0", false);
 
hw/s390x/sclp.c +2 -1
index f59195e15a..972995d928 100644
@@ -22,6 +22,7 @@
 #include "hw/s390x/event-facility.h"
 #include "hw/s390x/s390-pci-bus.h"
 #include "hw/s390x/ipl.h"
+#include "hw/s390x/s390-virtio-ccw.h"
 
 static inline SCLPDevice *get_sclp_device(void)
 {
@@ -110,7 +111,7 @@ static void read_SCP_info(SCLPDevice *sclp, SCCB *sccb)
         read_info->rnsize2 = cpu_to_be32(rnsize);
     }
 
-    /* we don't support standby memory, maxram_size is never exposed */
+    /* We don't support standby memory, maxram_size is exposed via diag260. */
     rnmax = machine->ram_size >> sclp->increment_size;
     if (rnmax < 0x10000) {
         read_info->rnmax = cpu_to_be16(rnmax);
include/hw/s390x/s390-virtio-ccw.h +3 -0
index cd1dccc6e3..efbd9f1da1 100644
@@ -41,6 +41,7 @@ typedef struct S390CcwMachineClass {
     bool cpu_model_allowed;
     bool css_migration_enabled;
     bool hpage_1m_allowed;
+    bool memory_devices_allowed;
 } S390CcwMachineClass;
 
 /* runtime-instrumentation allowed by the machine */
@@ -49,6 +50,8 @@ bool ri_allowed(void);
 bool cpu_model_allowed(void);
 /* 1M huge page mappings allowed by the machine */
 bool hpage_1m_allowed(void);
+/* Allow memory devices are allowed (memory device address space created). */
+bool memory_devices_allowed(void);
 
 /**
  * Returns true if (vmstate based) migration of the channel subsystem
target/s390x/diag.c +8 -2
index 5378fcf582..9abd23b64a 100644
@@ -27,14 +27,20 @@ void handle_diag_260(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
 {
     MachineState *ms = MACHINE(qdev_get_machine());
     const ram_addr_t initial_ram_size = ms->ram_size;
+    const ram_addr_t maximum_ram_size = ms->maxram_size;
     const uint64_t subcode = env->regs[r3];
 
     switch (subcode) {
     case 0xc:
         /* The first storage extent maps to our initial ram. */
         env->regs[r1] = initial_ram_size - 1;
-        /* The highest addressable byte maps to the initial ram size for now. */
-        env->regs[r3] = initial_ram_size - 1;
+        if (memory_devices_allowed()) {
+            /* The highest addressable byte maps to the maximum ram size. */
+            env->regs[r3] = maximum_ram_size - 1;
+        } else {
+            /* The highest addressable byte maps to the initial ram size. */
+            env->regs[r3] = initial_ram_size - 1;
+        }
         break;
     case 0x10: {
         ram_addr_t addr, length;
Signature · 1 line
2.26.2
#10 lore raw

RFC PATCH v2 5/6 s390x: implement virtio-mem-ccw

In reply to David Hildenbrand

Add a proper CCW proxy device, similar to the PCI variant.

---
 hw/s390x/virtio-ccw-mem.c | 167 ++++++++++++++++++++++++++++++++++++++
 hw/s390x/virtio-ccw.h     |  13 +++
 2 files changed, 180 insertions(+)
 create mode 100644 hw/s390x/virtio-ccw-mem.c
hw/s390x/virtio-ccw-mem.c new file +167 -0
new file mode 100644
index 0000000000..7b45c68ce4
@@ -0,0 +1,167 @@
+/*
+ * Virtio MEM CCW device
+ *
+ * Copyright (C) 2020 Red Hat, Inc.
+ *
+ * Authors:
+ *  David Hildenbrand <[email protected]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/qdev-properties.h"
+#include "hw/virtio/virtio.h"
+#include "qapi/error.h"
+#include "qemu/module.h"
+#include "virtio-ccw.h"
+#include "hw/mem/memory-device.h"
+#include "qapi/qapi-events-misc.h"
+
+static void virtio_ccw_mem_realize(VirtioCcwDevice *ccw_dev, Error **errp)
+{
+    VirtIOMEMCcw *ccw_mem = VIRTIO_MEM_CCW(ccw_dev);
+    DeviceState *vdev = DEVICE(&ccw_mem->vdev);
+
+    qdev_realize(vdev, BUS(&ccw_dev->bus), errp);
+}
+
+static void virtio_ccw_mem_set_addr(MemoryDeviceState *md, uint64_t addr,
+                                    Error **errp)
+{
+    object_property_set_uint(OBJECT(md), addr, VIRTIO_MEM_ADDR_PROP, errp);
+}
+
+static uint64_t virtio_ccw_mem_get_addr(const MemoryDeviceState *md)
+{
+    return object_property_get_uint(OBJECT(md), VIRTIO_MEM_ADDR_PROP,
+                                    &error_abort);
+}
+
+static MemoryRegion *virtio_ccw_mem_get_memory_region(MemoryDeviceState *md,
+                                                      Error **errp)
+{
+    VirtIOMEMCcw *ccw_mem = VIRTIO_MEM_CCW(md);
+    VirtIOMEM *vmem = VIRTIO_MEM(&ccw_mem->vdev);
+    VirtIOMEMClass *vmc = VIRTIO_MEM_GET_CLASS(vmem);
+
+    return vmc->get_memory_region(vmem, errp);
+}
+
+static uint64_t virtio_ccw_mem_get_plugged_size(const MemoryDeviceState *md,
+                                                Error **errp)
+{
+    return object_property_get_uint(OBJECT(md), VIRTIO_MEM_SIZE_PROP,
+                                    errp);
+}
+
+static void virtio_ccw_mem_fill_device_info(const MemoryDeviceState *md,
+                                            MemoryDeviceInfo *info)
+{
+    VirtioMEMDeviceInfo *vi = g_new0(VirtioMEMDeviceInfo, 1);
+    VirtIOMEMCcw *ccw_mem = VIRTIO_MEM_CCW(md);
+    VirtIOMEM *vmem = VIRTIO_MEM(&ccw_mem->vdev);
+    VirtIOMEMClass *vpc = VIRTIO_MEM_GET_CLASS(vmem);
+    DeviceState *dev = DEVICE(md);
+
+    if (dev->id) {
+        vi->has_id = true;
+        vi->id = g_strdup(dev->id);
+    }
+
+    /* let the real device handle everything else */
+    vpc->fill_device_info(vmem, vi);
+
+    info->u.virtio_mem.data = vi;
+    info->type = MEMORY_DEVICE_INFO_KIND_VIRTIO_MEM;
+}
+
+static void virtio_ccw_mem_size_change_notify(Notifier *notifier, void *data)
+{
+    VirtIOMEMCcw *ccw_mem = container_of(notifier, VirtIOMEMCcw,
+                                         size_change_notifier);
+    DeviceState *dev = DEVICE(ccw_mem);
+    const uint64_t * const size_p = data;
+    const char *id = NULL;
+
+    if (dev->id) {
+        id = g_strdup(dev->id);
+    }
+
+    qapi_event_send_memory_device_size_change(!!id, id, *size_p);
+}
+
+static void virtio_ccw_mem_instance_init(Object *obj)
+{
+    VirtioCcwDevice *ccw_dev = VIRTIO_CCW_DEVICE(obj);
+    VirtIOMEMCcw *ccw_mem = VIRTIO_MEM_CCW(obj);
+    VirtIOMEMClass *vmc;
+    VirtIOMEM *vmem;
+
+    ccw_dev->force_revision_1 = true;
+    virtio_instance_init_common(obj, &ccw_mem->vdev, sizeof(ccw_mem->vdev),
+                                TYPE_VIRTIO_MEM);
+
+    ccw_mem->size_change_notifier.notify = virtio_ccw_mem_size_change_notify;
+    vmem = VIRTIO_MEM(&ccw_mem->vdev);
+    vmc = VIRTIO_MEM_GET_CLASS(vmem);
+    /*
+     * We never remove the notifier again, as we expect both devices to
+     * disappear at the same time.
+     */
+    vmc->add_size_change_notifier(vmem, &ccw_mem->size_change_notifier);
+
+    object_property_add_alias(obj, VIRTIO_MEM_BLOCK_SIZE_PROP,
+                              OBJECT(&ccw_mem->vdev),
+                              VIRTIO_MEM_BLOCK_SIZE_PROP);
+    object_property_add_alias(obj, VIRTIO_MEM_SIZE_PROP, OBJECT(&ccw_mem->vdev),
+                              VIRTIO_MEM_SIZE_PROP);
+    object_property_add_alias(obj, VIRTIO_MEM_REQUESTED_SIZE_PROP,
+                              OBJECT(&ccw_mem->vdev),
+                              VIRTIO_MEM_REQUESTED_SIZE_PROP);
+}
+
+static Property virtio_ccw_mem_properties[] = {
+    DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
+                    VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void virtio_ccw_mem_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    VirtIOCCWDeviceClass *k = VIRTIO_CCW_DEVICE_CLASS(klass);
+    MemoryDeviceClass *mdc = MEMORY_DEVICE_CLASS(klass);
+
+    k->realize = virtio_ccw_mem_realize;
+    device_class_set_props(dc, virtio_ccw_mem_properties);
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+
+    mdc->get_addr = virtio_ccw_mem_get_addr;
+    mdc->set_addr = virtio_ccw_mem_set_addr;
+    mdc->get_plugged_size = virtio_ccw_mem_get_plugged_size;
+    mdc->get_memory_region = virtio_ccw_mem_get_memory_region;
+    mdc->fill_device_info = virtio_ccw_mem_fill_device_info;
+}
+
+static const TypeInfo virtio_ccw_mem = {
+    .name = TYPE_VIRTIO_MEM_CCW,
+    .parent = TYPE_VIRTIO_CCW_DEVICE,
+    .instance_size = sizeof(VirtIOMEMCcw),
+    .instance_init = virtio_ccw_mem_instance_init,
+    .class_init = virtio_ccw_mem_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_MEMORY_DEVICE },
+        { }
+    },
+};
+
+static void virtio_ccw_mem_register(void)
+{
+    type_register_static(&virtio_ccw_mem);
+}
+
+type_init(virtio_ccw_mem_register)
hw/s390x/virtio-ccw.h +13 -0
index c0e3355248..77aa87c41f 100644
@@ -29,6 +29,7 @@
 #endif /* CONFIG_VHOST_VSOCK */
 #include "hw/virtio/virtio-gpu.h"
 #include "hw/virtio/virtio-input.h"
+#include "hw/virtio/virtio-mem.h"
 
 #include "hw/s390x/s390_flic.h"
 #include "hw/s390x/css.h"
@@ -256,4 +257,16 @@ typedef struct VirtIOInputHIDCcw {
     VirtIOInputHID vdev;
 } VirtIOInputHIDCcw;
 
+/* virtio-mem-ccw */
+
+#define TYPE_VIRTIO_MEM_CCW "virtio-mem-ccw"
+#define VIRTIO_MEM_CCW(obj) \
+        OBJECT_CHECK(VirtIOMEMCcw, (obj), TYPE_VIRTIO_MEM_CCW)
+
+typedef struct VirtIOMEMCcw {
+    VirtioCcwDevice parent_obj;
+    VirtIOMEM vdev;
+    Notifier size_change_notifier;
+} VirtIOMEMCcw;
+
 #endif
Signature · 1 line
2.26.2
#11 lore raw

RFC PATCH v2 6/6 s390x: initial support for virtio-mem

In reply to David Hildenbrand

Let's wire up the initial, basic virtio-mem implementation in QEMU. It will have to see some important extensions (esp., resizeable allocations) before it can be considered production ready. Also, the focus on the Linux driver side is on memory hotplug, there are a lot of things optimize in the future to improve memory unplug capabilities. However, the basics are in place.

Block migration for now, as we'll have to take proper care of storage keys and storage attributes. Also, make sure to not hotplug huge pages to a setup without huge pages.

With a Linux guest that supports virtio-mem (and has CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE set for now), a basic example.

1. Start a VM with 2G initial memory and a virtio-mem device with a maximum capacity of 18GB (and an initial size of 300M):

    sudo qemu-system-s390x \
        --enable-kvm \
        -m 2G,maxmem=20G \
        -smp 4 \
        -nographic \
        -chardev socket,id=monitor,path=/var/tmp/monitor,server,nowait \
        -mon chardev=monitor,mode=readline \
        -net nic -net user \
        -hda s390x.cow2 \
        -object memory-backend-ram,id=mem0,size=18G \
        -device virtio-mem-ccw,id=vm0,memdev=mem0,requested-size=300M

2. Query the current size of virtio-mem device:

    (qemu) info memory-devices
    Memory device [virtio-mem]: "vm0"
      memaddr: 0x80000000
      node: 0
      requested-size: 314572800
      size: 314572800
      max-size: 19327352832
      block-size: 1048576
      memdev: /objects/mem0

3. Request to grow it to 8GB:

    (qemu) qom-set vm0 requested-size 8G
    (qemu) info memory-devices
    Memory device [virtio-mem]: "vm0"
      memaddr: 0x80000000
      node: 0
      requested-size: 8589934592
      size: 8589934592
      max-size: 19327352832
      block-size: 1048576
      memdev: /objects/mem0

4. Request to shrink it to 800M (might take a while, might not fully succeed, and might not be able to remove memory blocks in Linux):
(qemu) qom-set vm0 requested-size 800M
(qemu) info memory-devices
Memory device [virtio-mem]: "vm0"

    memaddr: 0x80000000
    node: 0
    requested-size: 838860800
    size: 838860800
    max-size: 19327352832
    block-size: 1048576
    memdev: /objects/mem0

Note: Due to lack of resizeable allocations, we will go ahead and reserve a 18GB vmalloc area + size the QEMU RAM slot + KVM mamory slot 18GB. echo 1 > /proc/sys/vm/overcommit_memory might be required for now. In the future, this area will instead grow on actual demand and shrink when possible.

---
 hw/s390x/Kconfig           |   1 +
 hw/s390x/Makefile.objs     |   1 +
 hw/s390x/s390-virtio-ccw.c | 116 ++++++++++++++++++++++++++++++++++++-
 hw/virtio/virtio-mem.c     |   2 +
 4 files changed, 118 insertions(+), 2 deletions(-)
hw/s390x/Kconfig +1 -0
index 5e7d8a2bae..b8619c1adc 100644
@@ -10,3 +10,4 @@ config S390_CCW_VIRTIO
     select SCLPCONSOLE
     select VIRTIO_CCW
     select MSI_NONBROKEN
+    select VIRTIO_MEM_SUPPORTED
hw/s390x/Makefile.objs +1 -0
index a46a1c7894..924775d6f0 100644
@@ -20,6 +20,7 @@ obj-$(CONFIG_VIRTIO_NET) += virtio-ccw-net.o
 obj-$(CONFIG_VIRTIO_BLK) += virtio-ccw-blk.o
 obj-$(call land,$(CONFIG_VIRTIO_9P),$(CONFIG_VIRTFS)) += virtio-ccw-9p.o
 obj-$(CONFIG_VHOST_VSOCK) += vhost-vsock-ccw.o
+obj-$(CONFIG_VIRTIO_MEM) += virtio-ccw-mem.o
 endif
 obj-y += css-bridge.o
 obj-y += ccw-device.o
hw/s390x/s390-virtio-ccw.c +114 -2

151 lines not on this page · raw message

hw/virtio/virtio-mem.c +2 -0
index 65850530e7..e1b3275089 100644
@@ -53,6 +53,8 @@
  */
 #if defined(TARGET_X86_64) || defined(TARGET_I386)
 #define VIRTIO_MEM_USABLE_EXTENT (2 * (128 * MiB))
+#elif defined(TARGET_S390X)
+#define VIRTIO_MEM_USABLE_EXTENT (2 * (256 * MiB))
 #else
 #error VIRTIO_MEM_USABLE_EXTENT not defined
 #endif
Signature · 1 line
2.26.2