Skip to contentSkip to conversation

selftest: KVM: Add intra host migration

PATCH In review Peter Gonda 2 participants 5 messages Aug 30, 2021 – Aug 31, 2021

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH v6 3/3 selftest: KVM: Add intra host migration

Adds testcases for intra host migration for SEV and SEV-ES. Also adds locking test to confirm no deadlock exists.

---
 tools/testing/selftests/kvm/Makefile          |   1 +
 .../selftests/kvm/x86_64/sev_vm_tests.c       | 152 ++++++++++++++++++
 2 files changed, 153 insertions(+)
 create mode 100644 tools/testing/selftests/kvm/x86_64/sev_vm_tests.c
tools/testing/selftests/kvm/Makefile +1 -0
index 5832f510a16c..de6e64d5c9c4 100644
@@ -71,6 +71,7 @@ TEST_GEN_PROGS_x86_64 += x86_64/tsc_msrs_test
 TEST_GEN_PROGS_x86_64 += x86_64/vmx_pmu_msrs_test
 TEST_GEN_PROGS_x86_64 += x86_64/xen_shinfo_test
 TEST_GEN_PROGS_x86_64 += x86_64/xen_vmcall_test
+TEST_GEN_PROGS_x86_64 += x86_64/sev_vm_tests
 TEST_GEN_PROGS_x86_64 += access_tracking_perf_test
 TEST_GEN_PROGS_x86_64 += demand_paging_test
 TEST_GEN_PROGS_x86_64 += dirty_log_test
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +150 -0

150 lines not on this page · raw message

Signature · 1 line
2.33.0.259.gc128427fd7-goog
#1 lore raw
Quoted 86 lines · Peter Gonda

Adds testcases for intra host migration for SEV and SEV-ES. Also adds locking test to confirm no deadlock exists.

---
 tools/testing/selftests/kvm/Makefile          |   1 +
 .../selftests/kvm/x86_64/sev_vm_tests.c       | 152 ++++++++++++++++++
 2 files changed, 153 insertions(+)
 create mode 100644 tools/testing/selftests/kvm/x86_64/sev_vm_tests.c
tools/testing/selftests/kvm/Makefile +1 -0
index 5832f510a16c..de6e64d5c9c4 100644
@@ -71,6 +71,7 @@ TEST_GEN_PROGS_x86_64 += x86_64/tsc_msrs_test
 TEST_GEN_PROGS_x86_64 += x86_64/vmx_pmu_msrs_test
 TEST_GEN_PROGS_x86_64 += x86_64/xen_shinfo_test
 TEST_GEN_PROGS_x86_64 += x86_64/xen_vmcall_test
+TEST_GEN_PROGS_x86_64 += x86_64/sev_vm_tests
 TEST_GEN_PROGS_x86_64 += access_tracking_perf_test
 TEST_GEN_PROGS_x86_64 += demand_paging_test
 TEST_GEN_PROGS_x86_64 += dirty_log_test
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +50 -0
new file mode 100644
index 000000000000..50a770316628
@@ -0,0 +1,150 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <linux/kvm.h>
+#include <linux/psp-sev.h>
+#include <stdio.h>
+#include <sys/ioctl.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <pthread.h>
+
+#include "test_util.h"
+#include "kvm_util.h"
+#include "processor.h"
+#include "svm_util.h"
+#include "kvm_util.h"
+#include "kselftest.h"
+#include "../lib/kvm_util_internal.h"
+
+#define SEV_DEV_PATH "/dev/sev"
+
+/*
+ * Open SEV_DEV_PATH if available, otherwise exit the entire program.
+ *
+ * Input Args:
+ *   flags - The flags to pass when opening SEV_DEV_PATH.
+ *
+ * Return:
+ *   The opened file descriptor of /dev/sev.
+ */
+static int open_sev_dev_path_or_exit(int flags)
+{
+       static int fd;
+
+       if (fd != 0)
+               return fd;
+
+       fd = open(SEV_DEV_PATH, flags);
+       if (fd < 0) {
+               print_skip("%s not available, is SEV not enabled? (errno: %d)",
+                          SEV_DEV_PATH, errno);
+               exit(KSFT_SKIP);
+       }
+
+       return fd;
+}
+
+static void sev_ioctl(int fd, int cmd_id, void *data)
+{
+       struct kvm_sev_cmd cmd = { 0 };
+       int ret;
+

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 60

Quoted 10 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +10 -0
+       TEST_ASSERT(cmd_id < KVM_SEV_NR_MAX, "Unknown SEV CMD : %d\n", cmd_id);
+
+       cmd.id = cmd_id;
+       cmd.sev_fd = open_sev_dev_path_or_exit(0);
+       cmd.data = (uint64_t)data;
+       ret = ioctl(fd, KVM_MEMORY_ENCRYPT_OP, &cmd);
+       TEST_ASSERT((ret == 0 || cmd.error == SEV_RET_SUCCESS),
+                   "%d failed: return code: %d, errno: %d, fw error: %d",
+                   cmd_id, ret, errno, cmd.error);
+}

nit: Since this function has two file descriptors, fd and cmd.sev_fd, can we rename fd to vm_fd?

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 70

Quoted 10 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +10 -0
+
+static struct kvm_vm *sev_vm_create(bool es)
+{
+       struct kvm_vm *vm;
+       struct kvm_sev_launch_start start = { 0 };
+       int i;
+
+       vm = vm_create(VM_MODE_DEFAULT, 0, O_RDWR);
+       sev_ioctl(vm->fd, es ? KVM_SEV_ES_INIT : KVM_SEV_INIT, NULL);
+       for (i = 0; i < 3; ++i)

nit: Consider moving 3 to a macro, like MAX_VCPU_IDX or maybe better defining something like NUM_VCPUS to be 4.

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 81

Quoted 11 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +11 -0
+               vm_vcpu_add(vm, i);
+       start.policy |= (es) << 2;
+       sev_ioctl(vm->fd, KVM_SEV_LAUNCH_START, &start);
+       if (es)
+               sev_ioctl(vm->fd, KVM_SEV_LAUNCH_UPDATE_VMSA, NULL);
+       return vm;
+}
+
+static void test_sev_migrate_from(bool es)
+{
+       struct kvm_vm *vms[3];

If we create a NUM_VCPUS macro, then we can use it here.

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 92

Quoted 11 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +11 -0
+       struct kvm_enable_cap cap = { 0 };
+       int i;
+
+       for (i = 0; i < sizeof(vms) / sizeof(struct kvm_vm *); ++i)
+               vms[i] = sev_vm_create(es);
+
+       cap.cap = KVM_CAP_VM_MIGRATE_ENC_CONTEXT_FROM;
+       for (i = 0; i < sizeof(vms) / sizeof(struct kvm_vm *) - 1; ++i) {
+               cap.args[0] = vms[i]->fd;
+               vm_enable_cap(vms[i + 1], &cap);
+       }

nit/optional: To me, the code would be more clear if we combined this loop with the one above and guarded calling vm_enable_cap() with if (i > 0). Also, maybe we can initialize cap when it's declared.

     struct kvm_enable_cap cap = { .cap = KVM_CAP_VM_MIGRATE_ENC_CONTEXT_FROM };
     int i;

     for (i = 0; i < sizeof(vms) / sizeof(struct kvm_vm *); ++i) {
          vms[i] = sev_vm_create(es);
          if (i > 0)
               vm_enable_cap(vms[i], &cap);
     }

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 95

Quoted 3 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +3 -0
+}
+
+#define LOCK_TESTING_THREADS 3

nit: Consider moving this macro to the top of the file.

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 104

Quoted 9 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +9 -0
+
+struct locking_thread_input {
+       struct kvm_vm *vm;
+       int source_fds[LOCK_TESTING_THREADS];
+};
+
+static void *locking_test_thread(void *arg)
+{
+       struct kvm_enable_cap cap = { 0 };

Maybe:
struct kvm_enable_cap cap = { .cap = KVM_CAP_VM_MIGRATE_ENC_CONTEXT_FROM };

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 108

Quoted 4 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +4 -0
+       int i, j;
+       struct locking_thread_input *input = (struct locking_test_thread *)arg;
+
+       cap.cap = KVM_CAP_VM_MIGRATE_ENC_CONTEXT_FROM;

If we initialize the cap field during the declaration, then this line goes away.

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 118

Quoted 10 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +10 -0
+
+       for (i = 0; i < 1000; ++i) {
+               j = input->source_fds[i % LOCK_TESTING_THREADS];
+               cap.args[0] = input->source_fds[j];
+               /*
+                * Call IOCTL directly without checking return code. We are
+                * simply trying to confirm there is no deadlock from userspace
+                * not check correctness of migration here.
+                */
+               ioctl(input->vm->fd, KVM_ENABLE_CAP, &cap);

Should we use vm_enable_cap() here?

Quoted 14 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +14 -0
+       }
+}
+
+static void test_sev_migrate_locking(void)
+{
+       struct locking_thread_input input[LOCK_TESTING_THREADS];
+       pthread_t pt[LOCK_TESTING_THREADS];
+       int i;
+
+       for (i = 0; i < LOCK_TESTING_THREADS; ++i) {
+               input[i].vm = sev_vm_create(/* es= */ false);
+               input[0].source_fds[i] = input[i].vm->fd;
+       }
+       memcpy(input[1].source_fds, input[0].source_fds,

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 142

Quoted 10 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +10 -0
+              sizeof(input[1].source_fds));
+       memcpy(input[2].source_fds, input[0].source_fds,
+              sizeof(input[2].source_fds));
+
+       for (i = 0; i < LOCK_TESTING_THREADS; ++i)
+               pthread_create(&pt[i], NULL, locking_test_thread, &input[i]);
+
+       for (i = 0; i < LOCK_TESTING_THREADS; ++i)
+               pthread_join(pt[i], NULL);
+}

I think this function/test case deserves a comment to capture some of the conversation we had on the list that led to Sean suggesting this test case. Speaking of which, should this test case have a Suggested-by tag for Sean, since he suggested this test?

Quoted 11 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +8 -1
+
+int main(int argc, char *argv[])
+{
+       test_sev_migrate_from(/* es= */ false);
+       test_sev_migrate_from(/* es= */ true);
+       test_sev_migrate_locking();
+       return 0;
+}
--

2.33.0.259.gc128427fd7-goog

Nice test!

#2 lore raw
Quoted 215 lines · Marc Orr
Quoted 86 lines · Peter Gonda

Adds testcases for intra host migration for SEV and SEV-ES. Also adds locking test to confirm no deadlock exists.

---
 tools/testing/selftests/kvm/Makefile          |   1 +
 .../selftests/kvm/x86_64/sev_vm_tests.c       | 152 ++++++++++++++++++
 2 files changed, 153 insertions(+)
 create mode 100644 tools/testing/selftests/kvm/x86_64/sev_vm_tests.c
tools/testing/selftests/kvm/Makefile +1 -0
index 5832f510a16c..de6e64d5c9c4 100644
@@ -71,6 +71,7 @@ TEST_GEN_PROGS_x86_64 += x86_64/tsc_msrs_test
 TEST_GEN_PROGS_x86_64 += x86_64/vmx_pmu_msrs_test
 TEST_GEN_PROGS_x86_64 += x86_64/xen_shinfo_test
 TEST_GEN_PROGS_x86_64 += x86_64/xen_vmcall_test
+TEST_GEN_PROGS_x86_64 += x86_64/sev_vm_tests
 TEST_GEN_PROGS_x86_64 += access_tracking_perf_test
 TEST_GEN_PROGS_x86_64 += demand_paging_test
 TEST_GEN_PROGS_x86_64 += dirty_log_test
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +50 -0
new file mode 100644
index 000000000000..50a770316628
@@ -0,0 +1,150 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <linux/kvm.h>
+#include <linux/psp-sev.h>
+#include <stdio.h>
+#include <sys/ioctl.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <pthread.h>
+
+#include "test_util.h"
+#include "kvm_util.h"
+#include "processor.h"
+#include "svm_util.h"
+#include "kvm_util.h"
+#include "kselftest.h"
+#include "../lib/kvm_util_internal.h"
+
+#define SEV_DEV_PATH "/dev/sev"
+
+/*
+ * Open SEV_DEV_PATH if available, otherwise exit the entire program.
+ *
+ * Input Args:
+ *   flags - The flags to pass when opening SEV_DEV_PATH.
+ *
+ * Return:
+ *   The opened file descriptor of /dev/sev.
+ */
+static int open_sev_dev_path_or_exit(int flags)
+{
+       static int fd;
+
+       if (fd != 0)
+               return fd;
+
+       fd = open(SEV_DEV_PATH, flags);
+       if (fd < 0) {
+               print_skip("%s not available, is SEV not enabled? (errno: %d)",
+                          SEV_DEV_PATH, errno);
+               exit(KSFT_SKIP);
+       }
+
+       return fd;
+}
+
+static void sev_ioctl(int fd, int cmd_id, void *data)
+{
+       struct kvm_sev_cmd cmd = { 0 };
+       int ret;
+

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 60

Quoted 10 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +10 -0
+       TEST_ASSERT(cmd_id < KVM_SEV_NR_MAX, "Unknown SEV CMD : %d\n", cmd_id);
+
+       cmd.id = cmd_id;
+       cmd.sev_fd = open_sev_dev_path_or_exit(0);
+       cmd.data = (uint64_t)data;
+       ret = ioctl(fd, KVM_MEMORY_ENCRYPT_OP, &cmd);
+       TEST_ASSERT((ret == 0 || cmd.error == SEV_RET_SUCCESS),
+                   "%d failed: return code: %d, errno: %d, fw error: %d",
+                   cmd_id, ret, errno, cmd.error);
+}

nit: Since this function has two file descriptors, fd and cmd.sev_fd, can we rename fd to vm_fd?

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 70

Quoted 10 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +10 -0
+
+static struct kvm_vm *sev_vm_create(bool es)
+{
+       struct kvm_vm *vm;
+       struct kvm_sev_launch_start start = { 0 };
+       int i;
+
+       vm = vm_create(VM_MODE_DEFAULT, 0, O_RDWR);
+       sev_ioctl(vm->fd, es ? KVM_SEV_ES_INIT : KVM_SEV_INIT, NULL);
+       for (i = 0; i < 3; ++i)

nit: Consider moving 3 to a macro, like MAX_VCPU_IDX or maybe better defining something like NUM_VCPUS to be 4.

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 81

Quoted 11 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +11 -0
+               vm_vcpu_add(vm, i);
+       start.policy |= (es) << 2;
+       sev_ioctl(vm->fd, KVM_SEV_LAUNCH_START, &start);
+       if (es)
+               sev_ioctl(vm->fd, KVM_SEV_LAUNCH_UPDATE_VMSA, NULL);
+       return vm;
+}
+
+static void test_sev_migrate_from(bool es)
+{
+       struct kvm_vm *vms[3];

If we create a NUM_VCPUS macro, then we can use it here.

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 92

Quoted 11 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +11 -0
+       struct kvm_enable_cap cap = { 0 };
+       int i;
+
+       for (i = 0; i < sizeof(vms) / sizeof(struct kvm_vm *); ++i)
+               vms[i] = sev_vm_create(es);
+
+       cap.cap = KVM_CAP_VM_MIGRATE_ENC_CONTEXT_FROM;
+       for (i = 0; i < sizeof(vms) / sizeof(struct kvm_vm *) - 1; ++i) {
+               cap.args[0] = vms[i]->fd;
+               vm_enable_cap(vms[i + 1], &cap);
+       }

nit/optional: To me, the code would be more clear if we combined this loop with the one above and guarded calling vm_enable_cap() with if (i > 0). Also, maybe we can initialize cap when it's declared.

     struct kvm_enable_cap cap = { .cap = KVM_CAP_VM_MIGRATE_ENC_CONTEXT_FROM };
     int i;

     for (i = 0; i < sizeof(vms) / sizeof(struct kvm_vm *); ++i) {
          vms[i] = sev_vm_create(es);
          if (i > 0)
               vm_enable_cap(vms[i], &cap);
     }

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 95

Quoted 3 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +3 -0
+}
+
+#define LOCK_TESTING_THREADS 3

nit: Consider moving this macro to the top of the file.

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 104

Quoted 9 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +9 -0
+
+struct locking_thread_input {
+       struct kvm_vm *vm;
+       int source_fds[LOCK_TESTING_THREADS];
+};
+
+static void *locking_test_thread(void *arg)
+{
+       struct kvm_enable_cap cap = { 0 };

Maybe:
struct kvm_enable_cap cap = { .cap = KVM_CAP_VM_MIGRATE_ENC_CONTEXT_FROM };

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 108

Quoted 4 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +4 -0
+       int i, j;
+       struct locking_thread_input *input = (struct locking_test_thread *)arg;
+
+       cap.cap = KVM_CAP_VM_MIGRATE_ENC_CONTEXT_FROM;

If we initialize the cap field during the declaration, then this line goes away.

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 118

Quoted 10 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +10 -0
+
+       for (i = 0; i < 1000; ++i) {
+               j = input->source_fds[i % LOCK_TESTING_THREADS];
+               cap.args[0] = input->source_fds[j];
+               /*
+                * Call IOCTL directly without checking return code. We are
+                * simply trying to confirm there is no deadlock from userspace
+                * not check correctness of migration here.
+                */
+               ioctl(input->vm->fd, KVM_ENABLE_CAP, &cap);

Should we use vm_enable_cap() here?

Quoted 21 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +21 -0
+       }
+}
+
+static void test_sev_migrate_locking(void)
+{
+       struct locking_thread_input input[LOCK_TESTING_THREADS];
+       pthread_t pt[LOCK_TESTING_THREADS];
+       int i;
+
+       for (i = 0; i < LOCK_TESTING_THREADS; ++i) {
+               input[i].vm = sev_vm_create(/* es= */ false);
+               input[0].source_fds[i] = input[i].vm->fd;
+       }
+       memcpy(input[1].source_fds, input[0].source_fds,
+              sizeof(input[1].source_fds));
+       memcpy(input[2].source_fds, input[0].source_fds,
+              sizeof(input[2].source_fds));
+
+       for (i = 0; i < LOCK_TESTING_THREADS; ++i)
+               pthread_create(&pt[i], NULL, locking_test_thread, &input[i]);
+
Quoted 8 lines

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 142

Quoted 3 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +3 -0
+       for (i = 0; i < LOCK_TESTING_THREADS; ++i)
+               pthread_join(pt[i], NULL);
+}

I think this function/test case deserves a comment to capture some of the conversation we had on the list that led to Sean suggesting this test case. Speaking of which, should this test case have a Suggested-by tag for Sean, since he suggested this test?

Gah. I forgot to check the tags before sending my feedback. Of course, the suggested-by tag is already there. Second time I made this gaffe in the last couple of weeks. Sorry for the noise.

Quoted 14 lines
Quoted 10 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +8 -1
+
+int main(int argc, char *argv[])
+{
+       test_sev_migrate_from(/* es= */ false);
+       test_sev_migrate_from(/* es= */ true);
+       test_sev_migrate_locking();
+       return 0;
+}
--
Quoted 2 lines

2.33.0.259.gc128427fd7-goog

Nice test!

#3 lore raw
Quoted 194 lines · Marc Orr
Quoted 192 lines · Marc Orr
Quoted 86 lines · Peter Gonda

Adds testcases for intra host migration for SEV and SEV-ES. Also adds locking test to confirm no deadlock exists.

---
 tools/testing/selftests/kvm/Makefile          |   1 +
 .../selftests/kvm/x86_64/sev_vm_tests.c       | 152 ++++++++++++++++++
 2 files changed, 153 insertions(+)
 create mode 100644 tools/testing/selftests/kvm/x86_64/sev_vm_tests.c
tools/testing/selftests/kvm/Makefile +1 -0
index 5832f510a16c..de6e64d5c9c4 100644
@@ -71,6 +71,7 @@ TEST_GEN_PROGS_x86_64 += x86_64/tsc_msrs_test
 TEST_GEN_PROGS_x86_64 += x86_64/vmx_pmu_msrs_test
 TEST_GEN_PROGS_x86_64 += x86_64/xen_shinfo_test
 TEST_GEN_PROGS_x86_64 += x86_64/xen_vmcall_test
+TEST_GEN_PROGS_x86_64 += x86_64/sev_vm_tests
 TEST_GEN_PROGS_x86_64 += access_tracking_perf_test
 TEST_GEN_PROGS_x86_64 += demand_paging_test
 TEST_GEN_PROGS_x86_64 += dirty_log_test
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +50 -0
new file mode 100644
index 000000000000..50a770316628
@@ -0,0 +1,150 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <linux/kvm.h>
+#include <linux/psp-sev.h>
+#include <stdio.h>
+#include <sys/ioctl.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <pthread.h>
+
+#include "test_util.h"
+#include "kvm_util.h"
+#include "processor.h"
+#include "svm_util.h"
+#include "kvm_util.h"
+#include "kselftest.h"
+#include "../lib/kvm_util_internal.h"
+
+#define SEV_DEV_PATH "/dev/sev"
+
+/*
+ * Open SEV_DEV_PATH if available, otherwise exit the entire program.
+ *
+ * Input Args:
+ *   flags - The flags to pass when opening SEV_DEV_PATH.
+ *
+ * Return:
+ *   The opened file descriptor of /dev/sev.
+ */
+static int open_sev_dev_path_or_exit(int flags)
+{
+       static int fd;
+
+       if (fd != 0)
+               return fd;
+
+       fd = open(SEV_DEV_PATH, flags);
+       if (fd < 0) {
+               print_skip("%s not available, is SEV not enabled? (errno: %d)",
+                          SEV_DEV_PATH, errno);
+               exit(KSFT_SKIP);
+       }
+
+       return fd;
+}
+
+static void sev_ioctl(int fd, int cmd_id, void *data)
+{
+       struct kvm_sev_cmd cmd = { 0 };
+       int ret;
+

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 60

Quoted 10 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +10 -0
+       TEST_ASSERT(cmd_id < KVM_SEV_NR_MAX, "Unknown SEV CMD : %d\n", cmd_id);
+
+       cmd.id = cmd_id;
+       cmd.sev_fd = open_sev_dev_path_or_exit(0);
+       cmd.data = (uint64_t)data;
+       ret = ioctl(fd, KVM_MEMORY_ENCRYPT_OP, &cmd);
+       TEST_ASSERT((ret == 0 || cmd.error == SEV_RET_SUCCESS),
+                   "%d failed: return code: %d, errno: %d, fw error: %d",
+                   cmd_id, ret, errno, cmd.error);
+}

nit: Since this function has two file descriptors, fd and cmd.sev_fd, can we rename fd to vm_fd?

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 70

Quoted 10 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +10 -0
+
+static struct kvm_vm *sev_vm_create(bool es)
+{
+       struct kvm_vm *vm;
+       struct kvm_sev_launch_start start = { 0 };
+       int i;
+
+       vm = vm_create(VM_MODE_DEFAULT, 0, O_RDWR);
+       sev_ioctl(vm->fd, es ? KVM_SEV_ES_INIT : KVM_SEV_INIT, NULL);
+       for (i = 0; i < 3; ++i)

nit: Consider moving 3 to a macro, like MAX_VCPU_IDX or maybe better defining something like NUM_VCPUS to be 4.

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 81

Quoted 11 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +11 -0
+               vm_vcpu_add(vm, i);
+       start.policy |= (es) << 2;
+       sev_ioctl(vm->fd, KVM_SEV_LAUNCH_START, &start);
+       if (es)
+               sev_ioctl(vm->fd, KVM_SEV_LAUNCH_UPDATE_VMSA, NULL);
+       return vm;
+}
+
+static void test_sev_migrate_from(bool es)
+{
+       struct kvm_vm *vms[3];

If we create a NUM_VCPUS macro, then we can use it here.

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 92

Quoted 11 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +11 -0
+       struct kvm_enable_cap cap = { 0 };
+       int i;
+
+       for (i = 0; i < sizeof(vms) / sizeof(struct kvm_vm *); ++i)
+               vms[i] = sev_vm_create(es);
+
+       cap.cap = KVM_CAP_VM_MIGRATE_ENC_CONTEXT_FROM;
+       for (i = 0; i < sizeof(vms) / sizeof(struct kvm_vm *) - 1; ++i) {
+               cap.args[0] = vms[i]->fd;
+               vm_enable_cap(vms[i + 1], &cap);
+       }

nit/optional: To me, the code would be more clear if we combined this loop with the one above and guarded calling vm_enable_cap() with if (i > 0). Also, maybe we can initialize cap when it's declared.

     struct kvm_enable_cap cap = { .cap = KVM_CAP_VM_MIGRATE_ENC_CONTEXT_FROM };
     int i;

     for (i = 0; i < sizeof(vms) / sizeof(struct kvm_vm *); ++i) {
          vms[i] = sev_vm_create(es);
          if (i > 0)
               vm_enable_cap(vms[i], &cap);
     }

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 95

Quoted 3 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +3 -0
+}
+
+#define LOCK_TESTING_THREADS 3

nit: Consider moving this macro to the top of the file.

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 104

Quoted 9 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +9 -0
+
+struct locking_thread_input {
+       struct kvm_vm *vm;
+       int source_fds[LOCK_TESTING_THREADS];
+};
+
+static void *locking_test_thread(void *arg)
+{
+       struct kvm_enable_cap cap = { 0 };

Maybe:
struct kvm_enable_cap cap = { .cap = KVM_CAP_VM_MIGRATE_ENC_CONTEXT_FROM };

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 108

Quoted 4 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +4 -0
+       int i, j;
+       struct locking_thread_input *input = (struct locking_test_thread *)arg;
+
+       cap.cap = KVM_CAP_VM_MIGRATE_ENC_CONTEXT_FROM;

If we initialize the cap field during the declaration, then this line goes away.

Quoted 10 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +10 -0
+
+       for (i = 0; i < 1000; ++i) {
+               j = input->source_fds[i % LOCK_TESTING_THREADS];
+               cap.args[0] = input->source_fds[j];
+               /*
+                * Call IOCTL directly without checking return code. We are
+                * simply trying to confirm there is no deadlock from userspace
+                * not check correctness of migration here.
+                */
+               ioctl(input->vm->fd, KVM_ENABLE_CAP, &cap);
Quoted 2 lines
Quoted 2 lines

Should we use vm_enable_cap() here?

I took all other suggestions but this one I just updated the comment. vm_enable_cap TEST_ASSERT()s if the ioctl fails. Since we are just randomly iterating through a bunch of ioclts we fail often so this would kill the test.

Quoted 48 lines
Quoted 21 lines
Quoted 21 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +21 -0
+       }
+}
+
+static void test_sev_migrate_locking(void)
+{
+       struct locking_thread_input input[LOCK_TESTING_THREADS];
+       pthread_t pt[LOCK_TESTING_THREADS];
+       int i;
+
+       for (i = 0; i < LOCK_TESTING_THREADS; ++i) {
+               input[i].vm = sev_vm_create(/* es= */ false);
+               input[0].source_fds[i] = input[i].vm->fd;
+       }
+       memcpy(input[1].source_fds, input[0].source_fds,
+              sizeof(input[1].source_fds));
+       memcpy(input[2].source_fds, input[0].source_fds,
+              sizeof(input[2].source_fds));
+
+       for (i = 0; i < LOCK_TESTING_THREADS; ++i)
+               pthread_create(&pt[i], NULL, locking_test_thread, &input[i]);
+
Quoted 8 lines

tools/testing/selftests/kvm/x86_64/sev_vm_tests.c line 142

Quoted 3 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +3 -0
+       for (i = 0; i < LOCK_TESTING_THREADS; ++i)
+               pthread_join(pt[i], NULL);
+}

I think this function/test case deserves a comment to capture some of the conversation we had on the list that led to Sean suggesting this test case. Speaking of which, should this test case have a Suggested-by tag for Sean, since he suggested this test?

Gah. I forgot to check the tags before sending my feedback. Of course, the suggested-by tag is already there. Second time I made this gaffe in the last couple of weeks. Sorry for the noise.

Quoted 14 lines
Quoted 10 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +8 -1
+
+int main(int argc, char *argv[])
+{
+       test_sev_migrate_from(/* es= */ false);
+       test_sev_migrate_from(/* es= */ true);
+       test_sev_migrate_locking();
+       return 0;
+}
--
Quoted 2 lines

2.33.0.259.gc128427fd7-goog

Nice test!

#4 lore raw
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 10 lines
tools/testing/selftests/kvm/x86_64/sev_vm_tests.c new file +10 -0
+
+       for (i = 0; i < 1000; ++i) {
+               j = input->source_fds[i % LOCK_TESTING_THREADS];
+               cap.args[0] = input->source_fds[j];
+               /*
+                * Call IOCTL directly without checking return code. We are
+                * simply trying to confirm there is no deadlock from userspace
+                * not check correctness of migration here.
+                */
+               ioctl(input->vm->fd, KVM_ENABLE_CAP, &cap);
Quoted 7 lines
Quoted 2 lines
Quoted 2 lines

Should we use vm_enable_cap() here?

I took all other suggestions but this one I just updated the comment. vm_enable_cap TEST_ASSERT()s if the ioctl fails. Since we are just randomly iterating through a bunch of ioclts we fail often so this would kill the test.

Gah, of course! My bad. Anyway, sounds good and thanks!