hyperv: VMBus implementation
v2 cover hyperv: VMBus implementation
This is a rebase of the old patchset from Roman for HyperV VMBus implementation.
How to use:
-device vmbus-bridge
Later on new paravirtualized devices can be implemented on top of it (Network/SCSI/etc.)
VMBus is a collection of technologies. At its lowest layer, it's a message passing and signaling mechanism, allowing efficient passing of messages to and from guest VMs. A layer higher, it's a mechanism for defining channels of communication, where each channel is tagged with a type (which implies a protocol) and a instance ID. A layer higher than that, it's a bus driver, serving as the basis of device enumeration within a VM, where a channel can optionally be exposed as a paravirtual device. When a server-side (paravirtual back-end) component wishes to offer a channel to a guest VM, it does so by specifying a channel type, a mode, and an instance ID. VMBus then exposes this in the guest.
More information about VMBus can be found in the file vmbuskernelmodeclientlibapi.h in Microsoft's WDK.
Jon Doron (7):
hyperv: expose API to determine if synic is enabled
hyperv: SControl is optional to enable SynIc
vmbus: add vmbus protocol definitions
vmbus: vmbus implementation
i386:pc: whitelist dynamic vmbus-bridge
i386: Hyper-V VMBus ACPI DSDT entry
vmbus: add infrastructure to save/load vmbus requests
Makefile.objs | 1 + hw/hyperv/Kconfig | 5 + hw/hyperv/Makefile.objs | 1 + hw/hyperv/hyperv.c | 250 ++- hw/hyperv/trace-events | 18 + hw/hyperv/vmbus.c | 2778 ++++++++++++++++++++++++++++++ hw/i386/acpi-build.c | 43 + hw/i386/pc_piix.c | 2 + hw/i386/pc_q35.c | 2 + include/hw/hyperv/hyperv.h | 3 + include/hw/hyperv/vmbus-bridge.h | 35 + include/hw/hyperv/vmbus-proto.h | 222 +++ include/hw/hyperv/vmbus.h | 230 +++ target/i386/hyperv.c | 2 + 14 files changed, 3510 insertions(+), 82 deletions(-) create mode 100644 hw/hyperv/trace-events create mode 100644 hw/hyperv/vmbus.c create mode 100644 include/hw/hyperv/vmbus-bridge.h create mode 100644 include/hw/hyperv/vmbus-proto.h create mode 100644 include/hw/hyperv/vmbus.h
Signature · 1 line
2.24.1
PATCH v2 1/7 hyperv: expose API to determine if synic is enabled
--- hw/hyperv/hyperv.c | 8 ++++++++ include/hw/hyperv/hyperv.h | 1 + 2 files changed, 9 insertions(+)
hw/hyperv/hyperv.c +8 -0
@@ -37,6 +37,13 @@ typedef struct SynICState {#define TYPE_SYNIC "hyperv-synic"#define SYNIC (obj) OBJECT_CHECK(SynICState, (obj), TYPE_SYNIC) +static bool synic_enabled; + +bool hyperv_is_synic_enabled (void ) +{ +return synic_enabled; +} +static SynICState *get_synic (CPUState *cs) {return SYNIC (object_resolve_path_component (OBJECT (cs),"synic" ));
@@ -133,6 +140,7 @@ void hyperv_synic_add(CPUState *cs)object_property_add_child (OBJECT (cs),"synic" , obj, &error_abort);object_unref (obj);object_property_set_bool (obj, true,"realized" , &error_abort); + synic_enabled = true; }void hyperv_synic_reset (CPUState *cs)
include/hw/hyperv/hyperv.h +1 -0
@@ -79,5 +79,6 @@ void hyperv_synic_add(CPUState *cs);void hyperv_synic_reset (CPUState *cs);void hyperv_synic_update (CPUState *cs,bool enable,hwaddr msg_page_addr,hwaddr event_page_addr); +bool hyperv_is_synic_enabled (void );#endif
Signature · 1 line
2.24.1
PATCH v2 2/7 hyperv: SControl is optional to enable SynIc
In reply to Jon DoronSynIc can be enabled regardless of the SControl mechanisim which can register a GSI for a given SintRoute.
This behaviour can achived by setting enabling SIMP and then the guest will poll on the message slot.
Once there is another message pending the host will set the message slot with the pending flag.
When the guest polls from the message slot, incase the pending flag is set it will write to the HV_X64_MSR_EOM indicating it has cleared the slow and we can try and push our message again.
--- hw/hyperv/hyperv.c | 242 ++++++++++++++++++++++++------------- include/hw/hyperv/hyperv.h | 2 + target/i386/hyperv.c | 2 + 3 files changed, 164 insertions(+), 82 deletions(-)
hw/hyperv/hyperv.c +160 -82
@@ -20,18 +20,72 @@#include "qemu/rcu_queue.h" #include "hw/hyperv/hyperv.h" +/* +* KVM has its own message producers (SynIC timers). To guarantee +* serialization with both KVM vcpu and the guest cpu, the messages are first +* staged in an intermediate area and then posted to the SynIC message page in +* the vcpu thread. +*/ +typedef struct HvSintStagedMessage { +/* message content staged by hyperv_post_msg */ +struct hyperv_message msg; +/* callback + data (r/o) to complete the processing in a BH */ +HvSintMsgCb cb; +void *cb_data; +/* message posting status filled by cpu_post_msg */ +int status; +/* passing the buck: */ +enum { +/* initial state */ +HV_STAGED_MSG_FREE , +/* +* hyperv_post_msg (e.g. in main loop) grabs the staged area (FREE -> +* BUSY), copies msg, and schedules cpu_post_msg on the assigned cpu +*/ +HV_STAGED_MSG_BUSY , +/* +* cpu_post_msg (vcpu thread) tries to copy staged msg to msg slot, +* notify the guest, records the status, marks the posting done (BUSY +* -> POSTED), and schedules sint_msg_bh BH +*/ +HV_STAGED_MSG_POSTED , +/* +* sint_msg_bh (BH) verifies that the posting is done, runs the +* callback, and starts over (POSTED -> FREE) +*/ + } state; +}HvSintStagedMessage ; + +struct SynICState ; + +struct HvSintRoute { +uint32_t sint; +struct SynICState *synic; +int gsi; +EventNotifier sint_set_notifier; +EventNotifier sint_ack_notifier; + +HvSintStagedMessage *staged_msg; + +unsigned refcount; +QLIST_ENTRY (HvSintRoute) link; +}; +typedef struct SynICState {DeviceState parent_obj;CPUState *cs; -bool enabled; +bool sctl_enabled;hwaddr msg_page_addr;hwaddr event_page_addr;MemoryRegion msg_page_mr;MemoryRegion event_page_mr;struct hyperv_message_page *msg_page;struct hyperv_event_flags_page *event_page; + +QemuMutex sint_routes_mutex; + QLIST_HEAD(, HvSintRoute) sint_routes; }SynICState ;#define TYPE_SYNIC "hyperv-synic"
@@ -49,11 +103,11 @@ static SynICState *get_synic(CPUState *cs)return SYNIC (object_resolve_path_component (OBJECT (cs),"synic" )); } -static void synic_update (SynICState *synic,bool enable, +static void synic_update (SynICState *synic,bool sctl_enable,hwaddr msg_page_addr,hwaddr event_page_addr) { - synic->enabled = enable; + synic->sctl_enabled = sctl_enable;if (synic->msg_page_addr != msg_page_addr) {if (synic->msg_page_addr) { memory_region_del_subregion(get_system_memory(),
@@ -78,7 +132,7 @@ static void synic_update(SynICState *synic, bool enable, } } -void hyperv_synic_update (CPUState *cs,bool enable, +void hyperv_synic_update (CPUState *cs,bool sctl_enable,hwaddr msg_page_addr,hwaddr event_page_addr) {SynICState *synic =get_synic (cs);
@@ -87,7 +141,7 @@ void hyperv_synic_update(CPUState *cs, bool enable,return ; } -synic_update (synic, enable, msg_page_addr, event_page_addr); +synic_update (synic, sctl_enable, msg_page_addr, event_page_addr); }static void synic_realize (DeviceState *dev,Error **errp)
@@ -108,16 +162,20 @@ static void synic_realize(DeviceState *dev, Error **errp) sizeof(*synic->event_page), &error_abort); synic->msg_page =memory_region_get_ram_ptr (&synic->msg_page_mr); synic->event_page =memory_region_get_ram_ptr (&synic->event_page_mr); +qemu_mutex_init (&synic->sint_routes_mutex); +QLIST_INIT (&synic->sint_routes);g_free (msgp_name);g_free (eventp_name); } +static void synic_reset (DeviceState *dev) {SynICState *synic =SYNIC (dev);memset (synic->msg_page,0 ,sizeof (*synic->msg_page));memset (synic->event_page,0 ,sizeof (*synic->event_page));synic_update (synic, false,0 ,0 ); +assert (QLIST_EMPTY (&synic->sint_routes)); }static void synic_class_init (ObjectClass *klass,void *data)
@@ -166,54 +224,6 @@ static void synic_register_types(void)type_init (synic_register_types) -/* -* KVM has its own message producers (SynIC timers). To guarantee -* serialization with both KVM vcpu and the guest cpu, the messages are first -* staged in an intermediate area and then posted to the SynIC message page in -* the vcpu thread. -*/ -typedef struct HvSintStagedMessage { -/* message content staged by hyperv_post_msg */ -struct hyperv_message msg; -/* callback + data (r/o) to complete the processing in a BH */ -HvSintMsgCb cb; -void *cb_data; -/* message posting status filled by cpu_post_msg */ -int status; -/* passing the buck: */ -enum { -/* initial state */ -HV_STAGED_MSG_FREE , -/* -* hyperv_post_msg (e.g. in main loop) grabs the staged area (FREE -> -* BUSY), copies msg, and schedules cpu_post_msg on the assigned cpu -*/ -HV_STAGED_MSG_BUSY , -/* -* cpu_post_msg (vcpu thread) tries to copy staged msg to msg slot, -* notify the guest, records the status, marks the posting done (BUSY -* -> POSTED), and schedules sint_msg_bh BH -*/ -HV_STAGED_MSG_POSTED , -/* -* sint_msg_bh (BH) verifies that the posting is done, runs the -* callback, and starts over (POSTED -> FREE) -*/ - } state; -} HvSintStagedMessage; - -struct HvSintRoute { -uint32_t sint; -SynICState *synic; -int gsi; -EventNotifier sint_set_notifier; -EventNotifier sint_ack_notifier; - -HvSintStagedMessage *staged_msg; - -unsigned refcount; -}; - static CPUState *hyperv_find_vcpu(uint32_t vp_index) {CPUState *cs =qemu_get_cpu (vp_index);
@@ -257,7 +267,7 @@ static void cpu_post_msg(CPUState *cs, run_on_cpu_data data)assert (staged_msg->state ==HV_STAGED_MSG_BUSY ); -if (!synic->enabled || !synic->msg_page_addr) { +if (!synic->msg_page_addr) { staged_msg->status = -ENXIO ; goto posted; }
@@ -341,7 +351,7 @@ int hyperv_set_event_flag(HvSintRoute *sint_route, unsigned eventno)if (eventno >HV_EVENT_FLAGS_COUNT ) {return -EINVAL ; } -if (!synic->enabled || !synic->event_page_addr) { +if (!synic->sctl_enabled || !synic->event_page_addr) {return -ENXIO ; }
@@ -362,11 +372,13 @@ int hyperv_set_event_flag(HvSintRoute *sint_route, unsigned eventno)HvSintRoute *hyperv_sint_route_new (uint32_t vp_index,uint32_t sint,HvSintMsgCb cb,void *cb_data) { -HvSintRoute *sint_route; -EventNotifier *ack_notifier; +HvSintRoute *sint_route =NULL ; +EventNotifier *ack_notifier =NULL ;int r, gsi;CPUState *cs;SynICState *synic; +bool ack_event_initialized = false, sint_notifier_initialized = false, + irqfd_initialized = false; cs =hyperv_find_vcpu (vp_index);if (!cs) {
@@ -379,57 +391,82 @@ HvSintRoute *hyperv_sint_route_new(uint32_t vp_index, uint32_t sint, } sint_route =g_new0 (HvSintRoute,1 ); - r =event_notifier_init (&sint_route->sint_set_notifier, false); -if (r) { - goto err; +if (!sint_route) { + goto cleanup_err; } + sint_route->gsi =0 ; + sint_route->synic = synic; + sint_route->sint = sint; + sint_route->refcount =1 ; ack_notifier = cb ? &sint_route->sint_ack_notifier :NULL ;if (ack_notifier) { sint_route->staged_msg =g_new0 (HvSintStagedMessage,1 ); +if (!sint_route->staged_msg) { + goto cleanup_err; + } sint_route->staged_msg->cb = cb; sint_route->staged_msg->cb_data = cb_data; r =event_notifier_init (ack_notifier, false);if (r) { - goto err_sint_set_notifier; + goto cleanup_err; } -event_notifier_set_handler (ack_notifier, sint_ack_handler); + ack_event_initialized = true; + } + +/* See if we are done or we need to setup a GSI for this SintRoute */ +if (!synic->sctl_enabled) { + goto cleanup; + } + +/* We need to setup a GSI for this SintRoute */ + r =event_notifier_init (&sint_route->sint_set_notifier, false); +if (r) { + goto cleanup_err; } + sint_notifier_initialized = true; gsi =kvm_irqchip_add_hv_sint_route (kvm_state, vp_index, sint);if (gsi <0 ) { - goto err_gsi; + goto cleanup_err; } + irqfd_initialized = true; r =kvm_irqchip_add_irqfd_notifier_gsi (kvm_state, &sint_route->sint_set_notifier, ack_notifier, gsi);if (r) { - goto err_irqfd; + goto cleanup_err; } sint_route->gsi = gsi; - sint_route->synic = synic; - sint_route->sint = sint; - sint_route->refcount =1 ; - +cleanup: +qemu_mutex_lock (&synic->sint_routes_mutex); +QLIST_INSERT_HEAD_RCU (&synic->sint_routes, sint_route, link); +qemu_mutex_unlock (&synic->sint_routes_mutex);return sint_route; -err_irqfd: -kvm_irqchip_release_virq (kvm_state, gsi); -err_gsi: +cleanup_err: +if (irqfd_initialized) { +kvm_irqchip_release_virq (kvm_state, gsi); + } + +if (sint_notifier_initialized) { +event_notifier_cleanup (&sint_route->sint_set_notifier); + } +if (ack_notifier) { -event_notifier_set_handler (ack_notifier,NULL ); -event_notifier_cleanup (ack_notifier); +if (ack_event_initialized) { +event_notifier_set_handler (ack_notifier,NULL ); +event_notifier_cleanup (ack_notifier); + } +g_free (sint_route->staged_msg); } -err_sint_set_notifier: -event_notifier_cleanup (&sint_route->sint_set_notifier); -err: -g_free (sint_route); +g_free (sint_route);return NULL ; }
@@ -440,6 +477,8 @@ void hyperv_sint_route_ref(HvSintRoute *sint_route)void hyperv_sint_route_unref (HvSintRoute *sint_route) { +SynICState *synic; +if (!sint_route) {return ; }
@@ -450,21 +489,33 @@ void hyperv_sint_route_unref(HvSintRoute *sint_route)return ; } -kvm_irqchip_remove_irqfd_notifier_gsi (kvm_state, - &sint_route->sint_set_notifier, - sint_route->gsi); -kvm_irqchip_release_virq (kvm_state, sint_route->gsi); + synic = sint_route->synic; +qemu_mutex_lock (&synic->sint_routes_mutex); +QLIST_REMOVE_RCU (sint_route, link); +qemu_mutex_unlock (&synic->sint_routes_mutex); + +if (sint_route->gsi) { +kvm_irqchip_remove_irqfd_notifier_gsi (kvm_state, + &sint_route->sint_set_notifier, + sint_route->gsi); +kvm_irqchip_release_virq (kvm_state, sint_route->gsi); +event_notifier_cleanup (&sint_route->sint_set_notifier); + } +if (sint_route->staged_msg) {event_notifier_set_handler (&sint_route->sint_ack_notifier,NULL );event_notifier_cleanup (&sint_route->sint_ack_notifier);g_free (sint_route->staged_msg); } -event_notifier_cleanup (&sint_route->sint_set_notifier);g_free (sint_route); }int hyperv_sint_route_set_sint (HvSintRoute *sint_route) { +if (!sint_route->gsi) { +return 0 ; + } +return event_notifier_set (&sint_route->sint_set_notifier); }
@@ -528,6 +579,33 @@ unlock:return ret; } +int hyperv_synic_eom (CPUState *cs) +{ +SynICState *synic =get_synic (cs); +HvSintRoute *sint_route; + +if (!synic) { +return -1 ; + } + +qemu_mutex_lock (&synic->sint_routes_mutex); + QLIST_FOREACH(sint_route, &synic->sint_routes, link) { +HvSintStagedMessage *staged_msg = sint_route->staged_msg; + +/* Skip a SintRoute that has a GSI registered with it */ +if (sint_route->gsi || +atomic_read (&staged_msg->state) !=HV_STAGED_MSG_POSTED ) { +continue ; + } + +aio_bh_schedule_oneshot (qemu_get_aio_context (), sint_msg_bh, + sint_route); + } +qemu_mutex_unlock (&synic->sint_routes_mutex); + +return 0 ; +} +uint16_t hyperv_hcall_post_message (uint64_t param,bool fast) {uint16_t ret;
include/hw/hyperv/hyperv.h +2 -0
@@ -28,6 +28,8 @@ void hyperv_sint_route_unref(HvSintRoute *sint_route);int hyperv_sint_route_set_sint (HvSintRoute *sint_route); +int hyperv_synic_eom (CPUState *cs); +/* * Submit a message to be posted in vcpu context. If the submission succeeds, * the status of posting the message is reported via the callback associated
target/i386/hyperv.c +2 -0
@@ -66,6 +66,8 @@ int kvm_hv_handle_exit(X86CPU *cpu, struct kvm_hyperv_exit *exit)case HV_X64_MSR_SIEFP : env->msr_hv_synic_evt_page = exit->u.synic.evt_page;break ; +case HV_X64_MSR_EOM : +return hyperv_synic_eom (CPU (cpu));default :return -1 ; }
Signature · 1 line
2.24.1
PATCH v2 3/7 vmbus: add vmbus protocol definitions
In reply to Jon DoronAdd a header with data structures and constants used in Hyper-V VMBus hypervisor <-> guest interactions.
Based on the respective stuff from Linux kernel.
--- include/hw/hyperv/vmbus-proto.h | 222 ++++++++++++++++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 include/hw/hyperv/vmbus-proto.h
include/hw/hyperv/vmbus-proto.h new file +222 -0
@@ -0,0 +1,222 @@ +/* +* QEMU Hyper-V VMBus support +* +* Copyright (c) 2017-2018 Virtuozzo International GmbH. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +*/ + +#ifndef HW_HYPERV_VMBUS_PROTO_H +#define HW_HYPERV_VMBUS_PROTO_H + +#define VMBUS_VERSION_WS2008 ((0 << 16) | (13)) +#define VMBUS_VERSION_WIN7 ((1 << 16) | (1)) +#define VMBUS_VERSION_WIN8 ((2 << 16) | (4)) +#define VMBUS_VERSION_WIN8_1 ((3 << 16) | (0)) +#define VMBUS_VERSION_WIN10 ((4 << 16) | (0)) +#define VMBUS_VERSION_INVAL -1 +#define VMBUS_VERSION_CURRENT VMBUS_VERSION_WIN10 + +#define VMBUS_MESSAGE_CONNECTION_ID 1 +#define VMBUS_EVENT_CONNECTION_ID 2 +#define VMBUS_MONITOR_CONNECTION_ID 3 +#define VMBUS_SINT 2 + +#define VMBUS_MSG_INVALID 0 +#define VMBUS_MSG_OFFERCHANNEL 1 +#define VMBUS_MSG_RESCIND_CHANNELOFFER 2 +#define VMBUS_MSG_REQUESTOFFERS 3 +#define VMBUS_MSG_ALLOFFERS_DELIVERED 4 +#define VMBUS_MSG_OPENCHANNEL 5 +#define VMBUS_MSG_OPENCHANNEL_RESULT 6 +#define VMBUS_MSG_CLOSECHANNEL 7 +#define VMBUS_MSG_GPADL_HEADER 8 +#define VMBUS_MSG_GPADL_BODY 9 +#define VMBUS_MSG_GPADL_CREATED 10 +#define VMBUS_MSG_GPADL_TEARDOWN 11 +#define VMBUS_MSG_GPADL_TORNDOWN 12 +#define VMBUS_MSG_RELID_RELEASED 13 +#define VMBUS_MSG_INITIATE_CONTACT 14 +#define VMBUS_MSG_VERSION_RESPONSE 15 +#define VMBUS_MSG_UNLOAD 16 +#define VMBUS_MSG_UNLOAD_RESPONSE 17 +#define VMBUS_MSG_COUNT 18 + +#define VMBUS_MESSAGE_SIZE_ALIGN sizeof(uint64_t) + +#define VMBUS_PACKET_INVALID 0x0 +#define VMBUS_PACKET_SYNCH 0x1 +#define VMBUS_PACKET_ADD_XFER_PAGESET 0x2 +#define VMBUS_PACKET_RM_XFER_PAGESET 0x3 +#define VMBUS_PACKET_ESTABLISH_GPADL 0x4 +#define VMBUS_PACKET_TEARDOWN_GPADL 0x5 +#define VMBUS_PACKET_DATA_INBAND 0x6 +#define VMBUS_PACKET_DATA_USING_XFER_PAGES 0x7 +#define VMBUS_PACKET_DATA_USING_GPADL 0x8 +#define VMBUS_PACKET_DATA_USING_GPA_DIRECT 0x9 +#define VMBUS_PACKET_CANCEL_REQUEST 0xa +#define VMBUS_PACKET_COMP 0xb +#define VMBUS_PACKET_DATA_USING_ADDITIONAL_PKT 0xc +#define VMBUS_PACKET_ADDITIONAL_DATA 0xd + +#define VMBUS_CHANNEL_USER_DATA_SIZE 120 + +#define VMBUS_OFFER_MONITOR_ALLOCATED 0x1 +#define VMBUS_OFFER_INTERRUPT_DEDICATED 0x1 + +#define VMBUS_RING_BUFFER_FEAT_PENDING_SZ (1ul << 0) + +#define VMBUS_CHANNEL_ENUMERATE_DEVICE_INTERFACE 0x1 +#define VMBUS_CHANNEL_SERVER_SUPPORTS_TRANSFER_PAGES 0x2 +#define VMBUS_CHANNEL_SERVER_SUPPORTS_GPADLS 0x4 +#define VMBUS_CHANNEL_NAMED_PIPE_MODE 0x10 +#define VMBUS_CHANNEL_LOOPBACK_OFFER 0x100 +#define VMBUS_CHANNEL_PARENT_OFFER 0x200 +#define VMBUS_CHANNEL_REQUEST_MONITORED_NOTIFICATION 0x400 +#define VMBUS_CHANNEL_TLNPI_PROVIDER_OFFER 0x2000 + +#define VMBUS_PACKET_FLAG_REQUEST_COMPLETION 1 + +typedef struct vmbus_message_header { +uint32_t message_type; +uint32_t _padding; +}vmbus_message_header ; + +typedef struct vmbus_message_initiate_contact { +vmbus_message_header header; +uint32_t version_requested; +uint32_t target_vcpu; +uint64_t interrupt_page; +uint64_t monitor_page1; +uint64_t monitor_page2; +}vmbus_message_initiate_contact ; + +typedef struct vmbus_message_version_response { +vmbus_message_header header; +uint8_t version_supported; +uint8_t status; +}vmbus_message_version_response ; + +typedef struct vmbus_message_offer_channel { +vmbus_message_header header; +uint8_t type_uuid[16 ]; +uint8_t instance_uuid[16 ]; +uint64_t _reserved1; +uint64_t _reserved2; +uint16_t channel_flags; +uint16_t mmio_size_mb; +uint8_t user_data[VMBUS_CHANNEL_USER_DATA_SIZE ]; +uint16_t sub_channel_index; +uint16_t _reserved3; +uint32_t child_relid; +uint8_t monitor_id; +uint8_t monitor_flags; +uint16_t interrupt_flags; +uint32_t connection_id; +}vmbus_message_offer_channel ; + +typedef struct vmbus_message_rescind_channel_offer { +vmbus_message_header header; +uint32_t child_relid; +}vmbus_message_rescind_channel_offer ; + +typedef struct vmbus_gpa_range { +uint32_t byte_count; +uint32_t byte_offset; +uint64_t pfn_array[]; +}vmbus_gpa_range ; + +typedef struct vmbus_message_gpadl_header { +vmbus_message_header header; +uint32_t child_relid; +uint32_t gpadl_id; +uint16_t range_buflen; +uint16_t rangecount; +vmbus_gpa_range range[]; +} QEMU_PACKED vmbus_message_gpadl_header; + +typedef struct vmbus_message_gpadl_body { +vmbus_message_header header; +uint32_t message_number; +uint32_t gpadl_id; +uint64_t pfn_array[]; +}vmbus_message_gpadl_body ; + +typedef struct vmbus_message_gpadl_created { +vmbus_message_header header; +uint32_t child_relid; +uint32_t gpadl_id; +uint32_t status; +}vmbus_message_gpadl_created ; + +typedef struct vmbus_message_gpadl_teardown { +vmbus_message_header header; +uint32_t child_relid; +uint32_t gpadl_id; +}vmbus_message_gpadl_teardown ; + +typedef struct vmbus_message_gpadl_torndown { +vmbus_message_header header; +uint32_t gpadl_id; +}vmbus_message_gpadl_torndown ; + +typedef struct vmbus_message_open_channel { +vmbus_message_header header; +uint32_t child_relid; +uint32_t open_id; +uint32_t ring_buffer_gpadl_id; +uint32_t target_vp; +uint32_t ring_buffer_offset; +uint8_t user_data[VMBUS_CHANNEL_USER_DATA_SIZE ]; +}vmbus_message_open_channel ; + +typedef struct vmbus_message_open_result { +vmbus_message_header header; +uint32_t child_relid; +uint32_t open_id; +uint32_t status; +}vmbus_message_open_result ; + +typedef struct vmbus_message_close_channel { +vmbus_message_header header; +uint32_t child_relid; +}vmbus_message_close_channel ; + +typedef struct vmbus_ring_buffer { +uint32_t write_index; +uint32_t read_index; +uint32_t interrupt_mask; +uint32_t pending_send_sz; +uint32_t _reserved1[12 ]; +uint32_t feature_bits; +}vmbus_ring_buffer ; + +typedef struct vmbus_packet_hdr { +uint16_t type; +uint16_t offset_qwords; +uint16_t len_qwords; +uint16_t flags; +uint64_t transaction_id; +}vmbus_packet_hdr ; + +typedef struct vmbus_pkt_gpa_direct { +uint32_t _reserved; +uint32_t rangecount; +vmbus_gpa_range range[]; +}vmbus_pkt_gpa_direct ; + +typedef struct vmbus_xferpg_range { +uint32_t byte_count; +uint32_t byte_offset; +}vmbus_xferpg_range ; + +typedef struct vmbus_pkt_xferpg { +uint16_t buffer_id; +uint8_t sender_owns_set; +uint8_t _reserved; +uint32_t rangecount; +vmbus_xferpg_range range[]; +}vmbus_pkt_xferpg ; + +#endif
Signature · 1 line
2.24.1
PATCH v2 4/7 vmbus: vmbus implementation
In reply to Jon DoronAdd the VMBus infrastructure -- bus, devices, root bridge, vmbus state machine, vmbus channel interactions, etc.
VMBus is a collection of technologies. At its lowest layer, it's a message passing and signaling mechanism, allowing efficient passing of messages to and from guest VMs. A layer higher, it's a mechanism for defining channels of communication, where each channel is tagged with a type (which implies a protocol) and a instance ID. A layer higher than that, it's a bus driver, serving as the basis of device enumeration within a VM, where a channel can optionally be exposed as a paravirtual device. When a server-side (paravirtual back-end) component wishes to offer a channel to a guest VM, it does so by specifying a channel type, a mode, and an instance ID. VMBus then exposes this in the guest.
More information about VMBus can be found in the file vmbuskernelmodeclientlibapi.h in Microsoft's WDK.
TODO:
- split into smaller palatable pieces
- more comments
- check and handle corner cases
Kudos to Evgeny Yakovlev (formerly [email protected]) and Andrey Smetatin (formerly [email protected]) for research and prototyping.
--- Makefile.objs | 1 + hw/hyperv/Kconfig | 5 + hw/hyperv/Makefile.objs | 1 + hw/hyperv/trace-events | 18 + hw/hyperv/vmbus.c | 2672 ++++++++++++++++++++++++++++++ include/hw/hyperv/vmbus-bridge.h | 32 + include/hw/hyperv/vmbus.h | 227 +++ 7 files changed, 2956 insertions(+) create mode 100644 hw/hyperv/trace-events create mode 100644 hw/hyperv/vmbus.c create mode 100644 include/hw/hyperv/vmbus-bridge.h create mode 100644 include/hw/hyperv/vmbus.h
Makefile.objs +1 -0
@@ -151,6 +151,7 @@ trace-events-subdirs += hw/block/dataplane trace-events-subdirs += hw/char trace-events-subdirs += hw/dma trace-events-subdirs += hw/hppa +trace-events-subdirs += hw/hyperv trace-events-subdirs += hw/i2c trace-events-subdirs += hw/i386 trace-events-subdirs += hw/i386/xen
hw/hyperv/Kconfig +5 -0
@@ -6,3 +6,8 @@ config HYPERV_TESTDEV bool default y if TEST_DEVICES depends on HYPERV + +config VMBUS + bool + default y + depends on HYPERV
hw/hyperv/Makefile.objs +1 -0
@@ -1,2 +1,3 @@ obj-y += hyperv.o obj-$(CONFIG_HYPERV_TESTDEV) += hyperv_testdev.o +obj-$(CONFIG_VMBUS) += vmbus.o
hw/hyperv/trace-events new file +18 -0
@@ -0,0 +1,18 @@ +# vmbus +vmbus_recv_message(uint32_t type, uint32_t size) "type %d size %d" +vmbus_signal_event(void) "" +vmbus_channel_notify_guest(uint32_t chan_id) "channel #%d" +vmbus_post_msg(uint32_t type, uint32_t size) "type %d size %d" +vmbus_msg_cb(int status) "message status %d" +vmbus_process_incoming_message(uint32_t message_type) "type %d" +vmbus_initiate_contact(uint16_t major, uint16_t minor, uint32_t vcpu, uint64_t monitor_page1, uint64_t monitor_page2, uint64_t interrupt_page) "version %d.%d target vp %d mon pages 0x%"PRIx64",0x%"PRIx64" int page 0x%"PRIx64 +vmbus_send_offer(uint32_t chan_id, void *dev) "channel #%d dev %p" +vmbus_terminate_offers(void) "" +vmbus_gpadl_header(uint32_t gpadl_id, uint16_t num_gfns) "gpadl #%d gfns %d" +vmbus_gpadl_body(uint32_t gpadl_id) "gpadl #%d" +vmbus_gpadl_created(uint32_t gpadl_id) "gpadl #%d" +vmbus_gpadl_teardown(uint32_t gpadl_id) "gpadl #%d" +vmbus_gpadl_torndown(uint32_t gpadl_id) "gpadl #%d" +vmbus_open_channel(uint32_t chan_id, uint32_t gpadl_id, uint32_t target_vp) "channel #%d gpadl #%d target vp %d" +vmbus_channel_open(uint32_t chan_id, uint32_t status) "channel #%d status %d" +vmbus_close_channel(uint32_t chan_id) "channel #%d"
hw/hyperv/vmbus.c new file +2672 -0
2,672 lines not on this page · raw message
include/hw/hyperv/vmbus-bridge.h new file +32 -0
@@ -0,0 +1,32 @@ +/* +* QEMU Hyper-V VMBus root bridge +* +* Copyright (c) 2017-2018 Virtuozzo International GmbH. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +*/ + +#ifndef HW_HYPERV_VMBUS_BRIDGE_H +#define HW_HYPERV_VMBUS_BRIDGE_H + +#include "hw/sysbus.h" + +#define TYPE_VMBUS_BRIDGE "vmbus-bridge" + +typedef struct VMBus VMBus ; + +typedef struct VMBusBridge { +SysBusDevice parent_obj; + +VMBus *bus; +}VMBusBridge ; + +#define VMBUS_BRIDGE (obj) OBJECT_CHECK(VMBusBridge, (obj), TYPE_VMBUS_BRIDGE) + +static inline VMBusBridge *vmbus_bridge_find (void ) +{ +return VMBUS_BRIDGE (object_resolve_path_type ("" ,TYPE_VMBUS_BRIDGE ,NULL )); +} + +#endif
include/hw/hyperv/vmbus.h new file +227 -0
227 lines not on this page · raw message
Signature · 1 line
2.24.1
PATCH v2 5/7 i386:pc: whitelist dynamic vmbus-bridge
In reply to Jon DoronAs vmbus-bridge is derived from sysbus device, it has to be whitelisted to be allowed to be created with -device.
--- hw/i386/pc_piix.c | 2 ++ hw/i386/pc_q35.c | 2 ++ 2 files changed, 4 insertions(+)
hw/i386/pc_piix.c +2 -0
@@ -60,6 +60,7 @@#include "migration/global_state.h" #include "migration/misc.h" #include "sysemu/numa.h" +#include "hw/hyperv/vmbus-bridge.h" #include "hw/mem/nvdimm.h" #define MAX_IDE_BUS 2
@@ -417,6 +418,7 @@ static void pc_i440fx_machine_options(MachineClass *m) m->default_machine_opts ="firmware=bios-256k.bin" ; m->default_display ="std" ;machine_class_allow_dynamic_sysbus_dev (m,TYPE_RAMFB_DEVICE ); +machine_class_allow_dynamic_sysbus_dev (m,TYPE_VMBUS_BRIDGE ); }static void pc_i440fx_5_0_machine_options (MachineClass *m)
hw/i386/pc_q35.c +2 -0
@@ -53,6 +53,7 @@#include "qapi/error.h" #include "qemu/error-report.h" #include "sysemu/numa.h" +#include "hw/hyperv/vmbus-bridge.h" #include "hw/mem/nvdimm.h" /* ICH9 AHCI has 6 ports */
@@ -346,6 +347,7 @@ static void pc_q35_machine_options(MachineClass *m)machine_class_allow_dynamic_sysbus_dev (m,TYPE_AMD_IOMMU_DEVICE );machine_class_allow_dynamic_sysbus_dev (m,TYPE_INTEL_IOMMU_DEVICE );machine_class_allow_dynamic_sysbus_dev (m,TYPE_RAMFB_DEVICE ); +machine_class_allow_dynamic_sysbus_dev (m,TYPE_VMBUS_BRIDGE ); m->max_cpus =288 ; }
Signature · 1 line
2.24.1
PATCH v2 6/7 i386: Hyper-V VMBus ACPI DSDT entry
In reply to Jon DoronGuest OS uses ACPI to discover VMBus presence. Add a corresponding entry to DSDT in case VMBus has been enabled.
Experimentally Windows guests were found to require this entry to include two IRQ resources. They seem to never be used but they still have to be there.
Make IRQ numbers user-configurable via corresponding properties; use 7 and 13 by default.
--- hw/hyperv/vmbus.c | 7 ++++++ hw/i386/acpi-build.c | 43 ++++++++++++++++++++++++++++++++ include/hw/hyperv/vmbus-bridge.h | 3 +++ 3 files changed, 53 insertions(+)
hw/hyperv/vmbus.c +7 -0
@@ -2641,6 +2641,12 @@ static const VMStateDescription vmstate_vmbus_bridge = { }, }; +static Property vmbus_bridge_props[] = { +DEFINE_PROP_UINT8 ("irq0" , VMBusBridge, irq0,7 ), +DEFINE_PROP_UINT8 ("irq1" , VMBusBridge, irq1,13 ), +DEFINE_PROP_END_OF_LIST () +}; +static void vmbus_bridge_class_init (ObjectClass *klass,void *data) {DeviceClass *k =DEVICE_CLASS (klass);
@@ -2651,6 +2657,7 @@ static void vmbus_bridge_class_init(ObjectClass *klass, void *data) sk->explicit_ofw_unit_address = vmbus_bridge_ofw_unit_address;set_bit (DEVICE_CATEGORY_BRIDGE , k->categories); k->vmsd = &vmstate_vmbus_bridge; +device_class_set_props (k, vmbus_bridge_props);/* override SysBusDevice's default */ k->user_creatable = true; }
hw/i386/acpi-build.c +43 -0
64 lines not on this page · raw message
include/hw/hyperv/vmbus-bridge.h +3 -0
@@ -19,6 +19,9 @@ typedef struct VMBus VMBus;typedef struct VMBusBridge {SysBusDevice parent_obj; +uint8_t irq0; +uint8_t irq1; +VMBus *bus; }VMBusBridge ;
Signature · 1 line
2.24.1
PATCH v2 7/7 vmbus: add infrastructure to save/load vmbus requests
In reply to Jon DoronThis can be allow to include controller-specific data while saving/loading in-flight scsi requests of the vmbus scsi controller.
--- hw/hyperv/vmbus.c | 99 +++++++++++++++++++++++++++++++++++++++ include/hw/hyperv/vmbus.h | 3 ++ 2 files changed, 102 insertions(+)
hw/hyperv/vmbus.c +99 -0
@@ -1272,6 +1272,105 @@ void vmbus_free_req(void *req)g_free (req); } +static const VMStateDescription vmstate_sgent = { + .name ="vmbus/sgentry" , + .version_id =0 , + .minimum_version_id =0 , + .fields = (VMStateField []) { +VMSTATE_UINT64 (base, ScatterGatherEntry), +VMSTATE_UINT64 (len, ScatterGatherEntry), +VMSTATE_END_OF_LIST () + } +}; + +typedef struct VMBusChanReqSave { +uint16_t chan_idx; +uint16_t pkt_type; +uint32_t msglen; +void *msg; +uint64_t transaction_id; +bool need_comp; +uint32_t num; +ScatterGatherEntry *sgl; +}VMBusChanReqSave ; + +static const VMStateDescription vmstate_vmbus_chan_req = { + .name ="vmbus/vmbus_chan_req" , + .version_id =0 , + .minimum_version_id =0 , + .fields = (VMStateField []) { +VMSTATE_UINT16 (chan_idx, VMBusChanReqSave), +VMSTATE_UINT16 (pkt_type, VMBusChanReqSave), +VMSTATE_UINT32 (msglen, VMBusChanReqSave), +VMSTATE_VBUFFER_ALLOC_UINT32 (msg, VMBusChanReqSave,0 ,NULL , msglen), +VMSTATE_UINT64 (transaction_id, VMBusChanReqSave), +VMSTATE_BOOL (need_comp, VMBusChanReqSave), +VMSTATE_UINT32 (num, VMBusChanReqSave), +VMSTATE_STRUCT_VARRAY_POINTER_UINT32 (sgl, VMBusChanReqSave, num, + vmstate_sgent, ScatterGatherEntry), +VMSTATE_END_OF_LIST () + } +}; + +void vmbus_save_req (QEMUFile *f,VMBusChanReq *req) +{ +VMBusChanReqSave req_save; + + req_save.chan_idx = req->chan->subchan_idx; + req_save.pkt_type = req->pkt_type; + req_save.msglen = req->msglen; + req_save.msg = req->msg; + req_save.transaction_id = req->transaction_id; + req_save.need_comp = req->need_comp; + req_save.num = req->sgl.nsg; + req_save.sgl =g_memdup (req->sgl.sg, + req_save.num *sizeof (ScatterGatherEntry)); + +vmstate_save_state (f, &vmstate_vmbus_chan_req, &req_save,NULL ); + +g_free (req_save.sgl); +} + +void *vmbus_load_req (QEMUFile *f,VMBusDevice *dev,uint32_t size) +{ +VMBusChanReqSave req_save; +VMBusChanReq *req =NULL ; +VMBusChannel *chan =NULL ; +uint32_t i; + +vmstate_load_state (f, &vmstate_vmbus_chan_req, &req_save,0 ); + +if (req_save.chan_idx >= dev->num_channels) { +error_report ("%s: %u(chan_idx) > %u(num_channels)" , __func__, + req_save.chan_idx, dev->num_channels); + goto out; + } + chan = &dev->channels[req_save.chan_idx]; + +if (vmbus_channel_reserve (chan,0 , req_save.msglen)) { + goto out; + } + + req =vmbus_alloc_req (chan, size, req_save.pkt_type, req_save.msglen, + req_save.transaction_id, req_save.need_comp); +if (req_save.msglen) { +memcpy (req->msg, req_save.msg, req_save.msglen); + } + +for (i =0 ; i < req_save.num; i++) { +qemu_sglist_add (&req->sgl, req_save.sgl[i].base, req_save.sgl[i].len); + } + +out: +if (req_save.msglen) { +g_free (req_save.msg); + } +if (req_save.num) { +g_free (req_save.sgl); + } +return req; +} +static void channel_event_cb (EventNotifier *e) {VMBusChannel *chan =container_of (e, VMBusChannel, notifier);
include/hw/hyperv/vmbus.h +3 -0
@@ -224,4 +224,7 @@ int vmbus_map_sgl(VMBusChanReq *req, DMADirection dir, struct iovec *iov,void vmbus_unmap_sgl (VMBusChanReq *req,DMADirection dir,struct iovec *iov,unsigned iov_cnt,size_t accessed); +void vmbus_save_req (QEMUFile *f,VMBusChanReq *req); +void *vmbus_load_req (QEMUFile *f,VMBusDevice *dev,uint32_t size); +#endif
Signature · 1 line
2.24.1
Patchew URL: https://patchew.org/QEMU/[email protected]/
Patchew URL: https://patchew.org/QEMU/[email protected]/
Hi,
This series failed the asan build test. Please find the testing commands and their output below. If you have Docker installed, you can probably reproduce it locally.
=== TEST SCRIPT BEGIN ===
#!/bin/bash
export ARCH=x86_64
make docker-image-fedora V=1 NETWORK=1
time make docker-test-debug@fedora TARGET_LIST=x86_64-softmmu J=14 NETWORK=1 === TEST SCRIPT END ===
CC x86_64-softmmu/hw/i386/../xenpv/xen_machine_pv.o
CC x86_64-softmmu/hw/i386/kvm/clock.o
CC x86_64-softmmu/hw/i386/kvm/apic.o
/tmp/qemu-test/src/hw/hyperv/vmbus.c:1758:57: error: use of logical '&&' with constant operand [-Werror,-Wconstant-logical-operand]
msg->version_requested && 0xffff,
^ ~~~~~~
/tmp/qemu-test/src/hw/hyperv/vmbus.c:1758:57: note: use '&' for a bitwise operation
---
~^~~~~~~~~1 error generated.
CC x86_64-softmmu/hw/i386/kvm/ioapic.o
make[1]: *** [/tmp/qemu-test/src/rules.mak:69: hw/hyperv/vmbus.o] Error 1 make[1]: *** Waiting for unfinished jobs....
make: *** [Makefile:527: x86_64-softmmu/all] Error 2 Traceback (most recent call last):
File "./tests/docker/docker.py", line 664, in <module>
sys.exit(main())
---
raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['sudo', '-n', 'docker', 'run', '--label', 'com.qemu.instance.uuid=08a51f9a8fa54e4f997e80e93986c8e8', '-u', '1003', '--security-opt', 'seccomp=unconfined', '--rm', '-e', 'TARGET_LIST=x86_64-softmmu', '-e', 'EXTRA_CONFIGURE_OPTS=', '-e', 'V=', '-e', 'J=14', '-e', 'DEBUG=', '-e', 'SHOW_ENV=', '-e', 'CCACHE_DIR=/var/tmp/ccache', '-v', '/home/patchew2/.cache/qemu-docker-ccache:/var/tmp/ccache:z', '-v', '/var/tmp/patchew-tester-tmp-hl8e3_pu/src/docker-src.2020-04-04-04.39.31.17034:/var/tmp/qemu:z,ro', 'qemu:fedora', '/var/tmp/qemu/run', 'test-debug']' returned non-zero exit status 2. filter=--filter=label=com.qemu.instance.uuid=08a51f9a8fa54e4f997e80e93986c8e8
make[1]: *** [docker-run] Error 1
make[1]: Leaving directory `/var/tmp/patchew-tester-tmp-hl8e3_pu/src'
make: *** [docker-run-test-debug@fedora] Error 2
real 4m12.960s
user 0m8.131s
The full log is available at
http://patchew.org/logs/[email protected]/testing.asan/?type=message.
---
Email generated automatically by Patchew [https://patchew.org/].
Please send your feedback to [email protected]