Enhance the hv_vmbus driver to support hibernation
Subsystems (4)
Dexuan Cui
maintainerv2 cover Enhance the hv_vmbus driver to support hibernation
Hi,
This is the first patchset to enable hibernation when Linux runs on Hyper-V.
A second patchset to enhance the high-level VSC drivers (hv_netvsc, hv_storvsc, etc.) for hibernation will be posted later. The second patchset depends on this first patchset, so I hope this pathset can be accepted soon.
The changes in this patchset are mostly straightforward new code that only runs when the VM enters hibernation, so IMHO it's pretty safe to have this patchset, because the hibernation feature never worked for Linux VM running on Hyper-V.
The patchset is rebased on the branch hyperv-next of
https://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git
and can also cleanly apply to Linus's tree.
Hi Greg, tglx,
I hope the patchset can go through Sasha's hyperv/linux.git tree, because the changes belong to the hv code and they need to be together to work properly.
Michael Kelley reviewed the v1 of the patchset, so I added his Reviewed-by for patch 1, 2, 3 and 7. Michael, please review the other 3 patches again, and give your Reviewed-by if the updated version is ok to you.
Changes in v2:
Patch 3: Improved the changelog and added a comment.
Patch 4: Remove the "hv_stimer_cleanup" in hv_synic_suspend(), because I
suppose https://lkml.org/lkml/2019/7/27/5 will be accepted. Also improved changelog and the comment.
Patch 5: Fixed the third argument of print_hex_dump_debug(). Also improved
the changelog.
Patch 6: Improved the changelog and the comment. Added a check for the
'vmbus_proto_version' in vmbus_bus_resume().
Thanks,
Dexuan
Dexuan Cui (7):
x86/hyper-v: Suspend/resume the hypercall page for hibernation clocksource/drivers: Suspend/resume Hyper-V clocksource for
hibernation
Drivers: hv: vmbus: Break out synic enable and disable operations
Drivers: hv: vmbus: Suspend/resume the synic for hibernation
Drivers: hv: vmbus: Ignore the offers when resuming from hibernation
Drivers: hv: vmbus: Suspend/resume the vmbus itself for hibernation
Drivers: hv: vmbus: Implement suspend/resume for VSC drivers for
hibernation
arch/x86/hyperv/hv_init.c | 34 +++++++++ drivers/clocksource/hyperv_timer.c | 25 +++++++ drivers/hv/channel_mgmt.c | 29 +++++++- drivers/hv/connection.c | 3 +- drivers/hv/hv.c | 66 ++++++++++-------- drivers/hv/hyperv_vmbus.h | 4 ++ drivers/hv/vmbus_drv.c | 138 +++++++++++++++++++++++++++++++++++++ include/linux/hyperv.h | 3 + 8 files changed, 270 insertions(+), 32 deletions(-)
Signature · 1 line
1.8.3.1
Dexuan Cui
maintainerPATCH v2 1/7 x86/hyper-v: Suspend/resume the hypercall page for hibernation
This is needed for hibernation, e.g. when we resume the old kernel, we need to disable the "current" kernel's hypercall page and then resume the old kernel's.
--- arch/x86/hyperv/hv_init.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+)
arch/x86/hyperv/hv_init.c +34 -0
@@ -20,6 +20,7 @@#include <linux/hyperv.h> #include <linux/slab.h> #include <linux/cpuhotplug.h> +#include <linux/syscore_ops.h> #include <clocksource/hyperv_timer.h> void *hv_hypercall_pg;
@@ -223,6 +224,34 @@ static int __init hv_pci_init(void)return 1 ; } +static int hv_suspend (void ) +{ +union hv_x64_msr_hypercall_contents hypercall_msr; + +/* Reset the hypercall page */ +rdmsrl (HV_X64_MSR_HYPERCALL , hypercall_msr.as_uint64); + hypercall_msr.enable =0 ; +wrmsrl (HV_X64_MSR_HYPERCALL , hypercall_msr.as_uint64); + +return 0 ; +} + +static void hv_resume (void ) +{ +union hv_x64_msr_hypercall_contents hypercall_msr; + +/* Re-enable the hypercall page */ +rdmsrl (HV_X64_MSR_HYPERCALL , hypercall_msr.as_uint64); + hypercall_msr.enable =1 ; + hypercall_msr.guest_physical_address =vmalloc_to_pfn (hv_hypercall_pg); +wrmsrl (HV_X64_MSR_HYPERCALL , hypercall_msr.as_uint64); +} + +static struct syscore_ops hv_syscore_ops = { + .suspend = hv_suspend, + .resume = hv_resume, +}; +/* * This function is to be invoked early in the boot sequence after the * hypervisor has been detected.
@@ -303,6 +332,9 @@ void __init hyperv_init(void)/* Register Hyper-V specific clocksource */ hv_init_clocksource (); + +register_syscore_ops (&hv_syscore_ops); +return ; remove_cpuhp_state:
@@ -322,6 +354,8 @@ void hyperv_cleanup(void) {union hv_x64_msr_hypercall_contents hypercall_msr; +unregister_syscore_ops (&hv_syscore_ops); +/* Reset our OS id */ wrmsrl (HV_X64_MSR_GUEST_OS_ID ,0 );
Signature · 1 line
1.8.3.1
Dexuan Cui
maintainerPATCH v2 2/7 clocksource/drivers: Suspend/resume Hyper-V clocksource for hibernation
In reply to Dexuan CuiThis is needed for hibernation, e.g. when we resume the old kernel, we need to disable the "current" kernel's TSC page and then resume the old kernel's.
--- drivers/clocksource/hyperv_timer.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+)
drivers/clocksource/hyperv_timer.c +25 -0
@@ -237,12 +237,37 @@ static u64 read_hv_clock_tsc(struct clocksource *arg)return read_hv_sched_clock_tsc (); } +static void suspend_hv_clock_tsc (struct clocksource *arg) +{ +u64 tsc_msr; + +/* Disable the TSC page */ +hv_get_reference_tsc (tsc_msr); + tsc_msr &= ~BIT_ULL (0 ); +hv_set_reference_tsc (tsc_msr); +} + + +static void resume_hv_clock_tsc (struct clocksource *arg) +{ +phys_addr_t phys_addr =page_to_phys (vmalloc_to_page (tsc_pg)); +u64 tsc_msr; + +/* Re-enable the TSC page */ +hv_get_reference_tsc (tsc_msr); + tsc_msr &=GENMASK_ULL (11 ,0 ); + tsc_msr |=BIT_ULL (0 ) | (u64 )phys_addr; +hv_set_reference_tsc (tsc_msr); +} +static struct clocksource hyperv_cs_tsc = { .name ="hyperv_clocksource_tsc_page" , .rating =400 , .read = read_hv_clock_tsc, .mask =CLOCKSOURCE_MASK (64 ), .flags =CLOCK_SOURCE_IS_CONTINUOUS , + .suspend= suspend_hv_clock_tsc, + .resume = resume_hv_clock_tsc, };#endif
Signature · 1 line
1.8.3.1
Dexuan Cui
maintainerPATCH v2 3/7 Drivers: hv: vmbus: Break out synic enable and disable operations
In reply to Dexuan CuiBreak out synic enable and disable operations into separate hv_synic_disable_regs() and hv_synic_enable_regs() functions for use by a later patch to support hibernation.
There is no functional change except the unnecessary check
"if (sctrl.enable != 1) return -EFAULT;" which is removed, because when we're in hv_synic_cleanup(), we're absolutely sure sctrl.enable must be 1.
--- drivers/hv/hv.c | 66 ++++++++++++++++++++++++++--------------------- drivers/hv/hyperv_vmbus.h | 2 ++ 2 files changed, 39 insertions(+), 29 deletions(-)
drivers/hv/hv.c +37 -29
@@ -154,7 +154,7 @@ void hv_synic_free(void)* retrieve the initialized message and event pages. Otherwise, we create and * initialize the message and event pages. */ -int hv_synic_init (unsigned int cpu) +void hv_synic_enable_regs (unsigned int cpu) {struct hv_per_cpu_context *hv_cpu =per_cpu_ptr (hv_context.cpu_context, cpu);
@@ -196,6 +196,11 @@ int hv_synic_init(unsigned int cpu) sctrl.enable =1 ;hv_set_synic_state (sctrl.as_uint64); +} + +int hv_synic_init (unsigned int cpu) +{ +hv_synic_enable_regs (cpu);hv_stimer_init (cpu);
@@ -205,20 +210,45 @@ int hv_synic_init(unsigned int cpu)/* * hv_synic_cleanup - Cleanup routine for hv_synic_init(). */ -int hv_synic_cleanup (unsigned int cpu) +void hv_synic_disable_regs (unsigned int cpu) {union hv_synic_sint shared_sint;union hv_synic_simp simp;union hv_synic_siefp siefp;union hv_synic_scontrol sctrl; + +hv_get_synint_state (VMBUS_MESSAGE_SINT , shared_sint.as_uint64); + + shared_sint.masked =1 ; + +/* Need to correctly cleanup in the case of SMP!!! */ +/* Disable the interrupt */ +hv_set_synint_state (VMBUS_MESSAGE_SINT , shared_sint.as_uint64); + +hv_get_simp (simp.as_uint64); + simp.simp_enabled =0 ; + simp.base_simp_gpa =0 ; + +hv_set_simp (simp.as_uint64); + +hv_get_siefp (siefp.as_uint64); + siefp.siefp_enabled =0 ; + siefp.base_siefp_gpa =0 ; + +hv_set_siefp (siefp.as_uint64); + +/* Disable the global synic bit */ +hv_get_synic_state (sctrl.as_uint64); + sctrl.enable =0 ; +hv_set_synic_state (sctrl.as_uint64); +} + +int hv_synic_cleanup (unsigned int cpu) +{struct vmbus_channel *channel, *sc;bool channel_found = false;unsigned long flags; -hv_get_synic_state (sctrl.as_uint64); -if (sctrl.enable !=1 ) -return -EFAULT ; -/* * Search for channels which are bound to the CPU we're about to * cleanup. In case we find one and vmbus is still connected we need to
@@ -249,29 +279,7 @@ int hv_synic_cleanup(unsigned int cpu)hv_stimer_cleanup (cpu); -hv_get_synint_state (VMBUS_MESSAGE_SINT , shared_sint.as_uint64); - - shared_sint.masked =1 ; - -/* Need to correctly cleanup in the case of SMP!!! */ -/* Disable the interrupt */ -hv_set_synint_state (VMBUS_MESSAGE_SINT , shared_sint.as_uint64); - -hv_get_simp (simp.as_uint64); - simp.simp_enabled =0 ; - simp.base_simp_gpa =0 ; - -hv_set_simp (simp.as_uint64); - -hv_get_siefp (siefp.as_uint64); - siefp.siefp_enabled =0 ; - siefp.base_siefp_gpa =0 ; - -hv_set_siefp (siefp.as_uint64); - -/* Disable the global synic bit */ - sctrl.enable =0 ; -hv_set_synic_state (sctrl.as_uint64); +hv_synic_disable_regs (cpu);return 0 ; }
drivers/hv/hyperv_vmbus.h +2 -0
@@ -171,8 +171,10 @@ extern int hv_post_message(union hv_connection_id connection_id,extern void hv_synic_free (void ); +extern void hv_synic_enable_regs (unsigned int cpu);extern int hv_synic_init (unsigned int cpu); +extern void hv_synic_disable_regs (unsigned int cpu);extern int hv_synic_cleanup (unsigned int cpu);/* Interface */
Signature · 1 line
1.8.3.1
Dexuan Cui
maintainerPATCH v2 4/7 Drivers: hv: vmbus: Suspend/resume the synic for hibernation
In reply to Dexuan CuiThis is needed when we resume the old kernel from the "current" kernel.
Note: when hv_synic_suspend() and hv_synic_resume() run, all the non-boot CPUs have been offlined, and interrupts are disabled on CPU0.
--- drivers/hv/vmbus_drv.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+)
drivers/hv/vmbus_drv.c +46 -0
@@ -30,6 +30,7 @@#include <linux/kdebug.h> #include <linux/efi.h> #include <linux/random.h> +#include <linux/syscore_ops.h> #include <clocksource/hyperv_timer.h> #include "hyperv_vmbus.h"
@@ -2086,6 +2087,47 @@ static void hv_crash_handler(struct pt_regs *regs)hyperv_cleanup (); }; +static int hv_synic_suspend (void ) +{ +/* +* When we reach here, all the non-boot CPUs have been offlined, and +* the stimers on them have been unbound in hv_synic_cleanup() -> +* hv_stimer_cleanup() -> clockevents_unbind_device(). +* +* hv_synic_suspend() only runs on CPU0 with interrupts disabled. Here +* we do not unbind the stimer on CPU0 because: 1) it's unnecessary +* because the interrupts remain disabled between syscore_suspend() +* and syscore_resume(): see create_image() and resume_target_kernel(); +* 2) the stimer on CPU0 is automatically disabled later by +* syscore_suspend() -> timekeeping_suspend() -> tick_suspend() -> ... +* -> clockevents_shutdown() -> ... -> hv_ce_shutdown(); 3) a warning +* would be triggered if we call clockevents_unbind_device(), which +* may sleep, in an interrupts-disabled context. So, we intentionally +* don't call hv_stimer_cleanup(0) here. +*/ + +hv_synic_disable_regs (0 ); + +return 0 ; +} + +static void hv_synic_resume (void ) +{ +hv_synic_enable_regs (0 ); + +/* +* Note: we don't need to call hv_stimer_init(0), because the timer +* on CPU0 is not unbound in hv_synic_suspend(), and the timer is +* automatically re-enabled in timekeeping_resume(). +*/ +} + +/* The callbacks run only on CPU0, with irqs_disabled. */ +static struct syscore_ops hv_synic_syscore_ops = { + .suspend = hv_synic_suspend, + .resume = hv_synic_resume, +}; +static int __init hv_acpi_init (void ) {int ret, t;
@@ -2116,6 +2158,8 @@ static int __init hv_acpi_init(void)hv_setup_kexec_handler (hv_kexec_handler);hv_setup_crash_handler (hv_crash_handler); +register_syscore_ops (&hv_synic_syscore_ops); +return 0 ; cleanup:
@@ -2128,6 +2172,8 @@ static void __exit vmbus_exit(void) {int cpu; +unregister_syscore_ops (&hv_synic_syscore_ops); +hv_remove_kexec_handler ();hv_remove_crash_handler (); vmbus_connection.conn_state =DISCONNECTED ;
Signature · 1 line
1.8.3.1
RE: [PATCH v2 4/7] Drivers: hv: vmbus: Suspend/resume the synic for hibernation
From: Dexuan Cui <[email protected]> Sent: Wednesday, July 31, 2019 10:52 AM
Quoted 11 lines
This is needed when we resume the old kernel from the "current" kernel.
Note: when
hv_synic_suspend()andhv_synic_resume()run, all the non-boot CPUs have been offlined, and interrupts are disabled on CPU0.Signed-off-by Dexuan Cui <[email protected]>--- drivers/hv/vmbus_drv.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+)
Dexuan Cui
maintainerPATCH v2 5/7 Drivers: hv: vmbus: Ignore the offers when resuming from hibernation
In reply to Dexuan CuiWhen the VM resumes, the host re-sends the offers. We should not add the offers to the global vmbus_connection.chn_list again.
Added some debug code, in case the host screws up the exact info related to the offers.
--- drivers/hv/channel_mgmt.c | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-)
drivers/hv/channel_mgmt.c +28 -1
@@ -854,12 +854,39 @@ void vmbus_initiate_unload(bool crash)static void vmbus_onoffer (struct vmbus_channel_message_header *hdr) {struct vmbus_channel_offer_channel *offer; -struct vmbus_channel *newchannel; +struct vmbus_channel *oldchannel, *newchannel; +size_t offer_sz; offer = (struct vmbus_channel_offer_channel *)hdr;trace_vmbus_onoffer (offer); +mutex_lock (&vmbus_connection.channel_mutex); + oldchannel =relid2channel (offer->child_relid); +mutex_unlock (&vmbus_connection.channel_mutex); + +if (oldchannel !=NULL ) { +atomic_dec (&vmbus_connection.offer_in_progress); + +/* +* We're resuming from hibernation: we expect the host to send +* exactly the same offers that we had before the hibernation. +*/ + offer_sz =sizeof (*offer); +if (memcmp (offer, &oldchannel->offermsg, offer_sz) ==0 ) +return ; + +pr_err ("Mismatched offer from the host (relid=%d)!\n" , + offer->child_relid); + +print_hex_dump_debug ("Old vmbus offer: " ,DUMP_PREFIX_OFFSET , +16 ,4 , &oldchannel->offermsg, offer_sz, + false); +print_hex_dump_debug ("New vmbus offer: " ,DUMP_PREFIX_OFFSET , +16 ,4 , offer, offer_sz, false); +return ; + } +/* Allocate the channel object and save this offer. */ newchannel =alloc_channel ();if (!newchannel) {
Signature · 1 line
1.8.3.1
RE: [PATCH v2 5/7] Drivers: hv: vmbus: Ignore the offers when resuming from hibernation
From: Dexuan Cui <[email protected]> Sent: Wednesday, July 31, 2019 10:52 AM
Quoted 4 lines
When the VM resumes, the host re-sends the offers. We should not add the offers to the global vmbus_connection.chn_list again.
Quoted 8 lines
Added some debug code, in case the host screws up the exact info related to the offers.
Signed-off-by Dexuan Cui <[email protected]>--- drivers/hv/channel_mgmt.c | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-)
Dexuan Cui
maintainerPATCH v2 6/7 Drivers: hv: vmbus: Suspend/resume the vmbus itself for hibernation
In reply to Dexuan CuiBefore Linux enters hibernation, it sends the CHANNELMSG_UNLOAD message to the host so all the offers are gone. After hibernation, Linux needs to re-negotiate with the host using the same vmbus protocol version (which was in use before hibernation), and ask the host to re-offer the vmbus devices.
--- drivers/hv/connection.c | 3 +-- drivers/hv/hyperv_vmbus.h | 2 ++ drivers/hv/vmbus_drv.c | 50 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-)
drivers/hv/connection.c +1 -2
@@ -59,8 +59,7 @@ static __u32 vmbus_get_next_version(__u32 current_version) } } -static int vmbus_negotiate_version (struct vmbus_channel_msginfo *msginfo, -__u32 version) +int vmbus_negotiate_version (struct vmbus_channel_msginfo *msginfo,u32 version) {int ret =0 ;unsigned int cur_cpu;
drivers/hv/hyperv_vmbus.h +2 -0
@@ -274,6 +274,8 @@ struct vmbus_msginfo {extern struct vmbus_connection vmbus_connection; +int vmbus_negotiate_version (struct vmbus_channel_msginfo *msginfo,u32 version); +static inline void vmbus_send_interrupt (u32 relid) {sync_set_bit (relid, vmbus_connection.send_int_page);
drivers/hv/vmbus_drv.c +50 -0
68 lines not on this page · raw message
Signature · 1 line
1.8.3.1
RE: [PATCH v2 6/7] Drivers: hv: vmbus: Suspend/resume the vmbus itself for hibernation
From: Dexuan Cui <[email protected]> Sent: Wednesday, July 31, 2019 10:52 AM
Quoted 7 lines
Before Linux enters hibernation, it sends the
CHANNELMSG_UNLOADmessage to the host so all the offers are gone. After hibernation, Linux needs to re-negotiate with the host using the same vmbus protocol version (which was in use before hibernation), and ask the host to re-offer the vmbus devices.
Quoted 8 lines
Signed-off-by Dexuan Cui <[email protected]>--- drivers/hv/connection.c | 3 +-- drivers/hv/hyperv_vmbus.h | 2 ++ drivers/hv/vmbus_drv.c | 50 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-)
Dexuan Cui
maintainerRE: [PATCH v2 6/7] Drivers: hv: vmbus: Suspend/resume the vmbus itself for hibernation
In reply to Dexuan Cuidrivers/hv/vmbus_drv.c static int vmbus_acpi_add(struct acpi_device · line 2101
Quoted 11 lines
From: Dexuan Cui <[email protected]>
Sent: Wednesday, July 31, 2019 10:52 AM
To: [email protected]; [email protected]; Stephendrivers/hv/vmbus_drv.c +3
-0@@ -2050,6 +2095,10 @@ static int vmbus_acpi_add(struct acpi_device *device) };MODULE_DEVICE_TABLE (acpi, vmbus_acpi_device_ids); +static const struct dev_pm_ops vmbus_bus_pm = { +SET_SYSTEM_SLEEP_PM_OPS (vmbus_bus_suspend, vmbus_bus_resume) +};
It turns out this " SET_SYSTEM_SLEEP_PM_OPS" should be changed to "SET_NOIRQ_SYSTEM_SLEEP_PM_OPS" to make NIC SR-IOV work with hibernation.
This is because the "pci_dev_pm_ops" uses the "noirq" callbacks. In the resume path, the "noirq" restore callback runs before the non-noirq callbacks, meaning the vmbus_bus_resume() and the pci-hyperv's .resume() must also run via the "noirq" callbacks.
Similarly, I also need to make the same change in this patch:
[PATCH v2 7/7] Drivers: hv: vmbus: Implement suspend/resume for VSC drivers for hibernation
BTW, I'd like to change the print_hex_dump_debug() to print_hex_dump() in this patch:
[PATCH v2 5/7] Drivers: hv: vmbus: Ignore the offers when resuming from hibernation
print_hex_dump_debug() outputs nothing unless we enable the dyndbg. This is not good as we do want see the error messages here, if there is an error.
I'll do more tests and post a v3 of the patchset.
Thanks,
-- Dexuan
Dexuan Cui
maintainerPATCH v2 7/7 Drivers: hv: vmbus: Implement suspend/resume for VSC drivers for hibernation
In reply to Dexuan CuiThe high-level VSC drivers will implement device-specific callbacks.
--- drivers/hv/vmbus_drv.c | 42 ++++++++++++++++++++++++++++++++++++++++++ include/linux/hyperv.h | 3 +++ 2 files changed, 45 insertions(+)
drivers/hv/vmbus_drv.c +42 -0
@@ -911,6 +911,43 @@ static void vmbus_shutdown(struct device *child_device) drv->shutdown (dev); } +/* +* vmbus_suspend - Suspend a vmbus device +*/ +static int vmbus_suspend (struct device *child_device) +{ +struct hv_driver *drv; +struct hv_device *dev =device_to_hv_device (child_device); + +/* The device may not be attached yet */ +if (!child_device->driver) +return 0 ; + + drv =drv_to_hv_drv (child_device->driver); +if (!drv->suspend) +return -EOPNOTSUPP ; + +return drv->suspend (dev); +} + +/* +* vmbus_resume - Resume a vmbus device +*/ +static int vmbus_resume (struct device *child_device) +{ +struct hv_driver *drv; +struct hv_device *dev =device_to_hv_device (child_device); + +/* The device may not be attached yet */ +if (!child_device->driver) +return 0 ; + + drv =drv_to_hv_drv (child_device->driver); +if (!drv->resume) +return -EOPNOTSUPP ; + +return drv->resume (dev); +}/* * vmbus_device_release - Final callback release of the vmbus child device
@@ -926,6 +963,10 @@ static void vmbus_device_release(struct device *device)kfree (hv_dev); } +static const struct dev_pm_ops vmbus_pm = { +SET_SYSTEM_SLEEP_PM_OPS (vmbus_suspend, vmbus_resume) +}; +/* The one and only one */ static struct bus_type hv_bus = { .name ="vmbus" ,
@@ -936,6 +977,7 @@ static void vmbus_device_release(struct device *device) .uevent = vmbus_uevent, .dev_groups = vmbus_dev_groups, .drv_groups = vmbus_drv_groups, + .pm = &vmbus_pm, };struct onmessage_work_context {
include/linux/hyperv.h +3 -0
@@ -1149,6 +1149,9 @@ struct hv_driver {int (*remove)(struct hv_device *);void (*shutdown)(struct hv_device *); +int (*suspend)(struct hv_device *); +int (*resume)(struct hv_device *); + };/* Base device object */
Signature · 1 line
1.8.3.1