Skip to contentSkip to conversation

FF-A VM to VM support

RFC PATCH Superseded Bertrand Marquis 4 participants 12 messages Oct 16, 2024 – Nov 4, 2024

v1 of 8 Superseded Latest is v8 →

lore
Share & tools
lore
#0 lore raw

cover FF-A VM to VM support

This RFC serie is a first proposal to enable VM to VM communications through FF-A on Arm.
It is sent as an RFC because some aspects of this serie will need to be discussed:
- How to control/configure which VM can communicate with which VM
- How to enable this without conflicting with Optee support
- Would a system communicating with Optee without FF-A but using FF-A to communicate between VM make any sense

One main feature missing from this serie is the support for memory sharing or lending through FF-A. Implementing such a feature is raising the question of how to cleanly have FF-A mem sharing living on the side of Xen grantables.

This serie must be applied on top of [1] which introduces some rework of FF-A support required by this patch serie.

!! This patch serie must no be merged !!

[1] https://lore.kernel.org/xen-devel/[email protected]/

Bertrand Marquis (4):
xen/arm: ffa: Introduce VM to VM support xen/arm: ffa: Add buffer full notification support xen/arm: ffa: Add indirect message between VM xen/arm: ffa: Enable VM to VM without firmware

 xen/arch/arm/tee/Kconfig        |  11 +++
 xen/arch/arm/tee/ffa.c          |  11 +++
 xen/arch/arm/tee/ffa_msg.c      |  96 +++++++++++++++++++--
 xen/arch/arm/tee/ffa_notif.c    | 142 ++++++++++++++++++--------------
 xen/arch/arm/tee/ffa_partinfo.c | 137 ++++++++++++++++++++++++------
 xen/arch/arm/tee/ffa_private.h  |  25 ++++++
 6 files changed, 329 insertions(+), 93 deletions(-)
Signature · 1 line
2.47.0
#1 lore raw

RFC PATCH 1/4 xen/arm: ffa: Introduce VM to VM support

Create a CONFIG_FFA_VM_TO_VM parameter to activate FFA communication between VMs.
When activated list VMs in the system with FF-A support in part_info_get.

WARNING: There is no filtering for now and all VMs are listed !!

---
 xen/arch/arm/tee/Kconfig        |  11 +++
 xen/arch/arm/tee/ffa_partinfo.c | 135 ++++++++++++++++++++++++++------
 xen/arch/arm/tee/ffa_private.h  |  12 +++
 3 files changed, 135 insertions(+), 23 deletions(-)
xen/arch/arm/tee/Kconfig +11 -0
index c5b0f88d7522..88a4c4c99154 100644
@@ -28,5 +28,16 @@ config FFA
 
 	  [1] https://developer.arm.com/documentation/den0077/latest
 
+config FFA_VM_TO_VM
+    bool "Enable FF-A between VMs (UNSUPPORTED)" if UNSUPPORTED
+    default n
+    depends on FFA
+    help
+      This option enables to use FF-A between VMs.
+      This is experimental and there is no access control so any
+      guest can communicate with any other guest.
+
+      If unsure, say N.
+
 endmenu
 
xen/arch/arm/tee/ffa_partinfo.c +112 -23

170 lines not on this page · raw message

xen/arch/arm/tee/ffa_private.h +12 -0
index d441c0ca5598..47dd6b5fadea 100644
@@ -187,6 +187,18 @@
  */
 #define FFA_PARTITION_INFO_GET_COUNT_FLAG BIT(0, U)
 
+/*
+ * Partition properties we give for a normal world VM:
+ * - can send direct message but not receive them
+ * - can handle indirect messages
+ * - can receive notifications
+ * 32/64 bit flag is set depending on the VM
+ */
+#define FFA_PART_VM_PROP    (FFA_PART_PROP_DIRECT_REQ_SEND | \
+                             FFA_PART_PROP_INDIRECT_MSGS | \
+                             FFA_PART_PROP_RECV_NOTIF | \
+                             FFA_PART_PROP_IS_PE_ID)
+
 /* Flags used in calls to FFA_NOTIFICATION_GET interface  */
 #define FFA_NOTIF_FLAG_BITMAP_SP        BIT(0, U)
 #define FFA_NOTIF_FLAG_BITMAP_VM        BIT(1, U)
Signature · 1 line
2.47.0
#2 lore raw

Re: [RFC PATCH 1/4] xen/arm: ffa: Introduce VM to VM support

Quoted 14 lines · Bertrand Marquis
xen/arch/arm/tee/ffa_partinfo.c +5 -1
index fde187dba4e5..d699a267cc76 100644
@@ -77,7 +77,21 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
     };
     uint32_t src_size, dst_size;
     void *dst_buf;
-    uint32_t ffa_sp_count = 0;
+    uint32_t ffa_vm_count = 0, ffa_sp_count = 0;
+#ifdef CONFIG_FFA_VM_TO_VM
+    struct domain *dom;
+
+    /* Count the number of VM with FF-A support */

xen/arch/arm/tee/ffa_partinfo.c void ffa_handle_partition_info_get(struct cpu_user_regs *regs) · line 94

Quoted 10 lines
xen/arch/arm/tee/ffa_partinfo.c +10 -0
+    rcu_read_lock(&domlist_read_lock);
+    for_each_domain( dom )
+    {
+        struct ffa_ctx *vm = dom->arch.tee;
+
+        if (dom != d && vm != NULL && vm->guest_vers != 0)
+            ffa_vm_count++;
+    }
+    rcu_read_unlock(&domlist_read_lock);
+#endif

...
struct domain *dom;

if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )
{
    /* Count the number of VM with FF-A support */ rcu_read_lock(&domlist_read_lock);
    ...
    rcu_read_unlock(&domlist_read_lock);
}

drops the explicit ifdef.  Hiding function-level variable declarations behind an ifdef like that works exactly once, and it doesn't make pleasant code.

~Andrew

#3 lore raw

Re: [RFC PATCH 1/4] xen/arm: ffa: Introduce VM to VM support

Hi Andrew,

Quoted 39 lines · Andrew Cooper
Quoted 14 lines · Bertrand Marquis
xen/arch/arm/tee/ffa_partinfo.c +5 -1
index fde187dba4e5..d699a267cc76 100644
@@ -77,7 +77,21 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
    };
    uint32_t src_size, dst_size;
    void *dst_buf;
-    uint32_t ffa_sp_count = 0;
+    uint32_t ffa_vm_count = 0, ffa_sp_count = 0;
+#ifdef CONFIG_FFA_VM_TO_VM
+    struct domain *dom;
+
+    /* Count the number of VM with FF-A support */

xen/arch/arm/tee/ffa_partinfo.c void ffa_handle_partition_info_get(struct cpu_user_regs *regs) · line 94

Quoted 10 lines
xen/arch/arm/tee/ffa_partinfo.c +10 -0
+    rcu_read_lock(&domlist_read_lock);
+    for_each_domain( dom )
+    {
+        struct ffa_ctx *vm = dom->arch.tee;
+
+        if (dom != d && vm != NULL && vm->guest_vers != 0)
+            ffa_vm_count++;
+    }
+    rcu_read_unlock(&domlist_read_lock);
+#endif

...
struct domain *dom;

if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )

{
    /* Count the number of VM with FF-A support */
    rcu_read_lock(&domlist_read_lock);
    ...
    rcu_read_unlock(&domlist_read_lock);
}
Quoted 3 lines

drops the explicit ifdef. Hiding function-level variable declarations behind an ifdef like that works exactly once, and it doesn't make pleasant code.

Ack I will fix that.

Cheers
Bertrand

#4 lore raw

Re: [RFC PATCH 1/4] xen/arm: ffa: Introduce VM to VM support

Quoted 41 lines · Bertrand Marquis

Hi Andrew,

Quoted 39 lines · Andrew Cooper
Quoted 14 lines · Bertrand Marquis
xen/arch/arm/tee/ffa_partinfo.c +5 -1
index fde187dba4e5..d699a267cc76 100644
@@ -77,7 +77,21 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
    };
    uint32_t src_size, dst_size;
    void *dst_buf;
-    uint32_t ffa_sp_count = 0;
+    uint32_t ffa_vm_count = 0, ffa_sp_count = 0;
+#ifdef CONFIG_FFA_VM_TO_VM
+    struct domain *dom;
+
+    /* Count the number of VM with FF-A support */

xen/arch/arm/tee/ffa_partinfo.c void ffa_handle_partition_info_get(struct cpu_user_regs *regs) · line 94

Quoted 10 lines
xen/arch/arm/tee/ffa_partinfo.c +10 -0
+    rcu_read_lock(&domlist_read_lock);
+    for_each_domain( dom )
+    {
+        struct ffa_ctx *vm = dom->arch.tee;
+
+        if (dom != d && vm != NULL && vm->guest_vers != 0)
+            ffa_vm_count++;
+    }
+    rcu_read_unlock(&domlist_read_lock);
+#endif

...
struct domain *dom;

if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )

{
    /* Count the number of VM with FF-A support */
    rcu_read_lock(&domlist_read_lock);
    ...
    rcu_read_unlock(&domlist_read_lock);
}
Quoted 5 lines
Quoted 3 lines

drops the explicit ifdef. Hiding function-level variable declarations behind an ifdef like that works exactly once, and it doesn't make pleasant code.

Ack I will fix that.

While at that, please also name struct domain * type variables "d", not "dom". For consistency with (almost) all other code we have.

Jan

#5 lore raw

Re: [RFC PATCH 1/4] xen/arm: ffa: Introduce VM to VM support

Hi Jan,

Quoted 44 lines · Jan Beulich
Quoted 41 lines · Bertrand Marquis

Hi Andrew,

Quoted 39 lines · Andrew Cooper
Quoted 14 lines · Bertrand Marquis
xen/arch/arm/tee/ffa_partinfo.c +5 -1
index fde187dba4e5..d699a267cc76 100644
@@ -77,7 +77,21 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
   };
   uint32_t src_size, dst_size;
   void *dst_buf;
-    uint32_t ffa_sp_count = 0;
+    uint32_t ffa_vm_count = 0, ffa_sp_count = 0;
+#ifdef CONFIG_FFA_VM_TO_VM
+    struct domain *dom;
+
+    /* Count the number of VM with FF-A support */

xen/arch/arm/tee/ffa_partinfo.c void ffa_handle_partition_info_get(struct cpu_user_regs *regs) · line 94

Quoted 10 lines
xen/arch/arm/tee/ffa_partinfo.c +10 -0
+    rcu_read_lock(&domlist_read_lock);
+    for_each_domain( dom )
+    {
+        struct ffa_ctx *vm = dom->arch.tee;
+
+        if (dom != d && vm != NULL && vm->guest_vers != 0)
+            ffa_vm_count++;
+    }
+    rcu_read_unlock(&domlist_read_lock);
+#endif

...
struct domain *dom;

if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )

{
   /* Count the number of VM with FF-A support */
   rcu_read_lock(&domlist_read_lock);
   ...
   rcu_read_unlock(&domlist_read_lock);
}
Quoted 8 lines
Quoted 5 lines
Quoted 3 lines

drops the explicit ifdef. Hiding function-level variable declarations behind an ifdef like that works exactly once, and it doesn't make pleasant code.

Ack I will fix that.

While at that, please also name struct domain * type variables "d", not "dom". For consistency with (almost) all other code we have.

Sure, I will fix that.

Cheers
Bertrand

Quoted 2 lines

Jan

#6 lore raw

RFC PATCH 2/4 xen/arm: ffa: Add buffer full notification support

In reply to Bertrand Marquis

Add support to raise a Rx buffer full notification to a VM. This function will be used for indirect message support between VM and is only activated if CONFIG_FFA_VM_TO_VM is selected.

Even if there are 32 framework notifications possible, right now only one is defined so the implementation is simplified to only handle the buffer full notification using a boolean. If other framework notifications have to be supported one day, the design will have to be modified to handle it properly.

---
 xen/arch/arm/tee/ffa_notif.c   | 26 +++++++++++++++++++++-----
 xen/arch/arm/tee/ffa_private.h | 13 +++++++++++++
 2 files changed, 34 insertions(+), 5 deletions(-)
xen/arch/arm/tee/ffa_notif.c +21 -5
index 3c6418e62e2b..052b3e364a70 100644
@@ -93,6 +93,7 @@ void ffa_handle_notification_info_get(struct cpu_user_regs *regs)
 void ffa_handle_notification_get(struct cpu_user_regs *regs)
 {
     struct domain *d = current->domain;
+    struct ffa_ctx *ctx = d->arch.tee;
     uint32_t recv = get_user_reg(regs, 1);
     uint32_t flags = get_user_reg(regs, 2);
     uint32_t w2 = 0;
@@ -132,11 +133,7 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
          */
         if ( ( flags  & FFA_NOTIF_FLAG_BITMAP_SP ) &&
              ( flags & FFA_NOTIF_FLAG_BITMAP_SPM ) )
-        {
-                struct ffa_ctx *ctx = d->arch.tee;
-
-                ACCESS_ONCE(ctx->notif.secure_pending) = false;
-        }
+            ACCESS_ONCE(ctx->notif.secure_pending) = false;
 
         arm_smccc_1_2_smc(&arg, &resp);
         e = ffa_get_ret_code(&resp);
@@ -156,6 +153,12 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
             w6 = resp.a6;
     }
 
+#ifdef CONFIG_FFA_VM_TO_VM
+    if ( flags & FFA_NOTIF_FLAG_BITMAP_HYP &&
+            test_and_clear_bool(ctx->notif.buff_full_pending) )
+        w7 = FFA_NOTIF_RX_BUFFER_FULL;
+#endif
+
     ffa_set_regs(regs, FFA_SUCCESS_32, 0, w2, w3, w4, w5, w6, w7);
 }
 
@@ -178,6 +181,19 @@ int ffa_handle_notification_set(struct cpu_user_regs *regs)
                            bitmap_hi);
 }
 
+#ifdef CONFIG_FFA_VM_TO_VM
+void ffa_raise_rx_buffer_full(struct domain *d)
+{
+    struct ffa_ctx *ctx = d->arch.tee;
+
+    if ( !ctx )
+        return;
+
+    if ( !test_and_set_bool(ctx->notif.buff_full_pending) )
+        vgic_inject_irq(d, d->vcpu[0], notif_sri_irq, true);
+}
+#endif
+
 /*
  * Extract a 16-bit ID (index n) from the successful return value from
  * FFA_NOTIFICATION_INFO_GET_64 or FFA_NOTIFICATION_INFO_GET_32. IDs are
xen/arch/arm/tee/ffa_private.h +13 -0
index 47dd6b5fadea..ad1dd04aeb7c 100644
@@ -210,6 +210,8 @@
 #define FFA_NOTIF_INFO_GET_ID_COUNT_SHIFT   7
 #define FFA_NOTIF_INFO_GET_ID_COUNT_MASK    0x1F
 
+#define FFA_NOTIF_RX_BUFFER_FULL        BIT(0, U)
+
 /* Feature IDs used with FFA_FEATURES */
 #define FFA_FEATURE_NOTIF_PEND_INTR     0x1U
 #define FFA_FEATURE_SCHEDULE_RECV_INTR  0x2U
@@ -298,6 +300,13 @@ struct ffa_ctx_notif {
      * pending global notifications.
      */
     bool secure_pending;
+
+#ifdef CONFIG_FFA_VM_TO_VM
+    /*
+     * Pending Hypervisor framework notifications
+     */
+    bool buff_full_pending;
+#endif
 };
 
 struct ffa_ctx {
@@ -370,6 +379,10 @@ void ffa_handle_notification_info_get(struct cpu_user_regs *regs);
 void ffa_handle_notification_get(struct cpu_user_regs *regs);
 int ffa_handle_notification_set(struct cpu_user_regs *regs);
 
+#ifdef CONFIG_FFA_VM_TO_VM
+void ffa_raise_rx_buffer_full(struct domain *d);
+#endif
+
 void ffa_handle_msg_send_direct_req(struct cpu_user_regs *regs, uint32_t fid);
 int32_t ffa_handle_msg_send2(struct cpu_user_regs *regs);
 
Signature · 1 line
2.47.0
#7 lore raw

RFC PATCH 3/4 xen/arm: ffa: Add indirect message between VM

In reply to Bertrand Marquis

Add support for indirect messages between VMs. This is only enabled if CONFIG_FFA_VM_TO_VM is selected.

---
 xen/arch/arm/tee/ffa_msg.c | 96 ++++++++++++++++++++++++++++++++++----
 1 file changed, 88 insertions(+), 8 deletions(-)
xen/arch/arm/tee/ffa_msg.c +88 -8
index 335f246ba657..25f184a06546 100644
@@ -95,9 +95,12 @@ int32_t ffa_handle_msg_send2(struct cpu_user_regs *regs)
     const struct ffa_part_msg_rxtx *src_msg;
     uint16_t dst_id, src_id;
     int32_t ret;
-
-    if ( !ffa_fw_supports_fid(FFA_MSG_SEND2) )
-        return FFA_RET_NOT_SUPPORTED;
+#ifdef CONFIG_FFA_VM_TO_VM
+    struct domain *dst_d;
+    struct ffa_ctx *dst_ctx;
+    struct ffa_part_msg_rxtx *dst_msg;
+    int err;
+#endif
 
     if ( !spin_trylock(&src_ctx->tx_lock) )
         return FFA_RET_BUSY;
@@ -106,10 +109,10 @@ int32_t ffa_handle_msg_send2(struct cpu_user_regs *regs)
     src_id = src_msg->send_recv_id >> 16;
     dst_id = src_msg->send_recv_id & GENMASK(15,0);
 
-    if ( src_id != ffa_get_vm_id(src_d) || !FFA_ID_IS_SECURE(dst_id) )
+    if ( src_id != ffa_get_vm_id(src_d) )
     {
         ret = FFA_RET_INVALID_PARAMETERS;
-        goto out_unlock_tx;
+        goto out;
     }
 
     /* check source message fits in buffer */
@@ -118,12 +121,89 @@ int32_t ffa_handle_msg_send2(struct cpu_user_regs *regs)
          src_msg->msg_offset < sizeof(struct ffa_part_msg_rxtx) )
     {
         ret = FFA_RET_INVALID_PARAMETERS;
-        goto out_unlock_tx;
+        goto out;
     }
 
-    ret = ffa_simple_call(FFA_MSG_SEND2, ((uint32_t)src_id) << 16, 0, 0, 0);
+    if ( FFA_ID_IS_SECURE(dst_id) )
+    {
+        /* Message for a secure partition */
+        if ( !ffa_fw_supports_fid(FFA_MSG_SEND2) )
+        {
+            ret = FFA_RET_NOT_SUPPORTED;
+            goto out;
+        }
+
+        ret = ffa_simple_call(FFA_MSG_SEND2, ((uint32_t)src_id) << 16, 0, 0,
+                              0);
+        goto out;
+    }
 
-out_unlock_tx:
+#ifndef CONFIG_FFA_VM_TO_VM
+    ret = FFA_RET_INVALID_PARAMETERS;
+#else
+    /* Message for a VM */
+    if ( dst_id == 0 )
+    {
+        /* FF-A ID 0 is the hypervisor, this is not valid */
+        ret = FFA_RET_INVALID_PARAMETERS;
+        goto out;
+    }
+
+    /* This is also checking that dest is not src */
+    err = rcu_lock_live_remote_domain_by_id(dst_id - 1, &dst_d);
+    if ( err )
+    {
+        ret = FFA_RET_INVALID_PARAMETERS;
+        goto out;
+    }
+
+    if ( dst_d->arch.tee == NULL )
+    {
+        ret = FFA_RET_INVALID_PARAMETERS;
+        goto out_unlock;
+    }
+
+    dst_ctx = dst_d->arch.tee;
+    if ( !dst_ctx->guest_vers )
+    {
+        ret = FFA_RET_INVALID_PARAMETERS;
+        goto out_unlock;
+    }
+
+    /* This also checks that destination has set a Rx buffer */
+    ret = ffa_rx_acquire(dst_d);
+    if ( ret )
+        goto out_unlock;
+
+    /* we need to have enough space in the destination buffer */
+    if ( dst_ctx->page_count * FFA_PAGE_SIZE <
+            (sizeof(struct ffa_part_msg_rxtx) + src_msg->msg_size) )
+    {
+        ret = FFA_RET_NO_MEMORY;
+        ffa_rx_release(dst_d);
+        goto out_unlock;
+    }
+
+    dst_msg = dst_ctx->rx;
+
+    /* prepare destination header */
+    dst_msg->flags = 0;
+    dst_msg->reserved = 0;
+    dst_msg->msg_offset = sizeof(struct ffa_part_msg_rxtx);
+    dst_msg->send_recv_id = src_msg->send_recv_id;
+    dst_msg->msg_size = src_msg->msg_size;
+
+    memcpy(dst_ctx->rx + sizeof(struct ffa_part_msg_rxtx),
+           src_ctx->tx + src_msg->msg_offset, src_msg->msg_size);
+
+    /* receiver rx buffer will be released by the receiver*/
+
+out_unlock:
+    rcu_unlock_domain(dst_d);
+    if ( !ret )
+        ffa_raise_rx_buffer_full(dst_d);
+#endif
+out:
     spin_unlock(&src_ctx->tx_lock);
     return ret;
 }
Signature · 1 line
2.47.0
#8 lore raw

RFC PATCH 4/4 xen/arm: ffa: Enable VM to VM without firmware

In reply to Bertrand Marquis

When VM to VM support is activated and there is no suitable FF-A support in the firmware, enable FF-A support for VMs to allow using it for VM to VM communications.
If there is Optee running in the secure world and using the non FF-A communication system, having CONFIG_FFA_VM_TO_VM could be non functional (if optee is probed first) or Optee could be non functional (if FF-A is probed first) so it is not recommended to activate the configuration option for such systems.

To make buffer full notification work between VMs when there is not firmware, rework the notification handling and modify the global flag to only be used as check for firmware notification support instead.

Modify part_info_get to return the list of VMs when there is no firmware support.

---
 xen/arch/arm/tee/ffa.c          |  11 +++
 xen/arch/arm/tee/ffa_notif.c    | 118 ++++++++++++++++----------------
 xen/arch/arm/tee/ffa_partinfo.c |   2 +
 3 files changed, 73 insertions(+), 58 deletions(-)
xen/arch/arm/tee/ffa.c +11 -0
index 21d41b452dc9..6d427864f3da 100644
@@ -324,8 +324,11 @@ static int ffa_domain_init(struct domain *d)
     struct ffa_ctx *ctx;
     int ret;
 
+#ifndef CONFIG_FFA_VM_TO_VM
     if ( !ffa_fw_version )
         return -ENODEV;
+#endif
+
     /*
      * We are using the domain_id + 1 as the FF-A ID for VMs as FF-A ID 0 is
      * reserved for the hypervisor and we only support secure endpoints using
@@ -549,7 +552,15 @@ err_no_fw:
     bitmap_zero(ffa_fw_abi_supported, FFA_ABI_BITMAP_SIZE);
     printk(XENLOG_WARNING "ARM FF-A No firmware support\n");
 
+#ifdef CONFIG_FFA_VM_TO_VM
+    INIT_LIST_HEAD(&ffa_teardown_head);
+    init_timer(&ffa_teardown_timer, ffa_teardown_timer_callback, NULL, 0);
+
+    printk(XENLOG_INFO "ARM FF-A only available between VMs\n");
+    return true;
+#else
     return false;
+#endif
 }
 
 static const struct tee_mediator_ops ffa_ops =
xen/arch/arm/tee/ffa_notif.c +60 -58

214 lines not on this page · raw message

xen/arch/arm/tee/ffa_partinfo.c +2 -0
index d699a267cc76..2e09440fe6c2 100644
@@ -128,12 +128,14 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
         goto out;
     }
 
+#ifndef CONFIG_FFA_VM_TO_VM
     if ( !ffa_fw_supports_fid(FFA_PARTITION_INFO_GET) )
     {
         /* Just give an empty partition list to the caller */
         ret = FFA_RET_OK;
         goto out;
     }
+#endif
 
     ret = ffa_rx_acquire(d);
     if ( ret != FFA_RET_OK )
Signature · 1 line
2.47.0
#9 lore raw

Re: [RFC PATCH 4/4] xen/arm: ffa: Enable VM to VM without firmware

Hi Bertrand,

Quoted 17 lines · Bertrand Marquis

When VM to VM support is activated and there is no suitable FF-A support in the firmware, enable FF-A support for VMs to allow using it for VM to VM communications.
If there is Optee running in the secure world and using the non FF-A communication system, having CONFIG_FFA_VM_TO_VM could be non functional (if optee is probed first) or Optee could be non functional (if FF-A is probed first) so it is not recommended to activate the configuration option for such systems.

To make buffer full notification work between VMs when there is not firmware, rework the notification handling and modify the global flag to only be used as check for firmware notification support instead.

Modify part_info_get to return the list of VMs when there is no firmware support.

Quoted 6 lines
---
 xen/arch/arm/tee/ffa.c          |  11 +++
 xen/arch/arm/tee/ffa_notif.c    | 118 ++++++++++++++++----------------
 xen/arch/arm/tee/ffa_partinfo.c |   2 +
 3 files changed, 73 insertions(+), 58 deletions(-)

I think it is desirable or at least a good goal to be able to have all TEE configurations enabled at compile time.

For optee_probe() to succeed, a DT node with compatible "linaro,optee-tz" must be present, and a trusted OS responding with the UUID used by OP-TEE. False positives can be ruled out unless the system is grossly misconfigured and shouldn't be used.

If we could make the probe order deterministic with OP-TEE before FF-A, we should be OK. If there is an odd system with OP-TEE SMC ABI in the secure world that wants to use FF-A VM to VM, removing the "linaro,optee-tz" compatible node from DT is enough to disable optee_probe() without recompiling Xen.

Cheers,
Jens

Quoted 282 lines
xen/arch/arm/tee/ffa.c +11 -0
index 21d41b452dc9..6d427864f3da 100644
@@ -324,8 +324,11 @@ static int ffa_domain_init(struct domain *d)
     struct ffa_ctx *ctx;
     int ret;
 
+#ifndef CONFIG_FFA_VM_TO_VM
     if ( !ffa_fw_version )
         return -ENODEV;
+#endif
+
     /*
      * We are using the domain_id + 1 as the FF-A ID for VMs as FF-A ID 0 is
      * reserved for the hypervisor and we only support secure endpoints using
@@ -549,7 +552,15 @@ err_no_fw:
     bitmap_zero(ffa_fw_abi_supported, FFA_ABI_BITMAP_SIZE);
     printk(XENLOG_WARNING "ARM FF-A No firmware support\n");
 
+#ifdef CONFIG_FFA_VM_TO_VM
+    INIT_LIST_HEAD(&ffa_teardown_head);
+    init_timer(&ffa_teardown_timer, ffa_teardown_timer_callback, NULL, 0);
+
+    printk(XENLOG_INFO "ARM FF-A only available between VMs\n");
+    return true;
+#else
     return false;
+#endif
 }
 
 static const struct tee_mediator_ops ffa_ops =
xen/arch/arm/tee/ffa_notif.c +60 -58
index 052b3e364a70..f2c87d1320de 100644
@@ -16,7 +16,7 @@
 
 #include "ffa_private.h"
 
-static bool __ro_after_init notif_enabled;
+static bool __ro_after_init fw_notif_enabled;
 static unsigned int __ro_after_init notif_sri_irq;
 
 int ffa_handle_notification_bind(struct cpu_user_regs *regs)
@@ -27,21 +27,17 @@ int ffa_handle_notification_bind(struct cpu_user_regs *regs)
     uint32_t bitmap_lo = get_user_reg(regs, 3);
     uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
     if ( (src_dst & 0xFFFFU) != ffa_get_vm_id(d) )
         return FFA_RET_INVALID_PARAMETERS;
 
     if ( flags )    /* Only global notifications are supported */
         return FFA_RET_DENIED;
 
-    /*
-     * We only support notifications from SP so no need to check the sender
-     * endpoint ID, the SPMC will take care of that for us.
-     */
-    return ffa_simple_call(FFA_NOTIFICATION_BIND, src_dst, flags, bitmap_hi,
-                           bitmap_lo);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )
+        return ffa_simple_call(FFA_NOTIFICATION_BIND, src_dst, flags,
+                               bitmap_hi, bitmap_lo);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 int ffa_handle_notification_unbind(struct cpu_user_regs *regs)
@@ -51,32 +47,36 @@ int ffa_handle_notification_unbind(struct cpu_user_regs *regs)
     uint32_t bitmap_lo = get_user_reg(regs, 3);
     uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
     if ( (src_dst & 0xFFFFU) != ffa_get_vm_id(d) )
         return FFA_RET_INVALID_PARAMETERS;
 
-    /*
-     * We only support notifications from SP so no need to check the
-     * destination endpoint ID, the SPMC will take care of that for us.
-     */
-    return  ffa_simple_call(FFA_NOTIFICATION_UNBIND, src_dst, 0, bitmap_hi,
-                            bitmap_lo);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )
+        return  ffa_simple_call(FFA_NOTIFICATION_UNBIND, src_dst, 0, bitmap_hi,
+                                bitmap_lo);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 void ffa_handle_notification_info_get(struct cpu_user_regs *regs)
 {
     struct domain *d = current->domain;
     struct ffa_ctx *ctx = d->arch.tee;
+    bool notif_pending = false;
 
-    if ( !notif_enabled )
+#ifndef CONFIG_FFA_VM_TO_VM
+    if ( !fw_notif_enabled )
     {
         ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
         return;
     }
+#endif
 
-    if ( test_and_clear_bool(ctx->notif.secure_pending) )
+    notif_pending = ctx->notif.secure_pending;
+#ifdef CONFIG_FFA_VM_TO_VM
+    notif_pending |= ctx->notif.buff_full_pending;
+#endif
+
+    if ( notif_pending )
     {
         /* A pending global notification for the guest */
         ffa_set_regs(regs, FFA_SUCCESS_64, 0,
@@ -103,11 +103,13 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
     uint32_t w6 = 0;
     uint32_t w7 = 0;
 
-    if ( !notif_enabled )
+#ifndef CONFIG_FFA_VM_TO_VM
+    if ( !fw_notif_enabled )
     {
         ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
         return;
     }
+#endif
 
     if ( (recv & 0xFFFFU) != ffa_get_vm_id(d) )
     {
@@ -115,7 +117,8 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
         return;
     }
 
-    if ( flags & ( FFA_NOTIF_FLAG_BITMAP_SP | FFA_NOTIF_FLAG_BITMAP_SPM ) )
+    if ( fw_notif_enabled && (flags & ( FFA_NOTIF_FLAG_BITMAP_SP
+                                        | FFA_NOTIF_FLAG_BITMAP_SPM )) )
     {
         struct arm_smccc_1_2_regs arg = {
             .a0 = FFA_NOTIFICATION_GET,
@@ -170,15 +173,14 @@ int ffa_handle_notification_set(struct cpu_user_regs *regs)
     uint32_t bitmap_lo = get_user_reg(regs, 3);
     uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
     if ( (src_dst >> 16) != ffa_get_vm_id(d) )
         return FFA_RET_INVALID_PARAMETERS;
 
-    /* Let the SPMC check the destination of the notification */
-    return ffa_simple_call(FFA_NOTIFICATION_SET, src_dst, flags, bitmap_lo,
-                           bitmap_hi);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )
+        return ffa_simple_call(FFA_NOTIFICATION_SET, src_dst, flags, bitmap_lo,
+                               bitmap_hi);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 #ifdef CONFIG_FFA_VM_TO_VM
@@ -190,7 +192,7 @@ void ffa_raise_rx_buffer_full(struct domain *d)
         return;
 
     if ( !test_and_set_bool(ctx->notif.buff_full_pending) )
-        vgic_inject_irq(d, d->vcpu[0], notif_sri_irq, true);
+        vgic_inject_irq(d, d->vcpu[0], GUEST_FFA_NOTIF_PEND_INTR_ID, true);
 }
 #endif
 
@@ -363,7 +365,7 @@ void ffa_notif_init_interrupt(void)
 {
     int ret;
 
-    if ( notif_enabled && notif_sri_irq < NR_GIC_SGI )
+    if ( fw_notif_enabled && notif_sri_irq < NR_GIC_SGI )
     {
         /*
          * An error here is unlikely since the primary CPU has already
@@ -394,47 +396,47 @@ void ffa_notif_init(void)
     int ret;
 
     /* Only enable fw notification if all ABIs we need are supported */
-    if ( !(ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_CREATE) &&
-           ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_DESTROY) &&
-           ffa_fw_supports_fid(FFA_NOTIFICATION_GET) &&
-           ffa_fw_supports_fid(FFA_NOTIFICATION_INFO_GET_64)) )
-        return;
-
-    arm_smccc_1_2_smc(&arg, &resp);
-    if ( resp.a0 != FFA_SUCCESS_32 )
-        return;
-
-    irq = resp.a2;
-    notif_sri_irq = irq;
-    if ( irq >= NR_GIC_SGI )
-        irq_set_type(irq, IRQ_TYPE_EDGE_RISING);
-    ret = request_irq(irq, 0, notif_irq_handler, "FF-A notif", NULL);
-    if ( ret )
+    if ( ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_CREATE) &&
+         ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_DESTROY) &&
+         ffa_fw_supports_fid(FFA_NOTIFICATION_GET) &&
+         ffa_fw_supports_fid(FFA_NOTIFICATION_INFO_GET_64) )
     {
-        printk(XENLOG_ERR "ffa: request_irq irq %u failed: error %d\n",
-               irq, ret);
-        return;
-    }
+        arm_smccc_1_2_smc(&arg, &resp);
+        if ( resp.a0 != FFA_SUCCESS_32 )
+            return;
 
-    notif_enabled = true;
+        irq = resp.a2;
+        notif_sri_irq = irq;
+        if ( irq >= NR_GIC_SGI )
+            irq_set_type(irq, IRQ_TYPE_EDGE_RISING);
+        ret = request_irq(irq, 0, notif_irq_handler, "FF-A notif", NULL);
+        if ( ret )
+        {
+            printk(XENLOG_ERR "ffa: request_irq irq %u failed: error %d\n",
+                   irq, ret);
+            return;
+        }
+        fw_notif_enabled = true;
+    }
 }
 
 int ffa_notif_domain_init(struct domain *d)
 {
     int32_t res;
 
-    if ( !notif_enabled )
-        return 0;
+    if ( fw_notif_enabled )
+    {
 
-    res = ffa_notification_bitmap_create(ffa_get_vm_id(d), d->max_vcpus);
-    if ( res )
-        return -ENOMEM;
+        res = ffa_notification_bitmap_create(ffa_get_vm_id(d), d->max_vcpus);
+        if ( res )
+            return -ENOMEM;
+    }
 
     return 0;
 }
 
 void ffa_notif_domain_destroy(struct domain *d)
 {
-    if ( notif_enabled )
+    if ( fw_notif_enabled )
         ffa_notification_bitmap_destroy(ffa_get_vm_id(d));
 }
xen/arch/arm/tee/ffa_partinfo.c +2 -1
index d699a267cc76..2e09440fe6c2 100644
@@ -128,12 +128,14 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
         goto out;
     }
 
+#ifndef CONFIG_FFA_VM_TO_VM
     if ( !ffa_fw_supports_fid(FFA_PARTITION_INFO_GET) )
     {
         /* Just give an empty partition list to the caller */
         ret = FFA_RET_OK;
         goto out;
     }
+#endif
 
     ret = ffa_rx_acquire(d);
     if ( ret != FFA_RET_OK )
--

2.47.0

#10 lore raw

Re: [RFC PATCH 4/4] xen/arm: ffa: Enable VM to VM without firmware

Hi Jens,

Quoted 38 lines · Jens Wiklander

Hi Bertrand,

Quoted 17 lines · Bertrand Marquis

When VM to VM support is activated and there is no suitable FF-A support in the firmware, enable FF-A support for VMs to allow using it for VM to VM communications.
If there is Optee running in the secure world and using the non FF-A communication system, having CONFIG_FFA_VM_TO_VM could be non functional (if optee is probed first) or Optee could be non functional (if FF-A is probed first) so it is not recommended to activate the configuration option for such systems.

To make buffer full notification work between VMs when there is not firmware, rework the notification handling and modify the global flag to only be used as check for firmware notification support instead.

Modify part_info_get to return the list of VMs when there is no firmware support.

Quoted 6 lines

---

xen/arch/arm/tee/ffa.c          |  11 +++
xen/arch/arm/tee/ffa_notif.c    | 118 ++++++++++++++++----------------
xen/arch/arm/tee/ffa_partinfo.c |   2 +
3 files changed, 73 insertions(+), 58 deletions(-)

I think it is desirable or at least a good goal to be able to have all TEE configurations enabled at compile time.

For optee_probe() to succeed, a DT node with compatible "linaro,optee-tz" must be present, and a trusted OS responding with the UUID used by OP-TEE. False positives can be ruled out unless the system is grossly misconfigured and shouldn't be used.

Quoted 5 lines

If we could make the probe order deterministic with OP-TEE before FF-A, we should be OK. If there is an odd system with OP-TEE SMC ABI in the secure world that wants to use FF-A VM to VM, removing the "linaro,optee-tz" compatible node from DT is enough to disable optee_probe() without recompiling Xen.

I do agree with the deterministic argument but I am not sure having the order forced is the right solution.

Maybe we could use a command line argument so that one could select explicitly the tee:
tee=ffa / tee=optee

If we could prevent to modify the device tree that will probably make things easier.

What do you think ?

Cheers
Bertrand

Quoted 285 lines

Cheers,
Jens

Quoted 281 lines
xen/arch/arm/tee/ffa.c +11 -0
index 21d41b452dc9..6d427864f3da 100644
@@ -324,8 +324,11 @@ static int ffa_domain_init(struct domain *d)
    struct ffa_ctx *ctx;
    int ret;
 
+#ifndef CONFIG_FFA_VM_TO_VM
    if ( !ffa_fw_version )
        return -ENODEV;
+#endif
+
    /*
     * We are using the domain_id + 1 as the FF-A ID for VMs as FF-A ID 0 is
     * reserved for the hypervisor and we only support secure endpoints using
@@ -549,7 +552,15 @@ err_no_fw:
    bitmap_zero(ffa_fw_abi_supported, FFA_ABI_BITMAP_SIZE);
    printk(XENLOG_WARNING "ARM FF-A No firmware support\n");
 
+#ifdef CONFIG_FFA_VM_TO_VM
+    INIT_LIST_HEAD(&ffa_teardown_head);
+    init_timer(&ffa_teardown_timer, ffa_teardown_timer_callback, NULL, 0);
+
+    printk(XENLOG_INFO "ARM FF-A only available between VMs\n");
+    return true;
+#else
    return false;
+#endif
 }
 

static const struct tee_mediator_ops ffa_ops =

xen/arch/arm/tee/ffa_notif.c +60 -58
index 052b3e364a70..f2c87d1320de 100644
@@ -16,7 +16,7 @@
 
 #include "ffa_private.h"
 
-static bool __ro_after_init notif_enabled;
+static bool __ro_after_init fw_notif_enabled;
 static unsigned int __ro_after_init notif_sri_irq;
 
 int ffa_handle_notification_bind(struct cpu_user_regs *regs)
@@ -27,21 +27,17 @@ int ffa_handle_notification_bind(struct cpu_user_regs *regs)
    uint32_t bitmap_lo = get_user_reg(regs, 3);
    uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
    if ( (src_dst & 0xFFFFU) != ffa_get_vm_id(d) )
        return FFA_RET_INVALID_PARAMETERS;
 
    if ( flags )    /* Only global notifications are supported */
        return FFA_RET_DENIED;
 
-    /*
-     * We only support notifications from SP so no need to check the sender
-     * endpoint ID, the SPMC will take care of that for us.
-     */
-    return ffa_simple_call(FFA_NOTIFICATION_BIND, src_dst, flags, bitmap_hi,
-                           bitmap_lo);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )
+        return ffa_simple_call(FFA_NOTIFICATION_BIND, src_dst, flags,
+                               bitmap_hi, bitmap_lo);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 int ffa_handle_notification_unbind(struct cpu_user_regs *regs)
@@ -51,32 +47,36 @@ int ffa_handle_notification_unbind(struct cpu_user_regs *regs)
    uint32_t bitmap_lo = get_user_reg(regs, 3);
    uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
    if ( (src_dst & 0xFFFFU) != ffa_get_vm_id(d) )
        return FFA_RET_INVALID_PARAMETERS;
 
-    /*
-     * We only support notifications from SP so no need to check the
-     * destination endpoint ID, the SPMC will take care of that for us.
-     */
-    return  ffa_simple_call(FFA_NOTIFICATION_UNBIND, src_dst, 0, bitmap_hi,
-                            bitmap_lo);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )
+        return  ffa_simple_call(FFA_NOTIFICATION_UNBIND, src_dst, 0, bitmap_hi,
+                                bitmap_lo);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 void ffa_handle_notification_info_get(struct cpu_user_regs *regs)
 {
    struct domain *d = current->domain;
    struct ffa_ctx *ctx = d->arch.tee;
+    bool notif_pending = false;
 
-    if ( !notif_enabled )
+#ifndef CONFIG_FFA_VM_TO_VM
+    if ( !fw_notif_enabled )
    {
        ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
        return;
    }
+#endif
 
-    if ( test_and_clear_bool(ctx->notif.secure_pending) )
+    notif_pending = ctx->notif.secure_pending;
+#ifdef CONFIG_FFA_VM_TO_VM
+    notif_pending |= ctx->notif.buff_full_pending;
+#endif
+
+    if ( notif_pending )
    {
        /* A pending global notification for the guest */
        ffa_set_regs(regs, FFA_SUCCESS_64, 0,
@@ -103,11 +103,13 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
    uint32_t w6 = 0;
    uint32_t w7 = 0;
 
-    if ( !notif_enabled )
+#ifndef CONFIG_FFA_VM_TO_VM
+    if ( !fw_notif_enabled )
    {
        ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
        return;
    }
+#endif
 
    if ( (recv & 0xFFFFU) != ffa_get_vm_id(d) )
    {
@@ -115,7 +117,8 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
        return;
    }
 
-    if ( flags & ( FFA_NOTIF_FLAG_BITMAP_SP | FFA_NOTIF_FLAG_BITMAP_SPM ) )
+    if ( fw_notif_enabled && (flags & ( FFA_NOTIF_FLAG_BITMAP_SP
+                                        | FFA_NOTIF_FLAG_BITMAP_SPM )) )
    {
        struct arm_smccc_1_2_regs arg = {
            .a0 = FFA_NOTIFICATION_GET,
@@ -170,15 +173,14 @@ int ffa_handle_notification_set(struct cpu_user_regs *regs)
    uint32_t bitmap_lo = get_user_reg(regs, 3);
    uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
    if ( (src_dst >> 16) != ffa_get_vm_id(d) )
        return FFA_RET_INVALID_PARAMETERS;
 
-    /* Let the SPMC check the destination of the notification */
-    return ffa_simple_call(FFA_NOTIFICATION_SET, src_dst, flags, bitmap_lo,
-                           bitmap_hi);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )
+        return ffa_simple_call(FFA_NOTIFICATION_SET, src_dst, flags, bitmap_lo,
+                               bitmap_hi);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 #ifdef CONFIG_FFA_VM_TO_VM
@@ -190,7 +192,7 @@ void ffa_raise_rx_buffer_full(struct domain *d)
        return;
 
    if ( !test_and_set_bool(ctx->notif.buff_full_pending) )
-        vgic_inject_irq(d, d->vcpu[0], notif_sri_irq, true);
+        vgic_inject_irq(d, d->vcpu[0], GUEST_FFA_NOTIF_PEND_INTR_ID, true);
 }
 #endif
 
@@ -363,7 +365,7 @@ void ffa_notif_init_interrupt(void)
 {
    int ret;
 
-    if ( notif_enabled && notif_sri_irq < NR_GIC_SGI )
+    if ( fw_notif_enabled && notif_sri_irq < NR_GIC_SGI )
    {
        /*
         * An error here is unlikely since the primary CPU has already
@@ -394,47 +396,47 @@ void ffa_notif_init(void)
    int ret;
 
    /* Only enable fw notification if all ABIs we need are supported */
-    if ( !(ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_CREATE) &&
-           ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_DESTROY) &&
-           ffa_fw_supports_fid(FFA_NOTIFICATION_GET) &&
-           ffa_fw_supports_fid(FFA_NOTIFICATION_INFO_GET_64)) )
-        return;
-
-    arm_smccc_1_2_smc(&arg, &resp);
-    if ( resp.a0 != FFA_SUCCESS_32 )
-        return;
-
-    irq = resp.a2;
-    notif_sri_irq = irq;
-    if ( irq >= NR_GIC_SGI )
-        irq_set_type(irq, IRQ_TYPE_EDGE_RISING);
-    ret = request_irq(irq, 0, notif_irq_handler, "FF-A notif", NULL);
-    if ( ret )
+    if ( ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_CREATE) &&
+         ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_DESTROY) &&
+         ffa_fw_supports_fid(FFA_NOTIFICATION_GET) &&
+         ffa_fw_supports_fid(FFA_NOTIFICATION_INFO_GET_64) )
    {
-        printk(XENLOG_ERR "ffa: request_irq irq %u failed: error %d\n",
-               irq, ret);
-        return;
-    }
+        arm_smccc_1_2_smc(&arg, &resp);
+        if ( resp.a0 != FFA_SUCCESS_32 )
+            return;
 
-    notif_enabled = true;
+        irq = resp.a2;
+        notif_sri_irq = irq;
+        if ( irq >= NR_GIC_SGI )
+            irq_set_type(irq, IRQ_TYPE_EDGE_RISING);
+        ret = request_irq(irq, 0, notif_irq_handler, "FF-A notif", NULL);
+        if ( ret )
+        {
+            printk(XENLOG_ERR "ffa: request_irq irq %u failed: error %d\n",
+                   irq, ret);
+            return;
+        }
+        fw_notif_enabled = true;
+    }
 }
 
 int ffa_notif_domain_init(struct domain *d)
 {
    int32_t res;
 
-    if ( !notif_enabled )
-        return 0;
+    if ( fw_notif_enabled )
+    {
 
-    res = ffa_notification_bitmap_create(ffa_get_vm_id(d), d->max_vcpus);
-    if ( res )
-        return -ENOMEM;
+        res = ffa_notification_bitmap_create(ffa_get_vm_id(d), d->max_vcpus);
+        if ( res )
+            return -ENOMEM;
+    }
 
    return 0;
 }
 
 void ffa_notif_domain_destroy(struct domain *d)
 {
-    if ( notif_enabled )
+    if ( fw_notif_enabled )
        ffa_notification_bitmap_destroy(ffa_get_vm_id(d));
 }
xen/arch/arm/tee/ffa_partinfo.c +2 -1
index d699a267cc76..2e09440fe6c2 100644
@@ -128,12 +128,14 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
        goto out;
    }
 
+#ifndef CONFIG_FFA_VM_TO_VM
    if ( !ffa_fw_supports_fid(FFA_PARTITION_INFO_GET) )
    {
        /* Just give an empty partition list to the caller */
        ret = FFA_RET_OK;
        goto out;
    }
+#endif
 
    ret = ffa_rx_acquire(d);
    if ( ret != FFA_RET_OK )
--

2.47.0

#11 lore raw

Re: [RFC PATCH 4/4] xen/arm: ffa: Enable VM to VM without firmware

Hi Bertrand,

Quoted 41 lines · Bertrand Marquis

Hi Jens,

Quoted 38 lines · Jens Wiklander

Hi Bertrand,

Quoted 17 lines · Bertrand Marquis

When VM to VM support is activated and there is no suitable FF-A support in the firmware, enable FF-A support for VMs to allow using it for VM to VM communications.
If there is Optee running in the secure world and using the non FF-A communication system, having CONFIG_FFA_VM_TO_VM could be non functional (if optee is probed first) or Optee could be non functional (if FF-A is probed first) so it is not recommended to activate the configuration option for such systems.

To make buffer full notification work between VMs when there is not firmware, rework the notification handling and modify the global flag to only be used as check for firmware notification support instead.

Modify part_info_get to return the list of VMs when there is no firmware support.

Quoted 6 lines

---

xen/arch/arm/tee/ffa.c          |  11 +++
xen/arch/arm/tee/ffa_notif.c    | 118 ++++++++++++++++----------------
xen/arch/arm/tee/ffa_partinfo.c |   2 +
3 files changed, 73 insertions(+), 58 deletions(-)

I think it is desirable or at least a good goal to be able to have all TEE configurations enabled at compile time.

For optee_probe() to succeed, a DT node with compatible "linaro,optee-tz" must be present, and a trusted OS responding with the UUID used by OP-TEE. False positives can be ruled out unless the system is grossly misconfigured and shouldn't be used.

Quoted 8 lines
Quoted 5 lines

If we could make the probe order deterministic with OP-TEE before FF-A, we should be OK. If there is an odd system with OP-TEE SMC ABI in the secure world that wants to use FF-A VM to VM, removing the "linaro,optee-tz" compatible node from DT is enough to disable optee_probe() without recompiling Xen.

I do agree with the deterministic argument but I am not sure having the order forced is the right solution.

CONFIG_FFA_VM_TO_VM ensures that FF-A probing always succeeds and claims the TEE configuration. Logically, this should be last after all other probing has been tried since it disables further probing.

Quoted 9 lines

Maybe we could use a command line argument so that one could select explicitly the tee:
tee=ffa / tee=optee

If we could prevent to modify the device tree that will probably make things easier.

What do you think ?

That works for me.

Cheers,
Jens

Quoted 291 lines

Cheers
Bertrand

Quoted 285 lines

Cheers,
Jens

Quoted 281 lines
xen/arch/arm/tee/ffa.c +11 -0
index 21d41b452dc9..6d427864f3da 100644
@@ -324,8 +324,11 @@ static int ffa_domain_init(struct domain *d)
    struct ffa_ctx *ctx;
    int ret;
 
+#ifndef CONFIG_FFA_VM_TO_VM
    if ( !ffa_fw_version )
        return -ENODEV;
+#endif
+
    /*
     * We are using the domain_id + 1 as the FF-A ID for VMs as FF-A ID 0 is
     * reserved for the hypervisor and we only support secure endpoints using
@@ -549,7 +552,15 @@ err_no_fw:
    bitmap_zero(ffa_fw_abi_supported, FFA_ABI_BITMAP_SIZE);
    printk(XENLOG_WARNING "ARM FF-A No firmware support\n");
 
+#ifdef CONFIG_FFA_VM_TO_VM
+    INIT_LIST_HEAD(&ffa_teardown_head);
+    init_timer(&ffa_teardown_timer, ffa_teardown_timer_callback, NULL, 0);
+
+    printk(XENLOG_INFO "ARM FF-A only available between VMs\n");
+    return true;
+#else
    return false;
+#endif
 }
 

static const struct tee_mediator_ops ffa_ops =

xen/arch/arm/tee/ffa_notif.c +60 -58
index 052b3e364a70..f2c87d1320de 100644
@@ -16,7 +16,7 @@
 
 #include "ffa_private.h"
 
-static bool __ro_after_init notif_enabled;
+static bool __ro_after_init fw_notif_enabled;
 static unsigned int __ro_after_init notif_sri_irq;
 
 int ffa_handle_notification_bind(struct cpu_user_regs *regs)
@@ -27,21 +27,17 @@ int ffa_handle_notification_bind(struct cpu_user_regs *regs)
    uint32_t bitmap_lo = get_user_reg(regs, 3);
    uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
    if ( (src_dst & 0xFFFFU) != ffa_get_vm_id(d) )
        return FFA_RET_INVALID_PARAMETERS;
 
    if ( flags )    /* Only global notifications are supported */
        return FFA_RET_DENIED;
 
-    /*
-     * We only support notifications from SP so no need to check the sender
-     * endpoint ID, the SPMC will take care of that for us.
-     */
-    return ffa_simple_call(FFA_NOTIFICATION_BIND, src_dst, flags, bitmap_hi,
-                           bitmap_lo);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )
+        return ffa_simple_call(FFA_NOTIFICATION_BIND, src_dst, flags,
+                               bitmap_hi, bitmap_lo);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 int ffa_handle_notification_unbind(struct cpu_user_regs *regs)
@@ -51,32 +47,36 @@ int ffa_handle_notification_unbind(struct cpu_user_regs *regs)
    uint32_t bitmap_lo = get_user_reg(regs, 3);
    uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
    if ( (src_dst & 0xFFFFU) != ffa_get_vm_id(d) )
        return FFA_RET_INVALID_PARAMETERS;
 
-    /*
-     * We only support notifications from SP so no need to check the
-     * destination endpoint ID, the SPMC will take care of that for us.
-     */
-    return  ffa_simple_call(FFA_NOTIFICATION_UNBIND, src_dst, 0, bitmap_hi,
-                            bitmap_lo);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )
+        return  ffa_simple_call(FFA_NOTIFICATION_UNBIND, src_dst, 0, bitmap_hi,
+                                bitmap_lo);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 void ffa_handle_notification_info_get(struct cpu_user_regs *regs)
 {
    struct domain *d = current->domain;
    struct ffa_ctx *ctx = d->arch.tee;
+    bool notif_pending = false;
 
-    if ( !notif_enabled )
+#ifndef CONFIG_FFA_VM_TO_VM
+    if ( !fw_notif_enabled )
    {
        ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
        return;
    }
+#endif
 
-    if ( test_and_clear_bool(ctx->notif.secure_pending) )
+    notif_pending = ctx->notif.secure_pending;
+#ifdef CONFIG_FFA_VM_TO_VM
+    notif_pending |= ctx->notif.buff_full_pending;
+#endif
+
+    if ( notif_pending )
    {
        /* A pending global notification for the guest */
        ffa_set_regs(regs, FFA_SUCCESS_64, 0,
@@ -103,11 +103,13 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
    uint32_t w6 = 0;
    uint32_t w7 = 0;
 
-    if ( !notif_enabled )
+#ifndef CONFIG_FFA_VM_TO_VM
+    if ( !fw_notif_enabled )
    {
        ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
        return;
    }
+#endif
 
    if ( (recv & 0xFFFFU) != ffa_get_vm_id(d) )
    {
@@ -115,7 +117,8 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
        return;
    }
 
-    if ( flags & ( FFA_NOTIF_FLAG_BITMAP_SP | FFA_NOTIF_FLAG_BITMAP_SPM ) )
+    if ( fw_notif_enabled && (flags & ( FFA_NOTIF_FLAG_BITMAP_SP
+                                        | FFA_NOTIF_FLAG_BITMAP_SPM )) )
    {
        struct arm_smccc_1_2_regs arg = {
            .a0 = FFA_NOTIFICATION_GET,
@@ -170,15 +173,14 @@ int ffa_handle_notification_set(struct cpu_user_regs *regs)
    uint32_t bitmap_lo = get_user_reg(regs, 3);
    uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
    if ( (src_dst >> 16) != ffa_get_vm_id(d) )
        return FFA_RET_INVALID_PARAMETERS;
 
-    /* Let the SPMC check the destination of the notification */
-    return ffa_simple_call(FFA_NOTIFICATION_SET, src_dst, flags, bitmap_lo,
-                           bitmap_hi);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )
+        return ffa_simple_call(FFA_NOTIFICATION_SET, src_dst, flags, bitmap_lo,
+                               bitmap_hi);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 #ifdef CONFIG_FFA_VM_TO_VM
@@ -190,7 +192,7 @@ void ffa_raise_rx_buffer_full(struct domain *d)
        return;
 
    if ( !test_and_set_bool(ctx->notif.buff_full_pending) )
-        vgic_inject_irq(d, d->vcpu[0], notif_sri_irq, true);
+        vgic_inject_irq(d, d->vcpu[0], GUEST_FFA_NOTIF_PEND_INTR_ID, true);
 }
 #endif
 
@@ -363,7 +365,7 @@ void ffa_notif_init_interrupt(void)
 {
    int ret;
 
-    if ( notif_enabled && notif_sri_irq < NR_GIC_SGI )
+    if ( fw_notif_enabled && notif_sri_irq < NR_GIC_SGI )
    {
        /*
         * An error here is unlikely since the primary CPU has already
@@ -394,47 +396,47 @@ void ffa_notif_init(void)
    int ret;
 
    /* Only enable fw notification if all ABIs we need are supported */
-    if ( !(ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_CREATE) &&
-           ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_DESTROY) &&
-           ffa_fw_supports_fid(FFA_NOTIFICATION_GET) &&
-           ffa_fw_supports_fid(FFA_NOTIFICATION_INFO_GET_64)) )
-        return;
-
-    arm_smccc_1_2_smc(&arg, &resp);
-    if ( resp.a0 != FFA_SUCCESS_32 )
-        return;
-
-    irq = resp.a2;
-    notif_sri_irq = irq;
-    if ( irq >= NR_GIC_SGI )
-        irq_set_type(irq, IRQ_TYPE_EDGE_RISING);
-    ret = request_irq(irq, 0, notif_irq_handler, "FF-A notif", NULL);
-    if ( ret )
+    if ( ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_CREATE) &&
+         ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_DESTROY) &&
+         ffa_fw_supports_fid(FFA_NOTIFICATION_GET) &&
+         ffa_fw_supports_fid(FFA_NOTIFICATION_INFO_GET_64) )
    {
-        printk(XENLOG_ERR "ffa: request_irq irq %u failed: error %d\n",
-               irq, ret);
-        return;
-    }
+        arm_smccc_1_2_smc(&arg, &resp);
+        if ( resp.a0 != FFA_SUCCESS_32 )
+            return;
 
-    notif_enabled = true;
+        irq = resp.a2;
+        notif_sri_irq = irq;
+        if ( irq >= NR_GIC_SGI )
+            irq_set_type(irq, IRQ_TYPE_EDGE_RISING);
+        ret = request_irq(irq, 0, notif_irq_handler, "FF-A notif", NULL);
+        if ( ret )
+        {
+            printk(XENLOG_ERR "ffa: request_irq irq %u failed: error %d\n",
+                   irq, ret);
+            return;
+        }
+        fw_notif_enabled = true;
+    }
 }
 
 int ffa_notif_domain_init(struct domain *d)
 {
    int32_t res;
 
-    if ( !notif_enabled )
-        return 0;
+    if ( fw_notif_enabled )
+    {
 
-    res = ffa_notification_bitmap_create(ffa_get_vm_id(d), d->max_vcpus);
-    if ( res )
-        return -ENOMEM;
+        res = ffa_notification_bitmap_create(ffa_get_vm_id(d), d->max_vcpus);
+        if ( res )
+            return -ENOMEM;
+    }
 
    return 0;
 }
 
 void ffa_notif_domain_destroy(struct domain *d)
 {
-    if ( notif_enabled )
+    if ( fw_notif_enabled )
        ffa_notification_bitmap_destroy(ffa_get_vm_id(d));
 }
xen/arch/arm/tee/ffa_partinfo.c +2 -1
index d699a267cc76..2e09440fe6c2 100644
@@ -128,12 +128,14 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
        goto out;
    }
 
+#ifndef CONFIG_FFA_VM_TO_VM
    if ( !ffa_fw_supports_fid(FFA_PARTITION_INFO_GET) )
    {
        /* Just give an empty partition list to the caller */
        ret = FFA_RET_OK;
        goto out;
    }
+#endif
 
    ret = ffa_rx_acquire(d);
    if ( ret != FFA_RET_OK )
--

2.47.0