Skip to contentSkip to conversation

FF-A VM to VM support

PATCH Superseded Bertrand Marquis 3 participants 22 messages Mar 10, 2025 – Mar 21, 2025

v2 of 8 Superseded Latest is v8 →

lore
Share & tools
lore
#0 lore raw

v2 cover FF-A VM to VM support

This patch serie adds support to use FF-A between VM for communications using indirect messages.

It adds a Kconfig parameter to enable this feature and marks it as experimental as for now there is no system to restrict communication rights between VM using this system.

It also adds support to use VM to VM communications using FF-A even if there is no firmware support for FF-A. As this feature enables FF-A support in all cases, we also introduce a new command line parameter to allow the user to force which tee is to be used between FF-A and Optee to have a solution to enable optee support if FF-A VM to VM is enabled.

Changes since v1 (rfc):
- add a tee command line parameter
- use IS_ENABLED instead of ifdef when possible
- rebase on latest staging

Bertrand Marquis (5):
xen/arm: Create tee command line parameter xen/arm: ffa: Introduce VM to VM support xen/arm: ffa: Add buffer full notification support xen/arm: ffa: Add indirect message between VM xen/arm: ffa: Enable VM to VM without firmware

 docs/misc/xen-command-line.pandoc  |  14 +++
 xen/arch/arm/include/asm/tee/tee.h |   4 +
 xen/arch/arm/tee/Kconfig           |  11 +++
 xen/arch/arm/tee/ffa.c             |  12 ++-
 xen/arch/arm/tee/ffa_msg.c         |  96 +++++++++++++++++--
 xen/arch/arm/tee/ffa_notif.c       | 138 +++++++++++++++------------
 xen/arch/arm/tee/ffa_partinfo.c    | 147 +++++++++++++++++++++++------
 xen/arch/arm/tee/ffa_private.h     |  29 ++++++
 xen/arch/arm/tee/tee.c             |  31 ++++++
 9 files changed, 380 insertions(+), 102 deletions(-)
Signature · 1 line
2.47.1
#1 lore raw

PATCH v2 1/5 xen/arm: Create tee command line parameter

Add a new command line parameter "tee=" to be used to explicitly select what tee mediator is to be used by Xen and fail if it does not exist or the probe function for it failed.

Without specifying which tee is to be used, Xen will use the first one for which the probe function succeeds which depends on the order of the mediator list which depends on the compiler.
Using the command line argument, it is now possible to explicit request a specific TEE mediator and panic on boot if it is not available.

---
Changes in v2:
- Patch introduced to add a command line selection of the TEE

---
 docs/misc/xen-command-line.pandoc  | 14 ++++++++++++++
 xen/arch/arm/include/asm/tee/tee.h |  4 ++++
 xen/arch/arm/tee/tee.c             | 31 ++++++++++++++++++++++++++++++
 3 files changed, 49 insertions(+)
docs/misc/xen-command-line.pandoc +14 -0
index 89db6e83be66..8464d7127890 100644
@@ -2651,6 +2651,20 @@ Specify the per-cpu trace buffer size in pages.
 
 Flag to enable TSC deadline as the APIC timer mode.
 
+### tee
+> `= <string>`
+
+Specify the TEE mediator to be probed and use.
+
+The default behaviour is to probe all supported TEEs supported by Xen and use
+the first one successfully probed. When this parameter is passed, Xen will
+probe only the TEE mediator passed as argument and boot will fail if this
+mediator is not properly probed or if the requested TEE is not supported by
+Xen.
+
+This parameter can be set to `optee` of `ffa` if the corresponding mediators
+are compiled in.
+
 ### tevt_mask
 > `= <integer>`
 
xen/arch/arm/include/asm/tee/tee.h +4 -0
index 0169fd746bcd..15d664e28dce 100644
@@ -55,6 +55,9 @@ struct tee_mediator_desc {
     /* Printable name of the TEE. */
     const char *name;
 
+    /* Command line name of the TEE (to be used with tee= cmdline option) */
+    const char *cmdline_name;
+
     /* Mediator callbacks as described above. */
     const struct tee_mediator_ops *ops;
 
@@ -77,6 +80,7 @@ void tee_free_domain_ctx(struct domain *d);
 static const struct tee_mediator_desc __tee_desc_##_name __used     \
 __section(".teemediator.info") = {                                  \
     .name = _namestr,                                               \
+    .cmdline_name = #_name,                                         \
     .ops = _ops,                                                    \
     .tee_type = _type                                               \
 }
xen/arch/arm/tee/tee.c +31 -0
index 3f65e45a7892..066b5ba40f9d 100644
@@ -19,12 +19,17 @@
 #include <xen/errno.h>
 #include <xen/init.h>
 #include <xen/types.h>
+#include <xen/param.h>
 
 #include <asm/tee/tee.h>
 
 extern const struct tee_mediator_desc _steemediator[], _eteemediator[];
 static const struct tee_mediator_desc __read_mostly *cur_mediator;
 
+/* Select the TEE mediator using a name on command line. */
+static char __initdata opt_mediator[16] = "";
+string_param("tee", opt_mediator);
+
 /*
  * TODO: Add function to alter Dom0 DTB, so we can properly describe
  * present TEE.
@@ -81,14 +86,40 @@ static int __init tee_init(void)
 {
     const struct tee_mediator_desc *desc;
 
+    if ( strcmp(opt_mediator, "") )
+        printk(XENLOG_INFO "TEE Mediator %s selected from command line\n",
+               opt_mediator);
+
+    /*
+     * When a specific TEE is selected using the 'tee=' command line
+     * argument, we panic if the probe fails or if the requested TEE is not
+     * supported.
+     */
     for ( desc = _steemediator; desc != _eteemediator; desc++ )
     {
+        if ( strcmp(opt_mediator, "") &&
+             strncmp(opt_mediator, desc->cmdline_name, sizeof(opt_mediator)) )
+            continue;
+
         if ( desc->ops->probe() )
         {
             printk(XENLOG_INFO "Using TEE mediator for %s\n", desc->name);
             cur_mediator = desc;
             return 0;
         }
+        else if ( strcmp(opt_mediator, "") )
+        {
+            panic("TEE mediator %s from command line probe failed\n",
+                  opt_mediator);
+            return -EFAULT;
+        }
+    }
+
+    if ( strcmp(opt_mediator, "") )
+    {
+        panic("TEE Mediator %s from command line not supported\n",
+              opt_mediator);
+        return -EINVAL;
     }
 
     return 0;
Signature · 1 line
2.47.1
#2 lore raw

Re: [PATCH v2 1/5] xen/arm: Create tee command line parameter

docs/misc/xen-command-line.pandoc Specify the per-cpu trace buffer size in pages. · line 2655

Quoted 8 lines · Bertrand Marquis
docs/misc/xen-command-line.pandoc +2 -0
@@ -2651,6 +2651,20 @@ Specify the per-cpu trace buffer size in pages.
 
 Flag to enable TSC deadline as the APIC timer mode.
 
+### tee
+> `= <string>`

This wants an arch restriction, like we have for other command line options supported only by one arch.

Jan

Quoted 14 lines
docs/misc/xen-command-line.pandoc +11 -0
+Specify the TEE mediator to be probed and use.
+
+The default behaviour is to probe all supported TEEs supported by Xen and use
+the first one successfully probed. When this parameter is passed, Xen will
+probe only the TEE mediator passed as argument and boot will fail if this
+mediator is not properly probed or if the requested TEE is not supported by
+Xen.
+
+This parameter can be set to `optee` of `ffa` if the corresponding mediators
+are compiled in.
+
 ### tevt_mask
 > `= <integer>`
 
#3 lore raw

Re: [PATCH v2 1/5] xen/arm: Create tee command line parameter

Hi Jan,

Quoted 7 lines · Jan Beulich
Quoted 4 lines · Bertrand Marquis
docs/misc/xen-command-line.pandoc +0 -0
@@ -2651,6 +2651,20 @@ Specify the per-cpu trace buffer size in pages.
 
Quoted 7 lines
Quoted 4 lines

Flag to enable TSC deadline as the APIC timer mode.

+### tee
+> = <string>

This wants an arch restriction, like we have for other command line options supported only by one arch.

Oh yes definitely.
I will fix that in the next version.

Thanks
Bertrand

Quoted 18 lines

Jan

Quoted 14 lines
docs/misc/xen-command-line.pandoc +11 -0
+Specify the TEE mediator to be probed and use.
+
+The default behaviour is to probe all supported TEEs supported by Xen and use
+the first one successfully probed. When this parameter is passed, Xen will
+probe only the TEE mediator passed as argument and boot will fail if this
+mediator is not properly probed or if the requested TEE is not supported by
+Xen.
+
+This parameter can be set to `optee` of `ffa` if the corresponding mediators
+are compiled in.
+

### tevt_mask

Quoted 1 line

= <integer>

#4 lore raw

PATCH v2 2/5 xen/arm: ffa: Introduce VM to VM support

In reply to Bertrand Marquis

Create a CONFIG_FFA_VM_TO_VM parameter to activate FFA communication between VMs.
When activated list VMs in the system with FF-A support in part_info_get.

WARNING: There is no filtering for now and all VMs are listed !!

---
Changes in v2:
- Switch ifdef to IS_ENABLED
- dom was not switched to d as requested by Jan because there is already a variable d pointing to the current domain and it must not be shadowed.

---
 xen/arch/arm/tee/Kconfig        |  11 +++
 xen/arch/arm/tee/ffa_partinfo.c | 144 +++++++++++++++++++++++++-------
 xen/arch/arm/tee/ffa_private.h  |  12 +++
 3 files changed, 137 insertions(+), 30 deletions(-)
xen/arch/arm/tee/Kconfig +11 -0
index c5b0f88d7522..88a4c4c99154 100644
@@ -28,5 +28,16 @@ config FFA
 
 	  [1] https://developer.arm.com/documentation/den0077/latest
 
+config FFA_VM_TO_VM
+    bool "Enable FF-A between VMs (UNSUPPORTED)" if UNSUPPORTED
+    default n
+    depends on FFA
+    help
+      This option enables to use FF-A between VMs.
+      This is experimental and there is no access control so any
+      guest can communicate with any other guest.
+
+      If unsure, say N.
+
 endmenu
 
xen/arch/arm/tee/ffa_partinfo.c +114 -30

179 lines not on this page · raw message

xen/arch/arm/tee/ffa_private.h +12 -0
index c4cd65538908..bd6877d8c632 100644
@@ -187,6 +187,18 @@
  */
 #define FFA_PARTITION_INFO_GET_COUNT_FLAG BIT(0, U)
 
+/*
+ * Partition properties we give for a normal world VM:
+ * - can send direct message but not receive them
+ * - can handle indirect messages
+ * - can receive notifications
+ * 32/64 bit flag is set depending on the VM
+ */
+#define FFA_PART_VM_PROP    (FFA_PART_PROP_DIRECT_REQ_SEND | \
+                             FFA_PART_PROP_INDIRECT_MSGS | \
+                             FFA_PART_PROP_RECV_NOTIF | \
+                             FFA_PART_PROP_IS_PE_ID)
+
 /* Flags used in calls to FFA_NOTIFICATION_GET interface  */
 #define FFA_NOTIF_FLAG_BITMAP_SP        BIT(0, U)
 #define FFA_NOTIF_FLAG_BITMAP_VM        BIT(1, U)
Signature · 1 line
2.47.1
#5 lore raw

Re: [PATCH v2 2/5] xen/arm: ffa: Introduce VM to VM support

Hi Bertrand,

Quoted 46 lines · Bertrand Marquis

Create a CONFIG_FFA_VM_TO_VM parameter to activate FFA communication between VMs.
When activated list VMs in the system with FF-A support in part_info_get.

WARNING: There is no filtering for now and all VMs are listed !!

---
Changes in v2:
- Switch ifdef to IS_ENABLED
- dom was not switched to d as requested by Jan because there is already a variable d pointing to the current domain and it must not be shadowed.

---
 xen/arch/arm/tee/Kconfig        |  11 +++
 xen/arch/arm/tee/ffa_partinfo.c | 144 +++++++++++++++++++++++++-------
 xen/arch/arm/tee/ffa_private.h  |  12 +++
 3 files changed, 137 insertions(+), 30 deletions(-)
xen/arch/arm/tee/Kconfig +11 -0
index c5b0f88d7522..88a4c4c99154 100644
@@ -28,5 +28,16 @@ config FFA
 
          [1] https://developer.arm.com/documentation/den0077/latest
 
+config FFA_VM_TO_VM
+    bool "Enable FF-A between VMs (UNSUPPORTED)" if UNSUPPORTED
+    default n
+    depends on FFA
+    help
+      This option enables to use FF-A between VMs.
+      This is experimental and there is no access control so any
+      guest can communicate with any other guest.
+
+      If unsure, say N.
+
 endmenu
 
xen/arch/arm/tee/ffa_partinfo.c +0 -0
index c0510ceb8338..7af1eca2d0c4 100644

xen/arch/arm/tee/ffa_partinfo.c void ffa_handle_partition_info_get(struct cpu_user_regs *regs) · line 81

Quoted 6 lines
xen/arch/arm/tee/ffa_partinfo.c +2 -1
@@ -77,7 +77,23 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
     };
     uint32_t src_size, dst_size;
     void *dst_buf;
-    uint32_t ffa_sp_count = 0;
+    uint32_t ffa_vm_count = 0, ffa_sp_count = 0;
+

This function is getting quite large and it's hard to follow the different lock states. How about splitting it into various helper functions?

xen/arch/arm/tee/ffa_partinfo.c void ffa_handle_partition_info_get(struct cpu_user_regs *regs) · line 86

Quoted 5 lines
xen/arch/arm/tee/ffa_partinfo.c +5 -0
+    if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )
+    {
+        struct domain *dom;
+
+        /* Count the number of VM with FF-A support */

Why do we need this now? Isn't it enough to count them below when we fill in dst_buf?

Quoted 102 lines
xen/arch/arm/tee/ffa_partinfo.c +61 -19
+        rcu_read_lock(&domlist_read_lock);
+        for_each_domain( dom )
+        {
+            struct ffa_ctx *vm = dom->arch.tee;
+
+            if (dom != d && vm != NULL && vm->guest_vers != 0)
+                ffa_vm_count++;
+        }
+        rcu_read_unlock(&domlist_read_lock);
+    }
 
     /*
      * If the guest is v1.0, he does not get back the entry size so we must
@@ -127,33 +143,38 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
 
     dst_buf = ctx->rx;
 
-    if ( !ffa_rx )
+    /* If not supported, we have ffa_sp_count=0 */
+    if ( ffa_fw_supports_fid(FFA_PARTITION_INFO_GET) )
     {
-        ret = FFA_RET_DENIED;
-        goto out_rx_release;
-    }
+        if ( !ffa_rx )
+        {
+            ret = FFA_RET_DENIED;
+            goto out_rx_release;
+        }
 
-    spin_lock(&ffa_rx_buffer_lock);
+        spin_lock(&ffa_rx_buffer_lock);
 
-    ret = ffa_partition_info_get(uuid, 0, &ffa_sp_count, &src_size);
+        ret = ffa_partition_info_get(uuid, 0, &ffa_sp_count, &src_size);
 
-    if ( ret )
-        goto out_rx_hyp_unlock;
+        if ( ret )
+            goto out_rx_hyp_unlock;
 
-    /*
-     * ffa_partition_info_get() succeeded so we now own the RX buffer we
-     * share with the SPMC. We must give it back using ffa_hyp_rx_release()
-     * once we've copied the content.
-     */
+        /*
+         * ffa_partition_info_get() succeeded so we now own the RX buffer we
+         * share with the SPMC. We must give it back using ffa_hyp_rx_release()
+         * once we've copied the content.
+         */
 
-    /* we cannot have a size smaller than 1.0 structure */
-    if ( src_size < sizeof(struct ffa_partition_info_1_0) )
-    {
-        ret = FFA_RET_NOT_SUPPORTED;
-        goto out_rx_hyp_release;
+        /* we cannot have a size smaller than 1.0 structure */
+        if ( src_size < sizeof(struct ffa_partition_info_1_0) )
+        {
+            ret = FFA_RET_NOT_SUPPORTED;
+            goto out_rx_hyp_release;
+        }
     }
 
-    if ( ctx->page_count * FFA_PAGE_SIZE < ffa_sp_count * dst_size )
+    if ( ctx->page_count * FFA_PAGE_SIZE <
+         (ffa_sp_count + ffa_vm_count) * dst_size )
     {
         ret = FFA_RET_NO_MEMORY;
         goto out_rx_hyp_release;
@@ -182,25 +203,88 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
         }
     }
 
+    if ( ffa_fw_supports_fid(FFA_PARTITION_INFO_GET) )
+    {
+        ffa_hyp_rx_release();
+        spin_unlock(&ffa_rx_buffer_lock);
+    }
+
+    if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) && ffa_vm_count )
+    {
+        struct domain *dom;
+        uint32_t curr = 0;
+
+        /* add the VM informations if any */
+        rcu_read_lock(&domlist_read_lock);
+        for_each_domain( dom )
+        {
+            struct ffa_ctx *vm = dom->arch.tee;
+
+            /*
+             * we do not add the VM calling to the list and only VMs with
+             * FF-A support
+             */
+            if (dom != d && vm != NULL && vm->guest_vers != 0)
+            {
+                /*
+                 * We do not have UUID info for VMs so use
+                 * the 1.0 structure so that we set UUIDs to
+                 * zero using memset

xen/arch/arm/tee/ffa_partinfo.c void ffa_handle_partition_info_get(struct cpu_user_regs *regs) · line 242

Quoted 10 lines
xen/arch/arm/tee/ffa_partinfo.c +10 -0
+                 */
+                struct ffa_partition_info_1_0 srcvm;
+
+                if ( curr == ffa_vm_count )
+                {
+                    /*
+                     * The number of domains changed since we counted them, we
+                     * can add new ones if there is enough space in the
+                     * destination buffer so check it or go out with an error.
+                     */

Why do we care if the number has changed? If it fits, all is good anyway and we're also updating ffa_vm_count with curr after the loop.

Cheers,
Jens

Quoted 84 lines
xen/arch/arm/tee/ffa_partinfo.c +36 -10
+                    ffa_vm_count++;
+                    if ( ctx->page_count * FFA_PAGE_SIZE <
+                         (ffa_sp_count + ffa_vm_count) * dst_size )
+                    {
+                        ret = FFA_RET_NO_MEMORY;
+                        rcu_read_unlock(&domlist_read_lock);
+                        goto out_rx_release;
+                    }
+                }
+
+                srcvm.id = ffa_get_vm_id(dom);
+                srcvm.execution_context = dom->max_vcpus;
+                srcvm.partition_properties = FFA_PART_VM_PROP;
+                if ( is_64bit_domain(dom) )
+                    srcvm.partition_properties |= FFA_PART_PROP_AARCH64_STATE;
+
+                memcpy(dst_buf, &srcvm, MIN(sizeof(srcvm), dst_size));
+
+                if ( dst_size > sizeof(srcvm) )
+                    memset(dst_buf + sizeof(srcvm), 0,
+                           dst_size - sizeof(srcvm));
+
+                dst_buf += dst_size;
+                curr++;
+            }
+        }
+        rcu_read_unlock(&domlist_read_lock);
+
+        /* the number of domains could have reduce since the initial count */
+        ffa_vm_count = curr;
+    }
+
+    goto out;
+
 out_rx_hyp_release:
     ffa_hyp_rx_release();
 out_rx_hyp_unlock:
     spin_unlock(&ffa_rx_buffer_lock);
 out_rx_release:
-    /*
-     * The calling VM RX buffer only contains data to be used by the VM if the
-     * call was successful, in which case the VM has to release the buffer
-     * once it has used the data.
-     * If something went wrong during the call, we have to release the RX
-     * buffer back to the SPMC as the VM will not do it.
-     */
-    if ( ret != FFA_RET_OK )
-        ffa_rx_release(d);
+    ffa_rx_release(d);
 out:
     if ( ret )
         ffa_set_regs_error(regs, ret);
     else
-        ffa_set_regs_success(regs, ffa_sp_count, dst_size);
+        ffa_set_regs_success(regs, ffa_sp_count + ffa_vm_count, dst_size);
 }
 
 static int32_t ffa_direct_req_send_vm(uint16_t sp_id, uint16_t vm_id,
xen/arch/arm/tee/ffa_private.h +12 -1
index c4cd65538908..bd6877d8c632 100644
@@ -187,6 +187,18 @@
  */
 #define FFA_PARTITION_INFO_GET_COUNT_FLAG BIT(0, U)
 
+/*
+ * Partition properties we give for a normal world VM:
+ * - can send direct message but not receive them
+ * - can handle indirect messages
+ * - can receive notifications
+ * 32/64 bit flag is set depending on the VM
+ */
+#define FFA_PART_VM_PROP    (FFA_PART_PROP_DIRECT_REQ_SEND | \
+                             FFA_PART_PROP_INDIRECT_MSGS | \
+                             FFA_PART_PROP_RECV_NOTIF | \
+                             FFA_PART_PROP_IS_PE_ID)
+
 /* Flags used in calls to FFA_NOTIFICATION_GET interface  */
 #define FFA_NOTIF_FLAG_BITMAP_SP        BIT(0, U)
 #define FFA_NOTIF_FLAG_BITMAP_VM        BIT(1, U)
--

2.47.1

#6 lore raw

Re: [PATCH v2 2/5] xen/arm: ffa: Introduce VM to VM support

Hi Jens,

Thanks a lot for the review.

Quoted 47 lines · Jens Wiklander

Hi Bertrand,

Quoted 41 lines · Bertrand Marquis

Create a CONFIG_FFA_VM_TO_VM parameter to activate FFA communication between VMs.
When activated list VMs in the system with FF-A support in part_info_get.

WARNING: There is no filtering for now and all VMs are listed !!

---
Changes in v2:
- Switch ifdef to IS_ENABLED
- dom was not switched to d as requested by Jan because there is already a variable d pointing to the current domain and it must not be shadowed.
---

xen/arch/arm/tee/Kconfig        |  11 +++
xen/arch/arm/tee/ffa_partinfo.c | 144 +++++++++++++++++++++++++-------
xen/arch/arm/tee/ffa_private.h  |  12 +++
3 files changed, 137 insertions(+), 30 deletions(-)
xen/arch/arm/tee/Kconfig +11 -0
index c5b0f88d7522..88a4c4c99154 100644
@@ -28,5 +28,16 @@ config FFA
 
         [1] https://developer.arm.com/documentation/den0077/latest
 
+config FFA_VM_TO_VM
+    bool "Enable FF-A between VMs (UNSUPPORTED)" if UNSUPPORTED
+    default n
+    depends on FFA
+    help
+      This option enables to use FF-A between VMs.
+      This is experimental and there is no access control so any
+      guest can communicate with any other guest.
+
+      If unsure, say N.
+

endmenu

Quoted 15 lines

xen/arch/arm/tee/ffa_partinfo.c void ffa_handle_partition_info_get(struct cpu_user_regs *regs) · line 81

Quoted 11 lines
xen/arch/arm/tee/ffa_partinfo.c +2 -1
index c0510ceb8338..7af1eca2d0c4 100644
@@ -77,7 +77,23 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
    };
    uint32_t src_size, dst_size;
    void *dst_buf;
-    uint32_t ffa_sp_count = 0;
+    uint32_t ffa_vm_count = 0, ffa_sp_count = 0;
+

This function is getting quite large and it's hard to follow the different lock states. How about splitting it into various helper functions?

Yes I agree.
I will try to find a good way to split this in smaller chunks.

Quoted 9 lines

xen/arch/arm/tee/ffa_partinfo.c void ffa_handle_partition_info_get(struct cpu_user_regs *regs) · line 86

Quoted 6 lines
xen/arch/arm/tee/ffa_partinfo.c +5 -0
+    if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )
+    {
+        struct domain *dom;
+
+        /* Count the number of VM with FF-A support */

Why do we need this now? Isn't it enough to count them below when we fill in dst_buf?

We need it in 3 places in the code after:
- to return the number of endpoint in case the flag is set (see final return)
- to check that there is enough space in the RX buffer
- to prevent going through the list of domains if none is to be listed anyway

Quoted 108 lines
Quoted 108 lines
xen/arch/arm/tee/ffa_partinfo.c +66 -19
+        rcu_read_lock(&domlist_read_lock);
+        for_each_domain( dom )
+        {
+            struct ffa_ctx *vm = dom->arch.tee;
+
+            if (dom != d && vm != NULL && vm->guest_vers != 0)
+                ffa_vm_count++;
+        }
+        rcu_read_unlock(&domlist_read_lock);
+    }
 
    /*
     * If the guest is v1.0, he does not get back the entry size so we must
@@ -127,33 +143,38 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
 
    dst_buf = ctx->rx;
 
-    if ( !ffa_rx )
+    /* If not supported, we have ffa_sp_count=0 */
+    if ( ffa_fw_supports_fid(FFA_PARTITION_INFO_GET) )
    {
-        ret = FFA_RET_DENIED;
-        goto out_rx_release;
-    }
+        if ( !ffa_rx )
+        {
+            ret = FFA_RET_DENIED;
+            goto out_rx_release;
+        }
 
-    spin_lock(&ffa_rx_buffer_lock);
+        spin_lock(&ffa_rx_buffer_lock);
 
-    ret = ffa_partition_info_get(uuid, 0, &ffa_sp_count, &src_size);
+        ret = ffa_partition_info_get(uuid, 0, &ffa_sp_count, &src_size);
 
-    if ( ret )
-        goto out_rx_hyp_unlock;
+        if ( ret )
+            goto out_rx_hyp_unlock;
 
-    /*
-     * ffa_partition_info_get() succeeded so we now own the RX buffer we
-     * share with the SPMC. We must give it back using ffa_hyp_rx_release()
-     * once we've copied the content.
-     */
+        /*
+         * ffa_partition_info_get() succeeded so we now own the RX buffer we
+         * share with the SPMC. We must give it back using ffa_hyp_rx_release()
+         * once we've copied the content.
+         */
 
-    /* we cannot have a size smaller than 1.0 structure */
-    if ( src_size < sizeof(struct ffa_partition_info_1_0) )
-    {
-        ret = FFA_RET_NOT_SUPPORTED;
-        goto out_rx_hyp_release;
+        /* we cannot have a size smaller than 1.0 structure */
+        if ( src_size < sizeof(struct ffa_partition_info_1_0) )
+        {
+            ret = FFA_RET_NOT_SUPPORTED;
+            goto out_rx_hyp_release;
+        }
    }
 
-    if ( ctx->page_count * FFA_PAGE_SIZE < ffa_sp_count * dst_size )
+    if ( ctx->page_count * FFA_PAGE_SIZE <
+         (ffa_sp_count + ffa_vm_count) * dst_size )
    {
        ret = FFA_RET_NO_MEMORY;
        goto out_rx_hyp_release;
@@ -182,25 +203,88 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
        }
    }
 
+    if ( ffa_fw_supports_fid(FFA_PARTITION_INFO_GET) )
+    {
+        ffa_hyp_rx_release();
+        spin_unlock(&ffa_rx_buffer_lock);
+    }
+
+    if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) && ffa_vm_count )
+    {
+        struct domain *dom;
+        uint32_t curr = 0;
+
+        /* add the VM informations if any */
+        rcu_read_lock(&domlist_read_lock);
+        for_each_domain( dom )
+        {
+            struct ffa_ctx *vm = dom->arch.tee;
+
+            /*
+             * we do not add the VM calling to the list and only VMs with
+             * FF-A support
+             */
+            if (dom != d && vm != NULL && vm->guest_vers != 0)
+            {
+                /*
+                 * We do not have UUID info for VMs so use
+                 * the 1.0 structure so that we set UUIDs to
+                 * zero using memset
+                 */
+                struct ffa_partition_info_1_0 srcvm;
+
+                if ( curr == ffa_vm_count )
+                {
Quoted 8 lines

xen/arch/arm/tee/ffa_partinfo.c void ffa_handle_partition_info_get(struct cpu_user_regs *regs) · line 242

Quoted 5 lines
xen/arch/arm/tee/ffa_partinfo.c +5 -0
+                    /*
+                     * The number of domains changed since we counted them, we
+                     * can add new ones if there is enough space in the
+                     * destination buffer so check it or go out with an error.
+                     */

Why do we care if the number has changed? If it fits, all is good anyway and we're also updating ffa_vm_count with curr after the loop.

Well this is exactly the point here, we check if we have enough space to return the data with the new domains and if not we return an error.

The point here is to make sure that if there are more domains than when we counted first we check that we have enough size and the update at the end is to handle the case where we have actually less domains than when we counted first.

If the number has changed we only care because we need to make sure we have enough space and because we need to return the right number to the caller.

Please tell me how you would like me to change this because I do not understand what I should modify here.

Cheers
Bertrand

Quoted 87 lines

Cheers,
Jens

Quoted 83 lines
xen/arch/arm/tee/ffa_partinfo.c +34 -0
+                    ffa_vm_count++;
+                    if ( ctx->page_count * FFA_PAGE_SIZE <
+                         (ffa_sp_count + ffa_vm_count) * dst_size )
+                    {
+                        ret = FFA_RET_NO_MEMORY;
+                        rcu_read_unlock(&domlist_read_lock);
+                        goto out_rx_release;
+                    }
+                }
+
+                srcvm.id = ffa_get_vm_id(dom);
+                srcvm.execution_context = dom->max_vcpus;
+                srcvm.partition_properties = FFA_PART_VM_PROP;
+                if ( is_64bit_domain(dom) )
+                    srcvm.partition_properties |= FFA_PART_PROP_AARCH64_STATE;
+
+                memcpy(dst_buf, &srcvm, MIN(sizeof(srcvm), dst_size));
+
+                if ( dst_size > sizeof(srcvm) )
+                    memset(dst_buf + sizeof(srcvm), 0,
+                           dst_size - sizeof(srcvm));
+
+                dst_buf += dst_size;
+                curr++;
+            }
+        }
+        rcu_read_unlock(&domlist_read_lock);
+
+        /* the number of domains could have reduce since the initial count */
+        ffa_vm_count = curr;
+    }
+
+    goto out;
+

out_rx_hyp_release:

    ffa_hyp_rx_release();
out_rx_hyp_unlock:
    spin_unlock(&ffa_rx_buffer_lock);

out_rx_release:

xen/arch/arm/tee/ffa_partinfo.c +1 -9
-    /*
-     * The calling VM RX buffer only contains data to be used by the VM if the
-     * call was successful, in which case the VM has to release the buffer
-     * once it has used the data.
-     * If something went wrong during the call, we have to release the RX
-     * buffer back to the SPMC as the VM will not do it.
-     */
-    if ( ret != FFA_RET_OK )
-        ffa_rx_release(d);
+    ffa_rx_release(d);

out:
if ( ret )
ffa_set_regs_error(regs, ret);
else

xen/arch/arm/tee/ffa_partinfo.c +1 -1
-        ffa_set_regs_success(regs, ffa_sp_count, dst_size);
+        ffa_set_regs_success(regs, ffa_sp_count + ffa_vm_count, dst_size);

}

static int32_t ffa_direct_req_send_vm(uint16_t sp_id, uint16_t vm_id,

xen/arch/arm/tee/ffa_private.h +12 -1
index c4cd65538908..bd6877d8c632 100644
@@ -187,6 +187,18 @@
 */
 #define FFA_PARTITION_INFO_GET_COUNT_FLAG BIT(0, U)
 
+/*
+ * Partition properties we give for a normal world VM:
+ * - can send direct message but not receive them
+ * - can handle indirect messages
+ * - can receive notifications
+ * 32/64 bit flag is set depending on the VM
+ */
+#define FFA_PART_VM_PROP    (FFA_PART_PROP_DIRECT_REQ_SEND | \
+                             FFA_PART_PROP_INDIRECT_MSGS | \
+                             FFA_PART_PROP_RECV_NOTIF | \
+                             FFA_PART_PROP_IS_PE_ID)
+
 /* Flags used in calls to FFA_NOTIFICATION_GET interface  */
 #define FFA_NOTIF_FLAG_BITMAP_SP        BIT(0, U)
 #define FFA_NOTIF_FLAG_BITMAP_VM        BIT(1, U)
--

2.47.1

#7 lore raw

Re: [PATCH v2 2/5] xen/arm: ffa: Introduce VM to VM support

Hi,

Quoted 78 lines · Bertrand Marquis

Hi Jens,

Thanks a lot for the review.

Quoted 47 lines · Jens Wiklander

Hi Bertrand,

Quoted 41 lines · Bertrand Marquis

Create a CONFIG_FFA_VM_TO_VM parameter to activate FFA communication between VMs.
When activated list VMs in the system with FF-A support in part_info_get.

WARNING: There is no filtering for now and all VMs are listed !!

---
Changes in v2:
- Switch ifdef to IS_ENABLED
- dom was not switched to d as requested by Jan because there is already a variable d pointing to the current domain and it must not be shadowed.
---

xen/arch/arm/tee/Kconfig        |  11 +++
xen/arch/arm/tee/ffa_partinfo.c | 144 +++++++++++++++++++++++++-------
xen/arch/arm/tee/ffa_private.h  |  12 +++
3 files changed, 137 insertions(+), 30 deletions(-)
xen/arch/arm/tee/Kconfig +11 -0
index c5b0f88d7522..88a4c4c99154 100644
@@ -28,5 +28,16 @@ config FFA
 
         [1] https://developer.arm.com/documentation/den0077/latest
 
+config FFA_VM_TO_VM
+    bool "Enable FF-A between VMs (UNSUPPORTED)" if UNSUPPORTED
+    default n
+    depends on FFA
+    help
+      This option enables to use FF-A between VMs.
+      This is experimental and there is no access control so any
+      guest can communicate with any other guest.
+
+      If unsure, say N.
+

endmenu

Quoted 15 lines

xen/arch/arm/tee/ffa_partinfo.c void ffa_handle_partition_info_get(struct cpu_user_regs *regs) · line 81

Quoted 11 lines
xen/arch/arm/tee/ffa_partinfo.c +2 -1
index c0510ceb8338..7af1eca2d0c4 100644
@@ -77,7 +77,23 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
    };
    uint32_t src_size, dst_size;
    void *dst_buf;
-    uint32_t ffa_sp_count = 0;
+    uint32_t ffa_vm_count = 0, ffa_sp_count = 0;
+

This function is getting quite large and it's hard to follow the different lock states. How about splitting it into various helper functions?

Yes I agree.
I will try to find a good way to split this in smaller chunks.

Quoted 7 lines
Quoted 6 lines
xen/arch/arm/tee/ffa_partinfo.c +5 -0
+    if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )
+    {
+        struct domain *dom;
+
+        /* Count the number of VM with FF-A support */
Quoted 7 lines
Quoted 2 lines

Why do we need this now? Isn't it enough to count them below when we fill in dst_buf?

We need it in 3 places in the code after:
- to return the number of endpoint in case the flag is set (see final return)
- to check that there is enough space in the RX buffer
- to prevent going through the list of domains if none is to be listed anyway

Got it, thanks. The first point is important, the others are essentially optimizations.

Quoted 126 lines
Quoted 109 lines
Quoted 109 lines
xen/arch/arm/tee/ffa_partinfo.c +66 -19
+        rcu_read_lock(&domlist_read_lock);
+        for_each_domain( dom )
+        {
+            struct ffa_ctx *vm = dom->arch.tee;
+
+            if (dom != d && vm != NULL && vm->guest_vers != 0)
+                ffa_vm_count++;
+        }
+        rcu_read_unlock(&domlist_read_lock);
+    }
 
    /*
     * If the guest is v1.0, he does not get back the entry size so we must
@@ -127,33 +143,38 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
 
    dst_buf = ctx->rx;
 
-    if ( !ffa_rx )
+    /* If not supported, we have ffa_sp_count=0 */
+    if ( ffa_fw_supports_fid(FFA_PARTITION_INFO_GET) )
    {
-        ret = FFA_RET_DENIED;
-        goto out_rx_release;
-    }
+        if ( !ffa_rx )
+        {
+            ret = FFA_RET_DENIED;
+            goto out_rx_release;
+        }
 
-    spin_lock(&ffa_rx_buffer_lock);
+        spin_lock(&ffa_rx_buffer_lock);
 
-    ret = ffa_partition_info_get(uuid, 0, &ffa_sp_count, &src_size);
+        ret = ffa_partition_info_get(uuid, 0, &ffa_sp_count, &src_size);
 
-    if ( ret )
-        goto out_rx_hyp_unlock;
+        if ( ret )
+            goto out_rx_hyp_unlock;
 
-    /*
-     * ffa_partition_info_get() succeeded so we now own the RX buffer we
-     * share with the SPMC. We must give it back using ffa_hyp_rx_release()
-     * once we've copied the content.
-     */
+        /*
+         * ffa_partition_info_get() succeeded so we now own the RX buffer we
+         * share with the SPMC. We must give it back using ffa_hyp_rx_release()
+         * once we've copied the content.
+         */
 
-    /* we cannot have a size smaller than 1.0 structure */
-    if ( src_size < sizeof(struct ffa_partition_info_1_0) )
-    {
-        ret = FFA_RET_NOT_SUPPORTED;
-        goto out_rx_hyp_release;
+        /* we cannot have a size smaller than 1.0 structure */
+        if ( src_size < sizeof(struct ffa_partition_info_1_0) )
+        {
+            ret = FFA_RET_NOT_SUPPORTED;
+            goto out_rx_hyp_release;
+        }
    }
 
-    if ( ctx->page_count * FFA_PAGE_SIZE < ffa_sp_count * dst_size )
+    if ( ctx->page_count * FFA_PAGE_SIZE <
+         (ffa_sp_count + ffa_vm_count) * dst_size )
    {
        ret = FFA_RET_NO_MEMORY;
        goto out_rx_hyp_release;
@@ -182,25 +203,88 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
        }
    }
 
+    if ( ffa_fw_supports_fid(FFA_PARTITION_INFO_GET) )
+    {
+        ffa_hyp_rx_release();
+        spin_unlock(&ffa_rx_buffer_lock);
+    }
+
+    if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) && ffa_vm_count )
+    {
+        struct domain *dom;
+        uint32_t curr = 0;
+
+        /* add the VM informations if any */
+        rcu_read_lock(&domlist_read_lock);
+        for_each_domain( dom )
+        {
+            struct ffa_ctx *vm = dom->arch.tee;
+
+            /*
+             * we do not add the VM calling to the list and only VMs with
+             * FF-A support
+             */
+            if (dom != d && vm != NULL && vm->guest_vers != 0)
+            {
+                /*
+                 * We do not have UUID info for VMs so use
+                 * the 1.0 structure so that we set UUIDs to
+                 * zero using memset
+                 */
+                struct ffa_partition_info_1_0 srcvm;
+
+                if ( curr == ffa_vm_count )
+                {
Quoted 8 lines

xen/arch/arm/tee/ffa_partinfo.c void ffa_handle_partition_info_get(struct cpu_user_regs *regs) · line 242

Quoted 5 lines
xen/arch/arm/tee/ffa_partinfo.c +5 -0
+                    /*
+                     * The number of domains changed since we counted them, we
+                     * can add new ones if there is enough space in the
+                     * destination buffer so check it or go out with an error.
+                     */

Why do we care if the number has changed? If it fits, all is good anyway and we're also updating ffa_vm_count with curr after the loop.

Well this is exactly the point here, we check if we have enough space to return the data with the new domains and if not we return an error.

The point here is to make sure that if there are more domains than when we counted first we check that we have enough size and the update at the end is to handle the case where we have actually less domains than when we counted first.

Quoted 6 lines

If the number has changed we only care because we need to make sure we have enough space and because we need to return the right number to the caller.

Please tell me how you would like me to change this because I do not understand what I should modify here.

I was thinking something like:

        for_each_domain( dom )
        {
            struct ffa_ctx *vm = dom->arch.tee;

            if (dom != d && vm != NULL && vm->guest_vers != 0)
            {
                ...
                ffa_vm_count++;
                if ( ctx->page_count * FFA_PAGE_SIZE <
                     (ffa_sp_count + ffa_vm_count) * dst_size )
                    ... error out

                ... copy data
                dst_buf += dst_size;
            }
        }

Cheers,
Jens

Quoted 93 lines

Cheers
Bertrand

Quoted 87 lines

Cheers,
Jens

Quoted 83 lines
xen/arch/arm/tee/ffa_partinfo.c +34 -0
+                    ffa_vm_count++;
+                    if ( ctx->page_count * FFA_PAGE_SIZE <
+                         (ffa_sp_count + ffa_vm_count) * dst_size )
+                    {
+                        ret = FFA_RET_NO_MEMORY;
+                        rcu_read_unlock(&domlist_read_lock);
+                        goto out_rx_release;
+                    }
+                }
+
+                srcvm.id = ffa_get_vm_id(dom);
+                srcvm.execution_context = dom->max_vcpus;
+                srcvm.partition_properties = FFA_PART_VM_PROP;
+                if ( is_64bit_domain(dom) )
+                    srcvm.partition_properties |= FFA_PART_PROP_AARCH64_STATE;
+
+                memcpy(dst_buf, &srcvm, MIN(sizeof(srcvm), dst_size));
+
+                if ( dst_size > sizeof(srcvm) )
+                    memset(dst_buf + sizeof(srcvm), 0,
+                           dst_size - sizeof(srcvm));
+
+                dst_buf += dst_size;
+                curr++;
+            }
+        }
+        rcu_read_unlock(&domlist_read_lock);
+
+        /* the number of domains could have reduce since the initial count */
+        ffa_vm_count = curr;
+    }
+
+    goto out;
+

out_rx_hyp_release:

    ffa_hyp_rx_release();
out_rx_hyp_unlock:
    spin_unlock(&ffa_rx_buffer_lock);

out_rx_release:

xen/arch/arm/tee/ffa_partinfo.c +1 -9
-    /*
-     * The calling VM RX buffer only contains data to be used by the VM if the
-     * call was successful, in which case the VM has to release the buffer
-     * once it has used the data.
-     * If something went wrong during the call, we have to release the RX
-     * buffer back to the SPMC as the VM will not do it.
-     */
-    if ( ret != FFA_RET_OK )
-        ffa_rx_release(d);
+    ffa_rx_release(d);

out:
if ( ret )
ffa_set_regs_error(regs, ret);
else

xen/arch/arm/tee/ffa_partinfo.c +1 -1
-        ffa_set_regs_success(regs, ffa_sp_count, dst_size);
+        ffa_set_regs_success(regs, ffa_sp_count + ffa_vm_count, dst_size);

}

static int32_t ffa_direct_req_send_vm(uint16_t sp_id, uint16_t vm_id,

xen/arch/arm/tee/ffa_private.h +12 -1
index c4cd65538908..bd6877d8c632 100644
@@ -187,6 +187,18 @@
 */
 #define FFA_PARTITION_INFO_GET_COUNT_FLAG BIT(0, U)
 
+/*
+ * Partition properties we give for a normal world VM:
+ * - can send direct message but not receive them
+ * - can handle indirect messages
+ * - can receive notifications
+ * 32/64 bit flag is set depending on the VM
+ */
+#define FFA_PART_VM_PROP    (FFA_PART_PROP_DIRECT_REQ_SEND | \
+                             FFA_PART_PROP_INDIRECT_MSGS | \
+                             FFA_PART_PROP_RECV_NOTIF | \
+                             FFA_PART_PROP_IS_PE_ID)
+
 /* Flags used in calls to FFA_NOTIFICATION_GET interface  */
 #define FFA_NOTIF_FLAG_BITMAP_SP        BIT(0, U)
 #define FFA_NOTIF_FLAG_BITMAP_VM        BIT(1, U)
--

2.47.1

#8 lore raw

Re: [PATCH v2 2/5] xen/arm: ffa: Introduce VM to VM support

Hi Jens,

Quoted 87 lines · Jens Wiklander

Hi,

Quoted 81 lines · Bertrand Marquis

Hi Jens,

Thanks a lot for the review.

Quoted 47 lines · Jens Wiklander

Hi Bertrand,

Quoted 41 lines · Bertrand Marquis

Create a CONFIG_FFA_VM_TO_VM parameter to activate FFA communication between VMs.
When activated list VMs in the system with FF-A support in part_info_get.

WARNING: There is no filtering for now and all VMs are listed !!

---
Changes in v2:
- Switch ifdef to IS_ENABLED
- dom was not switched to d as requested by Jan because there is already a variable d pointing to the current domain and it must not be shadowed.
---

xen/arch/arm/tee/Kconfig        |  11 +++
xen/arch/arm/tee/ffa_partinfo.c | 144 +++++++++++++++++++++++++-------
xen/arch/arm/tee/ffa_private.h  |  12 +++
3 files changed, 137 insertions(+), 30 deletions(-)
xen/arch/arm/tee/Kconfig +11 -0
index c5b0f88d7522..88a4c4c99154 100644
@@ -28,5 +28,16 @@ config FFA
 
        [1] https://developer.arm.com/documentation/den0077/latest
 
+config FFA_VM_TO_VM
+    bool "Enable FF-A between VMs (UNSUPPORTED)" if UNSUPPORTED
+    default n
+    depends on FFA
+    help
+      This option enables to use FF-A between VMs.
+      This is experimental and there is no access control so any
+      guest can communicate with any other guest.
+
+      If unsure, say N.
+

endmenu

Quoted 15 lines

xen/arch/arm/tee/ffa_partinfo.c void ffa_handle_partition_info_get(struct cpu_user_regs *regs) · line 81

Quoted 11 lines
xen/arch/arm/tee/ffa_partinfo.c +2 -1
index c0510ceb8338..7af1eca2d0c4 100644
@@ -77,7 +77,23 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
   };
   uint32_t src_size, dst_size;
   void *dst_buf;
-    uint32_t ffa_sp_count = 0;
+    uint32_t ffa_vm_count = 0, ffa_sp_count = 0;
+

This function is getting quite large and it's hard to follow the different lock states. How about splitting it into various helper functions?

Yes I agree.
I will try to find a good way to split this in smaller chunks.

Quoted 9 lines

xen/arch/arm/tee/ffa_partinfo.c void ffa_handle_partition_info_get(struct cpu_user_regs *regs) · line 86

Quoted 6 lines
xen/arch/arm/tee/ffa_partinfo.c +5 -0
+    if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )
+    {
+        struct domain *dom;
+
+        /* Count the number of VM with FF-A support */

Why do we need this now? Isn't it enough to count them below when we fill in dst_buf?

Quoted 7 lines
Quoted 4 lines

We need it in 3 places in the code after:
- to return the number of endpoint in case the flag is set (see final return)
- to check that there is enough space in the RX buffer
- to prevent going through the list of domains if none is to be listed anyway

Got it, thanks. The first point is important, the others are essentially optimizations.

With your previous point and me splitting things this will not be the case anymore.

Quoted 135 lines
Quoted 127 lines
Quoted 110 lines
Quoted 110 lines
xen/arch/arm/tee/ffa_partinfo.c +66 -19
+        rcu_read_lock(&domlist_read_lock);
+        for_each_domain( dom )
+        {
+            struct ffa_ctx *vm = dom->arch.tee;
+
+            if (dom != d && vm != NULL && vm->guest_vers != 0)
+                ffa_vm_count++;
+        }
+        rcu_read_unlock(&domlist_read_lock);
+    }
 
   /*
    * If the guest is v1.0, he does not get back the entry size so we must
@@ -127,33 +143,38 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
 
   dst_buf = ctx->rx;
 
-    if ( !ffa_rx )
+    /* If not supported, we have ffa_sp_count=0 */
+    if ( ffa_fw_supports_fid(FFA_PARTITION_INFO_GET) )
   {
-        ret = FFA_RET_DENIED;
-        goto out_rx_release;
-    }
+        if ( !ffa_rx )
+        {
+            ret = FFA_RET_DENIED;
+            goto out_rx_release;
+        }
 
-    spin_lock(&ffa_rx_buffer_lock);
+        spin_lock(&ffa_rx_buffer_lock);
 
-    ret = ffa_partition_info_get(uuid, 0, &ffa_sp_count, &src_size);
+        ret = ffa_partition_info_get(uuid, 0, &ffa_sp_count, &src_size);
 
-    if ( ret )
-        goto out_rx_hyp_unlock;
+        if ( ret )
+            goto out_rx_hyp_unlock;
 
-    /*
-     * ffa_partition_info_get() succeeded so we now own the RX buffer we
-     * share with the SPMC. We must give it back using ffa_hyp_rx_release()
-     * once we've copied the content.
-     */
+        /*
+         * ffa_partition_info_get() succeeded so we now own the RX buffer we
+         * share with the SPMC. We must give it back using ffa_hyp_rx_release()
+         * once we've copied the content.
+         */
 
-    /* we cannot have a size smaller than 1.0 structure */
-    if ( src_size < sizeof(struct ffa_partition_info_1_0) )
-    {
-        ret = FFA_RET_NOT_SUPPORTED;
-        goto out_rx_hyp_release;
+        /* we cannot have a size smaller than 1.0 structure */
+        if ( src_size < sizeof(struct ffa_partition_info_1_0) )
+        {
+            ret = FFA_RET_NOT_SUPPORTED;
+            goto out_rx_hyp_release;
+        }
   }
 
-    if ( ctx->page_count * FFA_PAGE_SIZE < ffa_sp_count * dst_size )
+    if ( ctx->page_count * FFA_PAGE_SIZE <
+         (ffa_sp_count + ffa_vm_count) * dst_size )
   {
       ret = FFA_RET_NO_MEMORY;
       goto out_rx_hyp_release;
@@ -182,25 +203,88 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
       }
   }
 
+    if ( ffa_fw_supports_fid(FFA_PARTITION_INFO_GET) )
+    {
+        ffa_hyp_rx_release();
+        spin_unlock(&ffa_rx_buffer_lock);
+    }
+
+    if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) && ffa_vm_count )
+    {
+        struct domain *dom;
+        uint32_t curr = 0;
+
+        /* add the VM informations if any */
+        rcu_read_lock(&domlist_read_lock);
+        for_each_domain( dom )
+        {
+            struct ffa_ctx *vm = dom->arch.tee;
+
+            /*
+             * we do not add the VM calling to the list and only VMs with
+             * FF-A support
+             */
+            if (dom != d && vm != NULL && vm->guest_vers != 0)
+            {
+                /*
+                 * We do not have UUID info for VMs so use
+                 * the 1.0 structure so that we set UUIDs to
+                 * zero using memset
+                 */
+                struct ffa_partition_info_1_0 srcvm;
+
+                if ( curr == ffa_vm_count )
+                {
Quoted 8 lines

xen/arch/arm/tee/ffa_partinfo.c void ffa_handle_partition_info_get(struct cpu_user_regs *regs) · line 242

Quoted 5 lines
xen/arch/arm/tee/ffa_partinfo.c +5 -0
+                    /*
+                     * The number of domains changed since we counted them, we
+                     * can add new ones if there is enough space in the
+                     * destination buffer so check it or go out with an error.
+                     */

Why do we care if the number has changed? If it fits, all is good anyway and we're also updating ffa_vm_count with curr after the loop.

Well this is exactly the point here, we check if we have enough space to return the data with the new domains and if not we return an error.

The point here is to make sure that if there are more domains than when we counted first we check that we have enough size and the update at the end is to handle the case where we have actually less domains than when we counted first.

Quoted 6 lines

If the number has changed we only care because we need to make sure we have enough space and because we need to return the right number to the caller.

Please tell me how you would like me to change this because I do not understand what I should modify here.

I was thinking something like:

Quoted 16 lines
       for_each_domain( dom )
       {
           struct ffa_ctx *vm = dom->arch.tee;

           if (dom != d && vm != NULL && vm->guest_vers != 0)
           {
               ...
               ffa_vm_count++;
               if ( ctx->page_count * FFA_PAGE_SIZE <
                    (ffa_sp_count + ffa_vm_count) * dst_size )
                   ... error out

               ... copy data
               dst_buf += dst_size;
           }
       }

Ok got you.
I will modify things to simplify and just check along the way if there is enough space to add an extra entry instead of trying to handle a corner case where the number changes in the middle that will make things simpler.
Stay tuned for v3 ;-)

Cheers
Bertrand

Quoted 95 lines

Cheers,
Jens

Quoted 91 lines

Cheers
Bertrand

Quoted 87 lines

Cheers,
Jens

Quoted 83 lines
xen/arch/arm/tee/ffa_partinfo.c +34 -0
+                    ffa_vm_count++;
+                    if ( ctx->page_count * FFA_PAGE_SIZE <
+                         (ffa_sp_count + ffa_vm_count) * dst_size )
+                    {
+                        ret = FFA_RET_NO_MEMORY;
+                        rcu_read_unlock(&domlist_read_lock);
+                        goto out_rx_release;
+                    }
+                }
+
+                srcvm.id = ffa_get_vm_id(dom);
+                srcvm.execution_context = dom->max_vcpus;
+                srcvm.partition_properties = FFA_PART_VM_PROP;
+                if ( is_64bit_domain(dom) )
+                    srcvm.partition_properties |= FFA_PART_PROP_AARCH64_STATE;
+
+                memcpy(dst_buf, &srcvm, MIN(sizeof(srcvm), dst_size));
+
+                if ( dst_size > sizeof(srcvm) )
+                    memset(dst_buf + sizeof(srcvm), 0,
+                           dst_size - sizeof(srcvm));
+
+                dst_buf += dst_size;
+                curr++;
+            }
+        }
+        rcu_read_unlock(&domlist_read_lock);
+
+        /* the number of domains could have reduce since the initial count */
+        ffa_vm_count = curr;
+    }
+
+    goto out;
+

out_rx_hyp_release:

   ffa_hyp_rx_release();
out_rx_hyp_unlock:
   spin_unlock(&ffa_rx_buffer_lock);

out_rx_release:

xen/arch/arm/tee/ffa_partinfo.c +1 -9
-    /*
-     * The calling VM RX buffer only contains data to be used by the VM if the
-     * call was successful, in which case the VM has to release the buffer
-     * once it has used the data.
-     * If something went wrong during the call, we have to release the RX
-     * buffer back to the SPMC as the VM will not do it.
-     */
-    if ( ret != FFA_RET_OK )
-        ffa_rx_release(d);
+    ffa_rx_release(d);

out:
if ( ret )

       ffa_set_regs_error(regs, ret);

else

xen/arch/arm/tee/ffa_partinfo.c +1 -1
-        ffa_set_regs_success(regs, ffa_sp_count, dst_size);
+        ffa_set_regs_success(regs, ffa_sp_count + ffa_vm_count, dst_size);

}

static int32_t ffa_direct_req_send_vm(uint16_t sp_id, uint16_t vm_id,

xen/arch/arm/tee/ffa_private.h +12 -1
index c4cd65538908..bd6877d8c632 100644
@@ -187,6 +187,18 @@
 */
 #define FFA_PARTITION_INFO_GET_COUNT_FLAG BIT(0, U)
 
+/*
+ * Partition properties we give for a normal world VM:
+ * - can send direct message but not receive them
+ * - can handle indirect messages
+ * - can receive notifications
+ * 32/64 bit flag is set depending on the VM
+ */
+#define FFA_PART_VM_PROP    (FFA_PART_PROP_DIRECT_REQ_SEND | \
+                             FFA_PART_PROP_INDIRECT_MSGS | \
+                             FFA_PART_PROP_RECV_NOTIF | \
+                             FFA_PART_PROP_IS_PE_ID)
+
 /* Flags used in calls to FFA_NOTIFICATION_GET interface  */
 #define FFA_NOTIF_FLAG_BITMAP_SP        BIT(0, U)
 #define FFA_NOTIF_FLAG_BITMAP_VM        BIT(1, U)
--

2.47.1

#9 lore raw

PATCH v2 3/5 xen/arm: ffa: Add buffer full notification support

In reply to Bertrand Marquis

Add support to raise a Rx buffer full notification to a VM. This function will be used for indirect message support between VM and is only activated if CONFIG_FFA_VM_TO_VM is selected.

Even if there are 32 framework notifications possible, right now only one is defined so the implementation is simplified to only handle the buffer full notification using a boolean. If other framework notifications have to be supported one day, the design will have to be modified to handle it properly.

---
Changes in v2:
- Code style fix

---
 xen/arch/arm/tee/ffa_notif.c   | 26 +++++++++++++++++++++-----
 xen/arch/arm/tee/ffa_private.h | 13 +++++++++++++
 2 files changed, 34 insertions(+), 5 deletions(-)
xen/arch/arm/tee/ffa_notif.c +21 -5
index 00efaf8f7353..d19aa5c5bef6 100644
@@ -93,6 +93,7 @@ void ffa_handle_notification_info_get(struct cpu_user_regs *regs)
 void ffa_handle_notification_get(struct cpu_user_regs *regs)
 {
     struct domain *d = current->domain;
+    struct ffa_ctx *ctx = d->arch.tee;
     uint32_t recv = get_user_reg(regs, 1);
     uint32_t flags = get_user_reg(regs, 2);
     uint32_t w2 = 0;
@@ -132,11 +133,7 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
          */
         if ( ( flags  & FFA_NOTIF_FLAG_BITMAP_SP ) &&
              ( flags & FFA_NOTIF_FLAG_BITMAP_SPM ) )
-        {
-                struct ffa_ctx *ctx = d->arch.tee;
-
-                ACCESS_ONCE(ctx->notif.secure_pending) = false;
-        }
+            ACCESS_ONCE(ctx->notif.secure_pending) = false;
 
         arm_smccc_1_2_smc(&arg, &resp);
         e = ffa_get_ret_code(&resp);
@@ -156,6 +153,12 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
             w6 = resp.a6;
     }
 
+#ifdef CONFIG_FFA_VM_TO_VM
+    if ( flags & FFA_NOTIF_FLAG_BITMAP_HYP &&
+          test_and_clear_bool(ctx->notif.buff_full_pending) )
+        w7 = FFA_NOTIF_RX_BUFFER_FULL;
+#endif
+
     ffa_set_regs(regs, FFA_SUCCESS_32, 0, w2, w3, w4, w5, w6, w7);
 }
 
@@ -178,6 +181,19 @@ int ffa_handle_notification_set(struct cpu_user_regs *regs)
                            bitmap_hi);
 }
 
+#ifdef CONFIG_FFA_VM_TO_VM
+void ffa_raise_rx_buffer_full(struct domain *d)
+{
+    struct ffa_ctx *ctx = d->arch.tee;
+
+    if ( !ctx )
+        return;
+
+    if ( !test_and_set_bool(ctx->notif.buff_full_pending) )
+        vgic_inject_irq(d, d->vcpu[0], notif_sri_irq, true);
+}
+#endif
+
 /*
  * Extract a 16-bit ID (index n) from the successful return value from
  * FFA_NOTIFICATION_INFO_GET_64 or FFA_NOTIFICATION_INFO_GET_32. IDs are
xen/arch/arm/tee/ffa_private.h +13 -0
index bd6877d8c632..1f5067d5d0c9 100644
@@ -210,6 +210,8 @@
 #define FFA_NOTIF_INFO_GET_ID_COUNT_SHIFT   7
 #define FFA_NOTIF_INFO_GET_ID_COUNT_MASK    0x1F
 
+#define FFA_NOTIF_RX_BUFFER_FULL        BIT(0, U)
+
 /* Feature IDs used with FFA_FEATURES */
 #define FFA_FEATURE_NOTIF_PEND_INTR     0x1U
 #define FFA_FEATURE_SCHEDULE_RECV_INTR  0x2U
@@ -298,6 +300,13 @@ struct ffa_ctx_notif {
      * pending global notifications.
      */
     bool secure_pending;
+
+#ifdef CONFIG_FFA_VM_TO_VM
+    /*
+     * Pending Hypervisor framework notifications
+     */
+    bool buff_full_pending;
+#endif
 };
 
 struct ffa_ctx {
@@ -369,6 +378,10 @@ void ffa_handle_notification_info_get(struct cpu_user_regs *regs);
 void ffa_handle_notification_get(struct cpu_user_regs *regs);
 int ffa_handle_notification_set(struct cpu_user_regs *regs);
 
+#ifdef CONFIG_FFA_VM_TO_VM
+void ffa_raise_rx_buffer_full(struct domain *d);
+#endif
+
 void ffa_handle_msg_send_direct_req(struct cpu_user_regs *regs, uint32_t fid);
 int32_t ffa_handle_msg_send2(struct cpu_user_regs *regs);
 
Signature · 1 line
2.47.1
#10 lore raw

Re: [PATCH v2 3/5] xen/arm: ffa: Add buffer full notification support

Hi Bertrand,

Quoted 92 lines · Bertrand Marquis

Add support to raise a Rx buffer full notification to a VM. This function will be used for indirect message support between VM and is only activated if CONFIG_FFA_VM_TO_VM is selected.

Even if there are 32 framework notifications possible, right now only one is defined so the implementation is simplified to only handle the buffer full notification using a boolean. If other framework notifications have to be supported one day, the design will have to be modified to handle it properly.

---
Changes in v2:
- Code style fix

---
 xen/arch/arm/tee/ffa_notif.c   | 26 +++++++++++++++++++++-----
 xen/arch/arm/tee/ffa_private.h | 13 +++++++++++++
 2 files changed, 34 insertions(+), 5 deletions(-)
xen/arch/arm/tee/ffa_notif.c +21 -5
index 00efaf8f7353..d19aa5c5bef6 100644
@@ -93,6 +93,7 @@ void ffa_handle_notification_info_get(struct cpu_user_regs *regs)
 void ffa_handle_notification_get(struct cpu_user_regs *regs)
 {
     struct domain *d = current->domain;
+    struct ffa_ctx *ctx = d->arch.tee;
     uint32_t recv = get_user_reg(regs, 1);
     uint32_t flags = get_user_reg(regs, 2);
     uint32_t w2 = 0;
@@ -132,11 +133,7 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
          */
         if ( ( flags  & FFA_NOTIF_FLAG_BITMAP_SP ) &&
              ( flags & FFA_NOTIF_FLAG_BITMAP_SPM ) )
-        {
-                struct ffa_ctx *ctx = d->arch.tee;
-
-                ACCESS_ONCE(ctx->notif.secure_pending) = false;
-        }
+            ACCESS_ONCE(ctx->notif.secure_pending) = false;
 
         arm_smccc_1_2_smc(&arg, &resp);
         e = ffa_get_ret_code(&resp);
@@ -156,6 +153,12 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
             w6 = resp.a6;
     }
 
+#ifdef CONFIG_FFA_VM_TO_VM
+    if ( flags & FFA_NOTIF_FLAG_BITMAP_HYP &&
+          test_and_clear_bool(ctx->notif.buff_full_pending) )
+        w7 = FFA_NOTIF_RX_BUFFER_FULL;
+#endif
+
     ffa_set_regs(regs, FFA_SUCCESS_32, 0, w2, w3, w4, w5, w6, w7);
 }
 
@@ -178,6 +181,19 @@ int ffa_handle_notification_set(struct cpu_user_regs *regs)
                            bitmap_hi);
 }
 
+#ifdef CONFIG_FFA_VM_TO_VM
+void ffa_raise_rx_buffer_full(struct domain *d)
+{
+    struct ffa_ctx *ctx = d->arch.tee;
+
+    if ( !ctx )
+        return;
+
+    if ( !test_and_set_bool(ctx->notif.buff_full_pending) )
+        vgic_inject_irq(d, d->vcpu[0], notif_sri_irq, true);
+}
+#endif
+
 /*
  * Extract a 16-bit ID (index n) from the successful return value from
  * FFA_NOTIFICATION_INFO_GET_64 or FFA_NOTIFICATION_INFO_GET_32. IDs are
xen/arch/arm/tee/ffa_private.h +2 -0
index bd6877d8c632..1f5067d5d0c9 100644
@@ -210,6 +210,8 @@
 #define FFA_NOTIF_INFO_GET_ID_COUNT_SHIFT   7
 #define FFA_NOTIF_INFO_GET_ID_COUNT_MASK    0x1F
 
+#define FFA_NOTIF_RX_BUFFER_FULL        BIT(0, U)
+
 /* Feature IDs used with FFA_FEATURES */
 #define FFA_FEATURE_NOTIF_PEND_INTR     0x1U
 #define FFA_FEATURE_SCHEDULE_RECV_INTR  0x2U

xen/arch/arm/tee/ffa_private.h struct ffa_ctx_notif { · line 309

Quoted 10 lines
xen/arch/arm/tee/ffa_private.h +7 -0
@@ -298,6 +300,13 @@ struct ffa_ctx_notif {
      * pending global notifications.
      */
     bool secure_pending;
+
+#ifdef CONFIG_FFA_VM_TO_VM
+    /*
+     * Pending Hypervisor framework notifications
+     */
+    bool buff_full_pending;
+#endif

It doesn't matter if there are one or two bools in this struct. However, the ifdef prevents using IS_ENABLED(CONFIG_FFA_VM_TO_VM) instead of an ifdef in the logic controlling access to this field.

Cheers,
Jens

Quoted 17 lines
xen/arch/arm/tee/ffa_private.h +4 -1
 };
 
 struct ffa_ctx {
@@ -369,6 +378,10 @@ void ffa_handle_notification_info_get(struct cpu_user_regs *regs);
 void ffa_handle_notification_get(struct cpu_user_regs *regs);
 int ffa_handle_notification_set(struct cpu_user_regs *regs);
 
+#ifdef CONFIG_FFA_VM_TO_VM
+void ffa_raise_rx_buffer_full(struct domain *d);
+#endif
+
 void ffa_handle_msg_send_direct_req(struct cpu_user_regs *regs, uint32_t fid);
 int32_t ffa_handle_msg_send2(struct cpu_user_regs *regs);
 
--

2.47.1

#11 lore raw

Re: [PATCH v2 3/5] xen/arm: ffa: Add buffer full notification support

Hi Jens,

Quoted 104 lines · Jens Wiklander

Hi Bertrand,

Quoted 98 lines · Bertrand Marquis

Add support to raise a Rx buffer full notification to a VM. This function will be used for indirect message support between VM and is only activated if CONFIG_FFA_VM_TO_VM is selected.

Even if there are 32 framework notifications possible, right now only one is defined so the implementation is simplified to only handle the buffer full notification using a boolean. If other framework notifications have to be supported one day, the design will have to be modified to handle it properly.

---
Changes in v2:
- Code style fix
---
xen/arch/arm/tee/ffa_notif.c | 26 +++++++++++++++++++++-----
xen/arch/arm/tee/ffa_private.h | 13 +++++++++++++

2 files changed, 34 insertions(+), 5 deletions(-)
xen/arch/arm/tee/ffa_notif.c +21 -5
index 00efaf8f7353..d19aa5c5bef6 100644
@@ -93,6 +93,7 @@ void ffa_handle_notification_info_get(struct cpu_user_regs *regs)
 void ffa_handle_notification_get(struct cpu_user_regs *regs)
 {
    struct domain *d = current->domain;
+    struct ffa_ctx *ctx = d->arch.tee;
    uint32_t recv = get_user_reg(regs, 1);
    uint32_t flags = get_user_reg(regs, 2);
    uint32_t w2 = 0;
@@ -132,11 +133,7 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
         */
        if ( ( flags  & FFA_NOTIF_FLAG_BITMAP_SP ) &&
             ( flags & FFA_NOTIF_FLAG_BITMAP_SPM ) )
-        {
-                struct ffa_ctx *ctx = d->arch.tee;
-
-                ACCESS_ONCE(ctx->notif.secure_pending) = false;
-        }
+            ACCESS_ONCE(ctx->notif.secure_pending) = false;
 
        arm_smccc_1_2_smc(&arg, &resp);
        e = ffa_get_ret_code(&resp);
@@ -156,6 +153,12 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
            w6 = resp.a6;
    }
 
+#ifdef CONFIG_FFA_VM_TO_VM
+    if ( flags & FFA_NOTIF_FLAG_BITMAP_HYP &&
+          test_and_clear_bool(ctx->notif.buff_full_pending) )
+        w7 = FFA_NOTIF_RX_BUFFER_FULL;
+#endif
+
    ffa_set_regs(regs, FFA_SUCCESS_32, 0, w2, w3, w4, w5, w6, w7);
 }
 
@@ -178,6 +181,19 @@ int ffa_handle_notification_set(struct cpu_user_regs *regs)
                           bitmap_hi);
 }
 
+#ifdef CONFIG_FFA_VM_TO_VM
+void ffa_raise_rx_buffer_full(struct domain *d)
+{
+    struct ffa_ctx *ctx = d->arch.tee;
+
+    if ( !ctx )
+        return;
+
+    if ( !test_and_set_bool(ctx->notif.buff_full_pending) )
+        vgic_inject_irq(d, d->vcpu[0], notif_sri_irq, true);
+}
+#endif
+
 /*
 * Extract a 16-bit ID (index n) from the successful return value from
 * FFA_NOTIFICATION_INFO_GET_64 or FFA_NOTIFICATION_INFO_GET_32. IDs are
xen/arch/arm/tee/ffa_private.h +5 -0
index bd6877d8c632..1f5067d5d0c9 100644
@@ -210,6 +210,8 @@
 #define FFA_NOTIF_INFO_GET_ID_COUNT_SHIFT   7
 #define FFA_NOTIF_INFO_GET_ID_COUNT_MASK    0x1F
 
+#define FFA_NOTIF_RX_BUFFER_FULL        BIT(0, U)
+
 /* Feature IDs used with FFA_FEATURES */
 #define FFA_FEATURE_NOTIF_PEND_INTR     0x1U
 #define FFA_FEATURE_SCHEDULE_RECV_INTR  0x2U
@@ -298,6 +300,13 @@ struct ffa_ctx_notif {
     * pending global notifications.
     */
    bool secure_pending;
+
+#ifdef CONFIG_FFA_VM_TO_VM
+    /*
Quoted 8 lines

xen/arch/arm/tee/ffa_private.h struct ffa_ctx_notif { · line 309

Quoted 4 lines
xen/arch/arm/tee/ffa_private.h +4 -0
+     * Pending Hypervisor framework notifications
+     */
+    bool buff_full_pending;
+#endif

It doesn't matter if there are one or two bools in this struct. However, the ifdef prevents using IS_ENABLED(CONFIG_FFA_VM_TO_VM) instead of an ifdef in the logic controlling access to this field.

Agree, I will make this field present all the time and switch to IS_ENABLED when now possible.

Cheers
Bertrand

Quoted 20 lines

Cheers,
Jens

Quoted 16 lines
xen/arch/arm/tee/ffa_private.h +4 -1
 };
 
 struct ffa_ctx {
@@ -369,6 +378,10 @@ void ffa_handle_notification_info_get(struct cpu_user_regs *regs);
 void ffa_handle_notification_get(struct cpu_user_regs *regs);
 int ffa_handle_notification_set(struct cpu_user_regs *regs);
 
+#ifdef CONFIG_FFA_VM_TO_VM
+void ffa_raise_rx_buffer_full(struct domain *d);
+#endif
+
 void ffa_handle_msg_send_direct_req(struct cpu_user_regs *regs, uint32_t fid);
 int32_t ffa_handle_msg_send2(struct cpu_user_regs *regs);
 
--

2.47.1

#12 lore raw

PATCH v2 4/5 xen/arm: ffa: Add indirect message between VM

In reply to Bertrand Marquis

Add support for indirect messages between VMs. This is only enabled if CONFIG_FFA_VM_TO_VM is selected.

---
Changes in v2:
- Switch ifdef to IS_ENABLED

---
 xen/arch/arm/tee/ffa_msg.c     | 96 +++++++++++++++++++++++++++++++---
 xen/arch/arm/tee/ffa_private.h |  4 ++
 2 files changed, 92 insertions(+), 8 deletions(-)
xen/arch/arm/tee/ffa_msg.c +88 -8
index ee594e737fc7..336d5bbf64f6 100644
@@ -96,9 +96,6 @@ int32_t ffa_handle_msg_send2(struct cpu_user_regs *regs)
     uint16_t dst_id, src_id;
     int32_t ret;
 
-    if ( !ffa_fw_supports_fid(FFA_MSG_SEND2) )
-        return FFA_RET_NOT_SUPPORTED;
-
     if ( !spin_trylock(&src_ctx->tx_lock) )
         return FFA_RET_BUSY;
 
@@ -106,10 +103,10 @@ int32_t ffa_handle_msg_send2(struct cpu_user_regs *regs)
     src_id = src_msg->send_recv_id >> 16;
     dst_id = src_msg->send_recv_id & GENMASK(15,0);
 
-    if ( src_id != ffa_get_vm_id(src_d) || !FFA_ID_IS_SECURE(dst_id) )
+    if ( src_id != ffa_get_vm_id(src_d) )
     {
         ret = FFA_RET_INVALID_PARAMETERS;
-        goto out_unlock_tx;
+        goto out;
     }
 
     /* check source message fits in buffer */
@@ -118,13 +115,96 @@ int32_t ffa_handle_msg_send2(struct cpu_user_regs *regs)
          src_msg->msg_offset < sizeof(struct ffa_part_msg_rxtx) )
     {
         ret = FFA_RET_INVALID_PARAMETERS;
-        goto out_unlock_tx;
+        goto out;
     }
 
-    ret = ffa_simple_call(FFA_MSG_SEND2,
+    if ( FFA_ID_IS_SECURE(dst_id) )
+    {
+        /* Message for a secure partition */
+        if ( !ffa_fw_supports_fid(FFA_MSG_SEND2) )
+        {
+            ret = FFA_RET_NOT_SUPPORTED;
+            goto out;
+        }
+
+        ret = ffa_simple_call(FFA_MSG_SEND2,
                           ((uint32_t)ffa_get_vm_id(src_d)) << 16, 0, 0, 0);
+        goto out;
+    }
 
-out_unlock_tx:
+    /* Message for a VM */
+    if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )
+    {
+        struct domain *dst_d;
+        struct ffa_ctx *dst_ctx;
+        struct ffa_part_msg_rxtx *dst_msg;
+        int err;
+
+        if ( dst_id == 0 )
+        {
+            /* FF-A ID 0 is the hypervisor, this is not valid */
+            ret = FFA_RET_INVALID_PARAMETERS;
+            goto out;
+        }
+
+        /* This is also checking that dest is not src */
+        err = rcu_lock_live_remote_domain_by_id(dst_id - 1, &dst_d);
+        if ( err )
+        {
+            ret = FFA_RET_INVALID_PARAMETERS;
+            goto out;
+        }
+
+        if ( dst_d->arch.tee == NULL )
+        {
+            ret = FFA_RET_INVALID_PARAMETERS;
+            goto out_unlock;
+        }
+
+        dst_ctx = dst_d->arch.tee;
+        if ( !dst_ctx->guest_vers )
+        {
+            ret = FFA_RET_INVALID_PARAMETERS;
+            goto out_unlock;
+        }
+
+        /* This also checks that destination has set a Rx buffer */
+        ret = ffa_rx_acquire(dst_d);
+        if ( ret )
+            goto out_unlock;
+
+        /* we need to have enough space in the destination buffer */
+        if ( dst_ctx->page_count * FFA_PAGE_SIZE <
+                (sizeof(struct ffa_part_msg_rxtx) + src_msg->msg_size) )
+        {
+            ret = FFA_RET_NO_MEMORY;
+            ffa_rx_release(dst_d);
+            goto out_unlock;
+        }
+
+        dst_msg = dst_ctx->rx;
+
+        /* prepare destination header */
+        dst_msg->flags = 0;
+        dst_msg->reserved = 0;
+        dst_msg->msg_offset = sizeof(struct ffa_part_msg_rxtx);
+        dst_msg->send_recv_id = src_msg->send_recv_id;
+        dst_msg->msg_size = src_msg->msg_size;
+
+        memcpy(dst_ctx->rx + sizeof(struct ffa_part_msg_rxtx),
+               src_ctx->tx + src_msg->msg_offset, src_msg->msg_size);
+
+        /* receiver rx buffer will be released by the receiver*/
+
+out_unlock:
+        rcu_unlock_domain(dst_d);
+        if ( !ret )
+            ffa_raise_rx_buffer_full(dst_d);
+    }
+    else
+        ret = FFA_RET_INVALID_PARAMETERS;
+
+out:
     spin_unlock(&src_ctx->tx_lock);
     return ret;
 }
xen/arch/arm/tee/ffa_private.h +4 -0
index 1f5067d5d0c9..340db229453c 100644
@@ -380,6 +380,10 @@ int ffa_handle_notification_set(struct cpu_user_regs *regs);
 
 #ifdef CONFIG_FFA_VM_TO_VM
 void ffa_raise_rx_buffer_full(struct domain *d);
+#else
+static inline void ffa_raise_rx_buffer_full(struct domain *d)
+{
+}
 #endif
 
 void ffa_handle_msg_send_direct_req(struct cpu_user_regs *regs, uint32_t fid);
Signature · 1 line
2.47.1
#13 lore raw

Re: [PATCH v2 4/5] xen/arm: ffa: Add indirect message between VM

Hi Bertrand,

Quoted 57 lines · Bertrand Marquis

Add support for indirect messages between VMs. This is only enabled if CONFIG_FFA_VM_TO_VM is selected.

---
Changes in v2:
- Switch ifdef to IS_ENABLED

---
 xen/arch/arm/tee/ffa_msg.c     | 96 +++++++++++++++++++++++++++++++---
 xen/arch/arm/tee/ffa_private.h |  4 ++
 2 files changed, 92 insertions(+), 8 deletions(-)
xen/arch/arm/tee/ffa_msg.c +11 -7
index ee594e737fc7..336d5bbf64f6 100644
@@ -96,9 +96,6 @@ int32_t ffa_handle_msg_send2(struct cpu_user_regs *regs)
     uint16_t dst_id, src_id;
     int32_t ret;
 
-    if ( !ffa_fw_supports_fid(FFA_MSG_SEND2) )
-        return FFA_RET_NOT_SUPPORTED;
-
     if ( !spin_trylock(&src_ctx->tx_lock) )
         return FFA_RET_BUSY;
 
@@ -106,10 +103,10 @@ int32_t ffa_handle_msg_send2(struct cpu_user_regs *regs)
     src_id = src_msg->send_recv_id >> 16;
     dst_id = src_msg->send_recv_id & GENMASK(15,0);
 
-    if ( src_id != ffa_get_vm_id(src_d) || !FFA_ID_IS_SECURE(dst_id) )
+    if ( src_id != ffa_get_vm_id(src_d) )
     {
         ret = FFA_RET_INVALID_PARAMETERS;
-        goto out_unlock_tx;
+        goto out;
     }
 
     /* check source message fits in buffer */
@@ -118,13 +115,96 @@ int32_t ffa_handle_msg_send2(struct cpu_user_regs *regs)
          src_msg->msg_offset < sizeof(struct ffa_part_msg_rxtx) )
     {
         ret = FFA_RET_INVALID_PARAMETERS;
-        goto out_unlock_tx;
+        goto out;
     }
 
-    ret = ffa_simple_call(FFA_MSG_SEND2,
+    if ( FFA_ID_IS_SECURE(dst_id) )
+    {
+        /* Message for a secure partition */
+        if ( !ffa_fw_supports_fid(FFA_MSG_SEND2) )
+        {
+            ret = FFA_RET_NOT_SUPPORTED;
+            goto out;
+        }

xen/arch/arm/tee/ffa_msg.c int32_t ffa_handle_msg_send2(struct cpu_user_regs *regs) · line 137

Quoted 10 lines
xen/arch/arm/tee/ffa_msg.c +7 -1
+
+        ret = ffa_simple_call(FFA_MSG_SEND2,
                           ((uint32_t)ffa_get_vm_id(src_d)) << 16, 0, 0, 0);
+        goto out;
+    }
 
-out_unlock_tx:
+    /* Message for a VM */
+    if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )
+    {

I would move this block into a helper function to isolate the needed cleanup etc, but that might be more a matter of taste so do as you prefer.

Quoted 78 lines
xen/arch/arm/tee/ffa_msg.c +70 -0
+        struct domain *dst_d;
+        struct ffa_ctx *dst_ctx;
+        struct ffa_part_msg_rxtx *dst_msg;
+        int err;
+
+        if ( dst_id == 0 )
+        {
+            /* FF-A ID 0 is the hypervisor, this is not valid */
+            ret = FFA_RET_INVALID_PARAMETERS;
+            goto out;
+        }
+
+        /* This is also checking that dest is not src */
+        err = rcu_lock_live_remote_domain_by_id(dst_id - 1, &dst_d);
+        if ( err )
+        {
+            ret = FFA_RET_INVALID_PARAMETERS;
+            goto out;
+        }
+
+        if ( dst_d->arch.tee == NULL )
+        {
+            ret = FFA_RET_INVALID_PARAMETERS;
+            goto out_unlock;
+        }
+
+        dst_ctx = dst_d->arch.tee;
+        if ( !dst_ctx->guest_vers )
+        {
+            ret = FFA_RET_INVALID_PARAMETERS;
+            goto out_unlock;
+        }
+
+        /* This also checks that destination has set a Rx buffer */
+        ret = ffa_rx_acquire(dst_d);
+        if ( ret )
+            goto out_unlock;
+
+        /* we need to have enough space in the destination buffer */
+        if ( dst_ctx->page_count * FFA_PAGE_SIZE <
+                (sizeof(struct ffa_part_msg_rxtx) + src_msg->msg_size) )
+        {
+            ret = FFA_RET_NO_MEMORY;
+            ffa_rx_release(dst_d);
+            goto out_unlock;
+        }
+
+        dst_msg = dst_ctx->rx;
+
+        /* prepare destination header */
+        dst_msg->flags = 0;
+        dst_msg->reserved = 0;
+        dst_msg->msg_offset = sizeof(struct ffa_part_msg_rxtx);
+        dst_msg->send_recv_id = src_msg->send_recv_id;
+        dst_msg->msg_size = src_msg->msg_size;
+
+        memcpy(dst_ctx->rx + sizeof(struct ffa_part_msg_rxtx),
+               src_ctx->tx + src_msg->msg_offset, src_msg->msg_size);
+
+        /* receiver rx buffer will be released by the receiver*/
+
+out_unlock:
+        rcu_unlock_domain(dst_d);
+        if ( !ret )
+            ffa_raise_rx_buffer_full(dst_d);
+    }
+    else
+        ret = FFA_RET_INVALID_PARAMETERS;
+
+out:
     spin_unlock(&src_ctx->tx_lock);
     return ret;
 }
xen/arch/arm/tee/ffa_private.h +0 -0
index 1f5067d5d0c9..340db229453c 100644

xen/arch/arm/tee/ffa_private.h int ffa_handle_notification_set(struct cpu_user_regs *regs); · line 386

Quoted 7 lines
xen/arch/arm/tee/ffa_private.h +4 -0
@@ -380,6 +380,10 @@ int ffa_handle_notification_set(struct cpu_user_regs *regs);
 
 #ifdef CONFIG_FFA_VM_TO_VM
 void ffa_raise_rx_buffer_full(struct domain *d);
+#else
+static inline void ffa_raise_rx_buffer_full(struct domain *d)
+{
+}

Shouldn't this go in the previous patch "xen/arm: ffa: Add buffer full notification support"?

Cheers,
Jens

Quoted 6 lines
xen/arch/arm/tee/ffa_private.h +0 -1
 #endif
 
 void ffa_handle_msg_send_direct_req(struct cpu_user_regs *regs, uint32_t fid);
--

2.47.1

#14 lore raw

Re: [PATCH v2 4/5] xen/arm: ffa: Add indirect message between VM

Hi Jens,

Quoted 69 lines · Jens Wiklander

Hi Bertrand,

Quoted 63 lines · Bertrand Marquis

Add support for indirect messages between VMs. This is only enabled if CONFIG_FFA_VM_TO_VM is selected.

---
Changes in v2:
- Switch ifdef to IS_ENABLED
---
xen/arch/arm/tee/ffa_msg.c | 96 +++++++++++++++++++++++++++++++---
xen/arch/arm/tee/ffa_private.h | 4 ++

2 files changed, 92 insertions(+), 8 deletions(-)
xen/arch/arm/tee/ffa_msg.c +15 -7
index ee594e737fc7..336d5bbf64f6 100644
@@ -96,9 +96,6 @@ int32_t ffa_handle_msg_send2(struct cpu_user_regs *regs)
    uint16_t dst_id, src_id;
    int32_t ret;
 
-    if ( !ffa_fw_supports_fid(FFA_MSG_SEND2) )
-        return FFA_RET_NOT_SUPPORTED;
-
    if ( !spin_trylock(&src_ctx->tx_lock) )
        return FFA_RET_BUSY;
 
@@ -106,10 +103,10 @@ int32_t ffa_handle_msg_send2(struct cpu_user_regs *regs)
    src_id = src_msg->send_recv_id >> 16;
    dst_id = src_msg->send_recv_id & GENMASK(15,0);
 
-    if ( src_id != ffa_get_vm_id(src_d) || !FFA_ID_IS_SECURE(dst_id) )
+    if ( src_id != ffa_get_vm_id(src_d) )
    {
        ret = FFA_RET_INVALID_PARAMETERS;
-        goto out_unlock_tx;
+        goto out;
    }
 
    /* check source message fits in buffer */
@@ -118,13 +115,96 @@ int32_t ffa_handle_msg_send2(struct cpu_user_regs *regs)
         src_msg->msg_offset < sizeof(struct ffa_part_msg_rxtx) )
    {
        ret = FFA_RET_INVALID_PARAMETERS;
-        goto out_unlock_tx;
+        goto out;
    }
 
-    ret = ffa_simple_call(FFA_MSG_SEND2,
+    if ( FFA_ID_IS_SECURE(dst_id) )
+    {
+        /* Message for a secure partition */
+        if ( !ffa_fw_supports_fid(FFA_MSG_SEND2) )
+        {
+            ret = FFA_RET_NOT_SUPPORTED;
+            goto out;
+        }
+
+        ret = ffa_simple_call(FFA_MSG_SEND2,
                          ((uint32_t)ffa_get_vm_id(src_d)) << 16, 0, 0, 0);
+        goto out;
+    }
 
Quoted 8 lines

xen/arch/arm/tee/ffa_msg.c int32_t ffa_handle_msg_send2(struct cpu_user_regs *regs) · line 137

Quoted 4 lines
xen/arch/arm/tee/ffa_msg.c +3 -1
-out_unlock_tx:
+    /* Message for a VM */
+    if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )
+    {

I would move this block into a helper function to isolate the needed cleanup etc, but that might be more a matter of taste so do as you prefer.

Yes that would be better. Will do.

Quoted 80 lines
Quoted 80 lines
xen/arch/arm/tee/ffa_msg.c +70 -0
+        struct domain *dst_d;
+        struct ffa_ctx *dst_ctx;
+        struct ffa_part_msg_rxtx *dst_msg;
+        int err;
+
+        if ( dst_id == 0 )
+        {
+            /* FF-A ID 0 is the hypervisor, this is not valid */
+            ret = FFA_RET_INVALID_PARAMETERS;
+            goto out;
+        }
+
+        /* This is also checking that dest is not src */
+        err = rcu_lock_live_remote_domain_by_id(dst_id - 1, &dst_d);
+        if ( err )
+        {
+            ret = FFA_RET_INVALID_PARAMETERS;
+            goto out;
+        }
+
+        if ( dst_d->arch.tee == NULL )
+        {
+            ret = FFA_RET_INVALID_PARAMETERS;
+            goto out_unlock;
+        }
+
+        dst_ctx = dst_d->arch.tee;
+        if ( !dst_ctx->guest_vers )
+        {
+            ret = FFA_RET_INVALID_PARAMETERS;
+            goto out_unlock;
+        }
+
+        /* This also checks that destination has set a Rx buffer */
+        ret = ffa_rx_acquire(dst_d);
+        if ( ret )
+            goto out_unlock;
+
+        /* we need to have enough space in the destination buffer */
+        if ( dst_ctx->page_count * FFA_PAGE_SIZE <
+                (sizeof(struct ffa_part_msg_rxtx) + src_msg->msg_size) )
+        {
+            ret = FFA_RET_NO_MEMORY;
+            ffa_rx_release(dst_d);
+            goto out_unlock;
+        }
+
+        dst_msg = dst_ctx->rx;
+
+        /* prepare destination header */
+        dst_msg->flags = 0;
+        dst_msg->reserved = 0;
+        dst_msg->msg_offset = sizeof(struct ffa_part_msg_rxtx);
+        dst_msg->send_recv_id = src_msg->send_recv_id;
+        dst_msg->msg_size = src_msg->msg_size;
+
+        memcpy(dst_ctx->rx + sizeof(struct ffa_part_msg_rxtx),
+               src_ctx->tx + src_msg->msg_offset, src_msg->msg_size);
+
+        /* receiver rx buffer will be released by the receiver*/
+
+out_unlock:
+        rcu_unlock_domain(dst_d);
+        if ( !ret )
+            ffa_raise_rx_buffer_full(dst_d);
+    }
+    else
+        ret = FFA_RET_INVALID_PARAMETERS;
+
+out:
    spin_unlock(&src_ctx->tx_lock);
    return ret;
 }
xen/arch/arm/tee/ffa_private.h +0 -0
index 1f5067d5d0c9..340db229453c 100644
@@ -380,6 +380,10 @@ int ffa_handle_notification_set(struct cpu_user_regs *regs);
 
Quoted 10 lines

xen/arch/arm/tee/ffa_private.h int ffa_handle_notification_set(struct cpu_user_regs *regs); · line 386

Quoted 6 lines
xen/arch/arm/tee/ffa_private.h +4 -0
 #ifdef CONFIG_FFA_VM_TO_VM
 void ffa_raise_rx_buffer_full(struct domain *d);
+#else
+static inline void ffa_raise_rx_buffer_full(struct domain *d)
+{
+}

Shouldn't this go in the previous patch "xen/arm: ffa: Add buffer full notification support"?

Definitely yes. I will move it back.

Cheers
Bertrand

Quoted 8 lines

Cheers,
Jens

Quoted 5 lines
xen/arch/arm/tee/ffa_private.h +0 -1
 #endif
 
 void ffa_handle_msg_send_direct_req(struct cpu_user_regs *regs, uint32_t fid);
--

2.47.1

#15 lore raw

PATCH v2 5/5 xen/arm: ffa: Enable VM to VM without firmware

In reply to Bertrand Marquis

When VM to VM support is activated and there is no suitable FF-A support in the firmware, enable FF-A support for VMs to allow using it for VM to VM communications.
If there is Optee running in the secure world and using the non FF-A communication system, having CONFIG_FFA_VM_TO_VM could be non functional (if optee is probed first) or Optee could be non functional (if FF-A is probed first) so it is not recommended to activate the configuration option for such systems.

To make buffer full notification work between VMs when there is not firmware, rework the notification handling and modify the global flag to only be used as check for firmware notification support instead.

Modify part_info_get to return the list of VMs when there is no firmware support.

---
Changes in v2:
- replace ifdef with IS_ENABLED when possible

---
 xen/arch/arm/tee/ffa.c          |  12 +++-
 xen/arch/arm/tee/ffa_notif.c    | 114 ++++++++++++++++----------------
 xen/arch/arm/tee/ffa_partinfo.c |   3 +-
 3 files changed, 69 insertions(+), 60 deletions(-)
xen/arch/arm/tee/ffa.c +11 -1
index 3bbdd7168a6b..f6582d2e855a 100644
@@ -324,8 +324,9 @@ static int ffa_domain_init(struct domain *d)
     struct ffa_ctx *ctx;
     int ret;
 
-    if ( !ffa_fw_version )
+    if ( !IS_ENABLED(CONFIG_FFA_VM_TO_VM) && !ffa_fw_version )
         return -ENODEV;
+
     /*
      * We are using the domain_id + 1 as the FF-A ID for VMs as FF-A ID 0 is
      * reserved for the hypervisor and we only support secure endpoints using
@@ -549,6 +550,15 @@ err_no_fw:
     bitmap_zero(ffa_fw_abi_supported, FFA_ABI_BITMAP_SIZE);
     printk(XENLOG_WARNING "ARM FF-A No firmware support\n");
 
+    if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )
+    {
+        INIT_LIST_HEAD(&ffa_teardown_head);
+        init_timer(&ffa_teardown_timer, ffa_teardown_timer_callback, NULL, 0);
+
+        printk(XENLOG_INFO "ARM FF-A only available between VMs\n");
+        return true;
+    }
+
     return false;
 }
 
xen/arch/arm/tee/ffa_notif.c +56 -58

206 lines not on this page · raw message

xen/arch/arm/tee/ffa_partinfo.c +2 -1
index 7af1eca2d0c4..291396c8f635 100644
@@ -130,7 +130,8 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
         goto out;
     }
 
-    if ( !ffa_fw_supports_fid(FFA_PARTITION_INFO_GET) )
+    if ( !IS_ENABLED(CONFIG_FFA_VM_TO_VM) &&
+         !ffa_fw_supports_fid(FFA_PARTITION_INFO_GET) )
     {
         /* Just give an empty partition list to the caller */
         ret = FFA_RET_OK;
Signature · 1 line
2.47.1
#16 lore raw

Re: [PATCH v2 5/5] xen/arm: ffa: Enable VM to VM without firmware

Hi Bertrand,

Quoted 5 lines · Bertrand Marquis

When VM to VM support is activated and there is no suitable FF-A support in the firmware, enable FF-A support for VMs to allow using it for VM to VM communications.
If there is Optee running in the secure world and using the non FF-A

It's spelled OP-TEE in text, and optee or OPTEE in identifiers.

Quoted 6 lines

communication system, having CONFIG_FFA_VM_TO_VM could be non functional (if optee is probed first) or Optee could be non functional (if FF-A is probed first) so it is not recommended to activate the configuration option for such systems.

To make buffer full notification work between VMs when there is not

s/not/no/

Quoted 70 lines

firmware, rework the notification handling and modify the global flag to only be used as check for firmware notification support instead.

Modify part_info_get to return the list of VMs when there is no firmware support.

---
Changes in v2:
- replace ifdef with IS_ENABLED when possible

---
 xen/arch/arm/tee/ffa.c          |  12 +++-
 xen/arch/arm/tee/ffa_notif.c    | 114 ++++++++++++++++----------------
 xen/arch/arm/tee/ffa_partinfo.c |   3 +-
 3 files changed, 69 insertions(+), 60 deletions(-)
xen/arch/arm/tee/ffa.c +11 -1
index 3bbdd7168a6b..f6582d2e855a 100644
@@ -324,8 +324,9 @@ static int ffa_domain_init(struct domain *d)
     struct ffa_ctx *ctx;
     int ret;
 
-    if ( !ffa_fw_version )
+    if ( !IS_ENABLED(CONFIG_FFA_VM_TO_VM) && !ffa_fw_version )
         return -ENODEV;
+
     /*
      * We are using the domain_id + 1 as the FF-A ID for VMs as FF-A ID 0 is
      * reserved for the hypervisor and we only support secure endpoints using
@@ -549,6 +550,15 @@ err_no_fw:
     bitmap_zero(ffa_fw_abi_supported, FFA_ABI_BITMAP_SIZE);
     printk(XENLOG_WARNING "ARM FF-A No firmware support\n");
 
+    if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )
+    {
+        INIT_LIST_HEAD(&ffa_teardown_head);
+        init_timer(&ffa_teardown_timer, ffa_teardown_timer_callback, NULL, 0);
+
+        printk(XENLOG_INFO "ARM FF-A only available between VMs\n");
+        return true;
+    }
+
     return false;
 }
 
xen/arch/arm/tee/ffa_notif.c +1 -4
index d19aa5c5bef6..0673e53a9def 100644
@@ -16,7 +16,7 @@
 
 #include "ffa_private.h"
 
-static bool __ro_after_init notif_enabled;
+static bool __ro_after_init fw_notif_enabled;
 static unsigned int __ro_after_init notif_sri_irq;
 
 int ffa_handle_notification_bind(struct cpu_user_regs *regs)
@@ -27,21 +27,17 @@ int ffa_handle_notification_bind(struct cpu_user_regs *regs)
     uint32_t bitmap_lo = get_user_reg(regs, 3);
     uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
     if ( (src_dst & 0xFFFFU) != ffa_get_vm_id(d) )
         return FFA_RET_INVALID_PARAMETERS;
 

xen/arch/arm/tee/ffa_notif.c int ffa_handle_notification_bind(struct cpu_user_regs *regs) · line 36

Quoted 10 lines
xen/arch/arm/tee/ffa_notif.c +1 -6
     if ( flags )    /* Only global notifications are supported */
         return FFA_RET_DENIED;
 
-    /*
-     * We only support notifications from SP so no need to check the sender
-     * endpoint ID, the SPMC will take care of that for us.
-     */
-    return ffa_simple_call(FFA_NOTIFICATION_BIND, src_dst, flags, bitmap_lo,
-                           bitmap_hi);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )

Please add space before and after '>>', here and in the function below in this patch.

Quoted 38 lines
xen/arch/arm/tee/ffa_notif.c +11 -10
+        return ffa_simple_call(FFA_NOTIFICATION_BIND, src_dst, flags,
+                               bitmap_lo, bitmap_hi);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 int ffa_handle_notification_unbind(struct cpu_user_regs *regs)
@@ -51,32 +47,34 @@ int ffa_handle_notification_unbind(struct cpu_user_regs *regs)
     uint32_t bitmap_lo = get_user_reg(regs, 3);
     uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
     if ( (src_dst & 0xFFFFU) != ffa_get_vm_id(d) )
         return FFA_RET_INVALID_PARAMETERS;
 
-    /*
-     * We only support notifications from SP so no need to check the
-     * destination endpoint ID, the SPMC will take care of that for us.
-     */
-    return  ffa_simple_call(FFA_NOTIFICATION_UNBIND, src_dst, 0, bitmap_lo,
-                            bitmap_hi);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )
+        return  ffa_simple_call(FFA_NOTIFICATION_UNBIND, src_dst, 0, bitmap_lo,
+                                bitmap_hi);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 void ffa_handle_notification_info_get(struct cpu_user_regs *regs)
 {
     struct domain *d = current->domain;
     struct ffa_ctx *ctx = d->arch.tee;
+    bool notif_pending = false;
 
-    if ( !notif_enabled )
+    if ( !IS_ENABLED(CONFIG_FFA_VM_TO_VM) && !fw_notif_enabled )

xen/arch/arm/tee/ffa_notif.c int ffa_handle_notification_unbind(struct cpu_user_regs *regs) · line 75

Quoted 10 lines
xen/arch/arm/tee/ffa_notif.c +4 -1
     {
         ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
         return;
     }
 
-    if ( test_and_clear_bool(ctx->notif.secure_pending) )
+    notif_pending = ctx->notif.secure_pending;
+#ifdef CONFIG_FFA_VM_TO_VM
+    notif_pending |= ctx->notif.buff_full_pending;
+#endif

Shouldn't ctx->notif.secure_pending and ctx->notif.secure_pending be cleared also, like:
notif_pending = test_and_clear_bool(ctx->notif.secure_pending) ||

                test_and_clear_bool(ctx->notif.buff_full_pending);
Quoted 15 lines
xen/arch/arm/tee/ffa_notif.c +3 -1
+
+    if ( notif_pending )
     {
         /* A pending global notification for the guest */
         ffa_set_regs(regs, FFA_SUCCESS_64, 0,
@@ -103,7 +101,7 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
     uint32_t w6 = 0;
     uint32_t w7 = 0;
 
-    if ( !notif_enabled )
+    if ( !IS_ENABLED(CONFIG_FFA_VM_TO_VM) && !fw_notif_enabled )
     {
         ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
         return;

xen/arch/arm/tee/ffa_notif.c void ffa_handle_notification_get(struct cpu_user_regs *regs) · line 117

Quoted 6 lines
xen/arch/arm/tee/ffa_notif.c +2 -1
@@ -115,7 +113,8 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
         return;
     }
 
-    if ( flags & ( FFA_NOTIF_FLAG_BITMAP_SP | FFA_NOTIF_FLAG_BITMAP_SPM ) )
+    if ( fw_notif_enabled && (flags & ( FFA_NOTIF_FLAG_BITMAP_SP
+                                        | FFA_NOTIF_FLAG_BITMAP_SPM )) )

Please end the previous line with the '|' operator.

Quoted 25 lines
xen/arch/arm/tee/ffa_notif.c +5 -6
     {
         struct arm_smccc_1_2_regs arg = {
             .a0 = FFA_NOTIFICATION_GET,
@@ -170,15 +169,14 @@ int ffa_handle_notification_set(struct cpu_user_regs *regs)
     uint32_t bitmap_lo = get_user_reg(regs, 3);
     uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
     if ( (src_dst >> 16) != ffa_get_vm_id(d) )
         return FFA_RET_INVALID_PARAMETERS;
 
-    /* Let the SPMC check the destination of the notification */
-    return ffa_simple_call(FFA_NOTIFICATION_SET, src_dst, flags, bitmap_lo,
-                           bitmap_hi);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )
+        return ffa_simple_call(FFA_NOTIFICATION_SET, src_dst, flags, bitmap_lo,
+                               bitmap_hi);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 #ifdef CONFIG_FFA_VM_TO_VM

xen/arch/arm/tee/ffa_notif.c void ffa_raise_rx_buffer_full(struct domain *d) · line 191

Quoted 5 lines
xen/arch/arm/tee/ffa_notif.c +1 -1
@@ -190,7 +188,7 @@ void ffa_raise_rx_buffer_full(struct domain *d)
         return;
 
     if ( !test_and_set_bool(ctx->notif.buff_full_pending) )
-        vgic_inject_irq(d, d->vcpu[0], notif_sri_irq, true);
+        vgic_inject_irq(d, d->vcpu[0], GUEST_FFA_NOTIF_PEND_INTR_ID, true);

Please move this to the patch "xen/arm: ffa: Add buffer full notification support"

Quoted 93 lines
xen/arch/arm/tee/ffa_notif.c +28 -28
 }
 #endif
 
@@ -363,7 +361,7 @@ void ffa_notif_init_interrupt(void)
 {
     int ret;
 
-    if ( notif_enabled && notif_sri_irq < NR_GIC_SGI )
+    if ( fw_notif_enabled && notif_sri_irq < NR_GIC_SGI )
     {
         /*
          * An error here is unlikely since the primary CPU has already
@@ -394,41 +392,41 @@ void ffa_notif_init(void)
     int ret;
 
     /* Only enable fw notification if all ABIs we need are supported */
-    if ( !(ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_CREATE) &&
-           ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_DESTROY) &&
-           ffa_fw_supports_fid(FFA_NOTIFICATION_GET) &&
-           ffa_fw_supports_fid(FFA_NOTIFICATION_INFO_GET_64)) )
-        return;
-
-    arm_smccc_1_2_smc(&arg, &resp);
-    if ( resp.a0 != FFA_SUCCESS_32 )
-        return;
-
-    irq = resp.a2;
-    notif_sri_irq = irq;
-    if ( irq >= NR_GIC_SGI )
-        irq_set_type(irq, IRQ_TYPE_EDGE_RISING);
-    ret = request_irq(irq, 0, notif_irq_handler, "FF-A notif", NULL);
-    if ( ret )
+    if ( ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_CREATE) &&
+         ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_DESTROY) &&
+         ffa_fw_supports_fid(FFA_NOTIFICATION_GET) &&
+         ffa_fw_supports_fid(FFA_NOTIFICATION_INFO_GET_64) )
     {
-        printk(XENLOG_ERR "ffa: request_irq irq %u failed: error %d\n",
-               irq, ret);
-        return;
-    }
+        arm_smccc_1_2_smc(&arg, &resp);
+        if ( resp.a0 != FFA_SUCCESS_32 )
+            return;
 
-    notif_enabled = true;
+        irq = resp.a2;
+        notif_sri_irq = irq;
+        if ( irq >= NR_GIC_SGI )
+            irq_set_type(irq, IRQ_TYPE_EDGE_RISING);
+        ret = request_irq(irq, 0, notif_irq_handler, "FF-A notif", NULL);
+        if ( ret )
+        {
+            printk(XENLOG_ERR "ffa: request_irq irq %u failed: error %d\n",
+                   irq, ret);
+            return;
+        }
+        fw_notif_enabled = true;
+    }
 }
 
 int ffa_notif_domain_init(struct domain *d)
 {
     int32_t res;
 
-    if ( !notif_enabled )
-        return 0;
+    if ( fw_notif_enabled )
+    {
 
-    res = ffa_notification_bitmap_create(ffa_get_vm_id(d), d->max_vcpus);
-    if ( res )
-        return -ENOMEM;
+        res = ffa_notification_bitmap_create(ffa_get_vm_id(d), d->max_vcpus);
+        if ( res )
+            return -ENOMEM;
+    }
 
     return 0;
 }
@@ -439,6 +437,6 @@ void ffa_notif_domain_destroy(struct domain *d)
      * Call bitmap_destroy even if bitmap create failed as the SPMC will
      * return a DENIED error that we will ignore.
      */
-    if ( notif_enabled )
+    if ( fw_notif_enabled )
         ffa_notification_bitmap_destroy(ffa_get_vm_id(d));
 }
xen/arch/arm/tee/ffa_partinfo.c +0 -0
index 7af1eca2d0c4..291396c8f635 100644

xen/arch/arm/tee/ffa_partinfo.c void ffa_handle_partition_info_get(struct cpu_user_regs *regs) · line 133

Quoted 5 lines
xen/arch/arm/tee/ffa_partinfo.c +1 -1
@@ -130,7 +130,8 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
         goto out;
     }
 
-    if ( !ffa_fw_supports_fid(FFA_PARTITION_INFO_GET) )
+    if ( !IS_ENABLED(CONFIG_FFA_VM_TO_VM) &&

Doesn't this belong in the patch "xen/arm: ffa: Introduce VM to VM support"? And wouldn't ffa_vm_count make more sense?

Cheers,
Jens

Quoted 7 lines
xen/arch/arm/tee/ffa_partinfo.c +1 -1
+         !ffa_fw_supports_fid(FFA_PARTITION_INFO_GET) )
     {
         /* Just give an empty partition list to the caller */
         ret = FFA_RET_OK;
--

2.47.1

#17 lore raw

Re: [PATCH v2 5/5] xen/arm: ffa: Enable VM to VM without firmware

Hi Jens,

Quoted 7 lines · Jens Wiklander

Hi Bertrand,

Quoted 1 line · Bertrand Marquis
Quoted 6 lines
Quoted 4 lines

When VM to VM support is activated and there is no suitable FF-A support in the firmware, enable FF-A support for VMs to allow using it for VM to VM communications.
If there is Optee running in the secure world and using the non FF-A

It's spelled OP-TEE in text, and optee or OPTEE in identifiers.

ack

Quoted 9 lines
Quoted 7 lines

communication system, having CONFIG_FFA_VM_TO_VM could be non functional (if optee is probed first) or Optee could be non functional (if FF-A is probed first) so it is not recommended to activate the configuration option for such systems.

To make buffer full notification work between VMs when there is not

s/not/no/

ack

Quoted 74 lines
Quoted 74 lines

firmware, rework the notification handling and modify the global flag to only be used as check for firmware notification support instead.

Modify part_info_get to return the list of VMs when there is no firmware support.

---
Changes in v2:
- replace ifdef with IS_ENABLED when possible
---

xen/arch/arm/tee/ffa.c          |  12 +++-
xen/arch/arm/tee/ffa_notif.c    | 114 ++++++++++++++++----------------
xen/arch/arm/tee/ffa_partinfo.c |   3 +-
3 files changed, 69 insertions(+), 60 deletions(-)
xen/arch/arm/tee/ffa.c +11 -1
index 3bbdd7168a6b..f6582d2e855a 100644
@@ -324,8 +324,9 @@ static int ffa_domain_init(struct domain *d)
    struct ffa_ctx *ctx;
    int ret;
 
-    if ( !ffa_fw_version )
+    if ( !IS_ENABLED(CONFIG_FFA_VM_TO_VM) && !ffa_fw_version )
        return -ENODEV;
+
    /*
     * We are using the domain_id + 1 as the FF-A ID for VMs as FF-A ID 0 is
     * reserved for the hypervisor and we only support secure endpoints using
@@ -549,6 +550,15 @@ err_no_fw:
    bitmap_zero(ffa_fw_abi_supported, FFA_ABI_BITMAP_SIZE);
    printk(XENLOG_WARNING "ARM FF-A No firmware support\n");
 
+    if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )
+    {
+        INIT_LIST_HEAD(&ffa_teardown_head);
+        init_timer(&ffa_teardown_timer, ffa_teardown_timer_callback, NULL, 0);
+
+        printk(XENLOG_INFO "ARM FF-A only available between VMs\n");
+        return true;
+    }
+
    return false;
 }
 
xen/arch/arm/tee/ffa_notif.c +1 -4
index d19aa5c5bef6..0673e53a9def 100644
@@ -16,7 +16,7 @@
 
 #include "ffa_private.h"
 
-static bool __ro_after_init notif_enabled;
+static bool __ro_after_init fw_notif_enabled;
 static unsigned int __ro_after_init notif_sri_irq;
 
 int ffa_handle_notification_bind(struct cpu_user_regs *regs)
@@ -27,21 +27,17 @@ int ffa_handle_notification_bind(struct cpu_user_regs *regs)
    uint32_t bitmap_lo = get_user_reg(regs, 3);
    uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
    if ( (src_dst & 0xFFFFU) != ffa_get_vm_id(d) )
        return FFA_RET_INVALID_PARAMETERS;
 
    if ( flags )    /* Only global notifications are supported */
        return FFA_RET_DENIED;
 
Quoted 10 lines

xen/arch/arm/tee/ffa_notif.c int ffa_handle_notification_bind(struct cpu_user_regs *regs) · line 36

Quoted 7 lines
xen/arch/arm/tee/ffa_notif.c +1 -6
-    /*
-     * We only support notifications from SP so no need to check the sender
-     * endpoint ID, the SPMC will take care of that for us.
-     */
-    return ffa_simple_call(FFA_NOTIFICATION_BIND, src_dst, flags, bitmap_lo,
-                           bitmap_hi);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )

Please add space before and after '>>', here and in the function below in this patch.

ack

Quoted 53 lines
Quoted 39 lines
xen/arch/arm/tee/ffa_notif.c +11 -10
+        return ffa_simple_call(FFA_NOTIFICATION_BIND, src_dst, flags,
+                               bitmap_lo, bitmap_hi);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 int ffa_handle_notification_unbind(struct cpu_user_regs *regs)
@@ -51,32 +47,34 @@ int ffa_handle_notification_unbind(struct cpu_user_regs *regs)
    uint32_t bitmap_lo = get_user_reg(regs, 3);
    uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
    if ( (src_dst & 0xFFFFU) != ffa_get_vm_id(d) )
        return FFA_RET_INVALID_PARAMETERS;
 
-    /*
-     * We only support notifications from SP so no need to check the
-     * destination endpoint ID, the SPMC will take care of that for us.
-     */
-    return  ffa_simple_call(FFA_NOTIFICATION_UNBIND, src_dst, 0, bitmap_lo,
-                            bitmap_hi);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )
+        return  ffa_simple_call(FFA_NOTIFICATION_UNBIND, src_dst, 0, bitmap_lo,
+                                bitmap_hi);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 void ffa_handle_notification_info_get(struct cpu_user_regs *regs)
 {
    struct domain *d = current->domain;
    struct ffa_ctx *ctx = d->arch.tee;
+    bool notif_pending = false;
 
-    if ( !notif_enabled )
+    if ( !IS_ENABLED(CONFIG_FFA_VM_TO_VM) && !fw_notif_enabled )

xen/arch/arm/tee/ffa_notif.c int ffa_handle_notification_unbind(struct cpu_user_regs *regs) · line 75

Quoted 10 lines
xen/arch/arm/tee/ffa_notif.c +4 -1
    {
        ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
        return;
    }
 
-    if ( test_and_clear_bool(ctx->notif.secure_pending) )
+    notif_pending = ctx->notif.secure_pending;
+#ifdef CONFIG_FFA_VM_TO_VM
+    notif_pending |= ctx->notif.buff_full_pending;
+#endif

Shouldn't ctx->notif.secure_pending and ctx->notif.secure_pending be cleared also, like:
notif_pending = test_and_clear_bool(ctx->notif.secure_pending) ||

Quoted 1 line
               test_and_clear_bool(ctx->notif.buff_full_pending);

Notification info get is returning who has pending notification but only notification get should erase pending notifications.

Quoted 19 lines
Quoted 19 lines
xen/arch/arm/tee/ffa_notif.c +3 -1
+
+    if ( notif_pending )
    {
        /* A pending global notification for the guest */
        ffa_set_regs(regs, FFA_SUCCESS_64, 0,
@@ -103,7 +101,7 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
    uint32_t w6 = 0;
    uint32_t w7 = 0;
 
-    if ( !notif_enabled )
+    if ( !IS_ENABLED(CONFIG_FFA_VM_TO_VM) && !fw_notif_enabled )
    {
        ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
        return;
@@ -115,7 +113,8 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
        return;
    }
 
Quoted 5 lines

xen/arch/arm/tee/ffa_notif.c void ffa_handle_notification_get(struct cpu_user_regs *regs) · line 117

Quoted 3 lines
xen/arch/arm/tee/ffa_notif.c +2 -1
-    if ( flags & ( FFA_NOTIF_FLAG_BITMAP_SP | FFA_NOTIF_FLAG_BITMAP_SPM ) )
+    if ( fw_notif_enabled && (flags & ( FFA_NOTIF_FLAG_BITMAP_SP
+                                        | FFA_NOTIF_FLAG_BITMAP_SPM )) )

Please end the previous line with the '|' operator.

ack

Quoted 28 lines
Quoted 28 lines
xen/arch/arm/tee/ffa_notif.c +5 -6
    {
        struct arm_smccc_1_2_regs arg = {
            .a0 = FFA_NOTIFICATION_GET,
@@ -170,15 +169,14 @@ int ffa_handle_notification_set(struct cpu_user_regs *regs)
    uint32_t bitmap_lo = get_user_reg(regs, 3);
    uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
    if ( (src_dst >> 16) != ffa_get_vm_id(d) )
        return FFA_RET_INVALID_PARAMETERS;
 
-    /* Let the SPMC check the destination of the notification */
-    return ffa_simple_call(FFA_NOTIFICATION_SET, src_dst, flags, bitmap_lo,
-                           bitmap_hi);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )
+        return ffa_simple_call(FFA_NOTIFICATION_SET, src_dst, flags, bitmap_lo,
+                               bitmap_hi);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 #ifdef CONFIG_FFA_VM_TO_VM
@@ -190,7 +188,7 @@ void ffa_raise_rx_buffer_full(struct domain *d)
        return;
 
Quoted 6 lines

xen/arch/arm/tee/ffa_notif.c void ffa_raise_rx_buffer_full(struct domain *d) · line 191

Quoted 3 lines
xen/arch/arm/tee/ffa_notif.c +1 -1
    if ( !test_and_set_bool(ctx->notif.buff_full_pending) )
-        vgic_inject_irq(d, d->vcpu[0], notif_sri_irq, true);
+        vgic_inject_irq(d, d->vcpu[0], GUEST_FFA_NOTIF_PEND_INTR_ID, true);

Please move this to the patch "xen/arm: ffa: Add buffer full notification support"

Ack

Quoted 97 lines
Quoted 97 lines
xen/arch/arm/tee/ffa_notif.c +28 -28
 }
 #endif
 
@@ -363,7 +361,7 @@ void ffa_notif_init_interrupt(void)
 {
    int ret;
 
-    if ( notif_enabled && notif_sri_irq < NR_GIC_SGI )
+    if ( fw_notif_enabled && notif_sri_irq < NR_GIC_SGI )
    {
        /*
         * An error here is unlikely since the primary CPU has already
@@ -394,41 +392,41 @@ void ffa_notif_init(void)
    int ret;
 
    /* Only enable fw notification if all ABIs we need are supported */
-    if ( !(ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_CREATE) &&
-           ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_DESTROY) &&
-           ffa_fw_supports_fid(FFA_NOTIFICATION_GET) &&
-           ffa_fw_supports_fid(FFA_NOTIFICATION_INFO_GET_64)) )
-        return;
-
-    arm_smccc_1_2_smc(&arg, &resp);
-    if ( resp.a0 != FFA_SUCCESS_32 )
-        return;
-
-    irq = resp.a2;
-    notif_sri_irq = irq;
-    if ( irq >= NR_GIC_SGI )
-        irq_set_type(irq, IRQ_TYPE_EDGE_RISING);
-    ret = request_irq(irq, 0, notif_irq_handler, "FF-A notif", NULL);
-    if ( ret )
+    if ( ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_CREATE) &&
+         ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_DESTROY) &&
+         ffa_fw_supports_fid(FFA_NOTIFICATION_GET) &&
+         ffa_fw_supports_fid(FFA_NOTIFICATION_INFO_GET_64) )
    {
-        printk(XENLOG_ERR "ffa: request_irq irq %u failed: error %d\n",
-               irq, ret);
-        return;
-    }
+        arm_smccc_1_2_smc(&arg, &resp);
+        if ( resp.a0 != FFA_SUCCESS_32 )
+            return;
 
-    notif_enabled = true;
+        irq = resp.a2;
+        notif_sri_irq = irq;
+        if ( irq >= NR_GIC_SGI )
+            irq_set_type(irq, IRQ_TYPE_EDGE_RISING);
+        ret = request_irq(irq, 0, notif_irq_handler, "FF-A notif", NULL);
+        if ( ret )
+        {
+            printk(XENLOG_ERR "ffa: request_irq irq %u failed: error %d\n",
+                   irq, ret);
+            return;
+        }
+        fw_notif_enabled = true;
+    }
 }
 
 int ffa_notif_domain_init(struct domain *d)
 {
    int32_t res;
 
-    if ( !notif_enabled )
-        return 0;
+    if ( fw_notif_enabled )
+    {
 
-    res = ffa_notification_bitmap_create(ffa_get_vm_id(d), d->max_vcpus);
-    if ( res )
-        return -ENOMEM;
+        res = ffa_notification_bitmap_create(ffa_get_vm_id(d), d->max_vcpus);
+        if ( res )
+            return -ENOMEM;
+    }
 
    return 0;
 }
@@ -439,6 +437,6 @@ void ffa_notif_domain_destroy(struct domain *d)
     * Call bitmap_destroy even if bitmap create failed as the SPMC will
     * return a DENIED error that we will ignore.
     */
-    if ( notif_enabled )
+    if ( fw_notif_enabled )
        ffa_notification_bitmap_destroy(ffa_get_vm_id(d));
 }
xen/arch/arm/tee/ffa_partinfo.c +0 -0
index 7af1eca2d0c4..291396c8f635 100644
@@ -130,7 +130,8 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
        goto out;
    }
 
Quoted 5 lines

xen/arch/arm/tee/ffa_partinfo.c void ffa_handle_partition_info_get(struct cpu_user_regs *regs) · line 133

Quoted 2 lines
xen/arch/arm/tee/ffa_partinfo.c +1 -1
-    if ( !ffa_fw_supports_fid(FFA_PARTITION_INFO_GET) )
+    if ( !IS_ENABLED(CONFIG_FFA_VM_TO_VM) &&

Doesn't this belong in the patch "xen/arm: ffa: Introduce VM to VM support"? And wouldn't ffa_vm_count make more sense?

ack. I will modify that and fold it into the VM to VM support patch.

Cheers
Bertrand

Quoted 10 lines

Cheers,
Jens

Quoted 6 lines
xen/arch/arm/tee/ffa_partinfo.c +1 -1
+         !ffa_fw_supports_fid(FFA_PARTITION_INFO_GET) )
    {
        /* Just give an empty partition list to the caller */
        ret = FFA_RET_OK;
--

2.47.1

#18 lore raw

Re: [PATCH v2 5/5] xen/arm: ffa: Enable VM to VM without firmware

Hi,

Quoted 173 lines · Bertrand Marquis

Hi Jens,

Quoted 7 lines · Jens Wiklander

Hi Bertrand,

Quoted 1 line · Bertrand Marquis
Quoted 6 lines
Quoted 4 lines

When VM to VM support is activated and there is no suitable FF-A support in the firmware, enable FF-A support for VMs to allow using it for VM to VM communications.
If there is Optee running in the secure world and using the non FF-A

It's spelled OP-TEE in text, and optee or OPTEE in identifiers.

ack

Quoted 9 lines
Quoted 7 lines

communication system, having CONFIG_FFA_VM_TO_VM could be non functional (if optee is probed first) or Optee could be non functional (if FF-A is probed first) so it is not recommended to activate the configuration option for such systems.

To make buffer full notification work between VMs when there is not

s/not/no/

ack

Quoted 74 lines
Quoted 74 lines

firmware, rework the notification handling and modify the global flag to only be used as check for firmware notification support instead.

Modify part_info_get to return the list of VMs when there is no firmware support.

---
Changes in v2:
- replace ifdef with IS_ENABLED when possible
---

xen/arch/arm/tee/ffa.c          |  12 +++-
xen/arch/arm/tee/ffa_notif.c    | 114 ++++++++++++++++----------------
xen/arch/arm/tee/ffa_partinfo.c |   3 +-
3 files changed, 69 insertions(+), 60 deletions(-)
xen/arch/arm/tee/ffa.c +11 -1
index 3bbdd7168a6b..f6582d2e855a 100644
@@ -324,8 +324,9 @@ static int ffa_domain_init(struct domain *d)
    struct ffa_ctx *ctx;
    int ret;
 
-    if ( !ffa_fw_version )
+    if ( !IS_ENABLED(CONFIG_FFA_VM_TO_VM) && !ffa_fw_version )
        return -ENODEV;
+
    /*
     * We are using the domain_id + 1 as the FF-A ID for VMs as FF-A ID 0 is
     * reserved for the hypervisor and we only support secure endpoints using
@@ -549,6 +550,15 @@ err_no_fw:
    bitmap_zero(ffa_fw_abi_supported, FFA_ABI_BITMAP_SIZE);
    printk(XENLOG_WARNING "ARM FF-A No firmware support\n");
 
+    if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )
+    {
+        INIT_LIST_HEAD(&ffa_teardown_head);
+        init_timer(&ffa_teardown_timer, ffa_teardown_timer_callback, NULL, 0);
+
+        printk(XENLOG_INFO "ARM FF-A only available between VMs\n");
+        return true;
+    }
+
    return false;
 }
 
xen/arch/arm/tee/ffa_notif.c +1 -4
index d19aa5c5bef6..0673e53a9def 100644
@@ -16,7 +16,7 @@
 
 #include "ffa_private.h"
 
-static bool __ro_after_init notif_enabled;
+static bool __ro_after_init fw_notif_enabled;
 static unsigned int __ro_after_init notif_sri_irq;
 
 int ffa_handle_notification_bind(struct cpu_user_regs *regs)
@@ -27,21 +27,17 @@ int ffa_handle_notification_bind(struct cpu_user_regs *regs)
    uint32_t bitmap_lo = get_user_reg(regs, 3);
    uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
    if ( (src_dst & 0xFFFFU) != ffa_get_vm_id(d) )
        return FFA_RET_INVALID_PARAMETERS;
 
    if ( flags )    /* Only global notifications are supported */
        return FFA_RET_DENIED;
 
Quoted 10 lines

xen/arch/arm/tee/ffa_notif.c int ffa_handle_notification_bind(struct cpu_user_regs *regs) · line 36

Quoted 7 lines
xen/arch/arm/tee/ffa_notif.c +1 -6
-    /*
-     * We only support notifications from SP so no need to check the sender
-     * endpoint ID, the SPMC will take care of that for us.
-     */
-    return ffa_simple_call(FFA_NOTIFICATION_BIND, src_dst, flags, bitmap_lo,
-                           bitmap_hi);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )

Please add space before and after '>>', here and in the function below in this patch.

ack

Quoted 53 lines
Quoted 39 lines
xen/arch/arm/tee/ffa_notif.c +11 -10
+        return ffa_simple_call(FFA_NOTIFICATION_BIND, src_dst, flags,
+                               bitmap_lo, bitmap_hi);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 int ffa_handle_notification_unbind(struct cpu_user_regs *regs)
@@ -51,32 +47,34 @@ int ffa_handle_notification_unbind(struct cpu_user_regs *regs)
    uint32_t bitmap_lo = get_user_reg(regs, 3);
    uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
    if ( (src_dst & 0xFFFFU) != ffa_get_vm_id(d) )
        return FFA_RET_INVALID_PARAMETERS;
 
-    /*
-     * We only support notifications from SP so no need to check the
-     * destination endpoint ID, the SPMC will take care of that for us.
-     */
-    return  ffa_simple_call(FFA_NOTIFICATION_UNBIND, src_dst, 0, bitmap_lo,
-                            bitmap_hi);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )
+        return  ffa_simple_call(FFA_NOTIFICATION_UNBIND, src_dst, 0, bitmap_lo,
+                                bitmap_hi);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 void ffa_handle_notification_info_get(struct cpu_user_regs *regs)
 {
    struct domain *d = current->domain;
    struct ffa_ctx *ctx = d->arch.tee;
+    bool notif_pending = false;
 
-    if ( !notif_enabled )
+    if ( !IS_ENABLED(CONFIG_FFA_VM_TO_VM) && !fw_notif_enabled )

xen/arch/arm/tee/ffa_notif.c int ffa_handle_notification_unbind(struct cpu_user_regs *regs) · line 75

Quoted 10 lines
xen/arch/arm/tee/ffa_notif.c +4 -1
    {
        ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
        return;
    }
 
-    if ( test_and_clear_bool(ctx->notif.secure_pending) )
+    notif_pending = ctx->notif.secure_pending;
+#ifdef CONFIG_FFA_VM_TO_VM
+    notif_pending |= ctx->notif.buff_full_pending;
+#endif

Shouldn't ctx->notif.secure_pending and ctx->notif.secure_pending be cleared also, like:
notif_pending = test_and_clear_bool(ctx->notif.secure_pending) ||

Quoted 1 line
               test_and_clear_bool(ctx->notif.buff_full_pending);
Quoted 2 lines

Notification info get is returning who has pending notification but only notification get should erase pending notifications.

FFA_NOTIFICATION_INFO_GET can return a "More pending notifications flag" in w2/x2 to inform the caller that it should call FFA_NOTIFICATION_INFO_GET again to get the remaining receiver endpoints. How can the ABI know where to resume the next time if the previous pending receivers aren't cleared?

The more pending notifications flag will not be needed here as we're dealing with a single endpoint at a time so it might be more of a philosophical question. I don't think it causes problems for the guest to keep secure_pending unchanged for FFA_NOTIFICATION_INFO_GET, but we should mention the changed behaviour in the commit message.

Cheers,
Jens

Quoted 185 lines
Quoted 20 lines
Quoted 20 lines
xen/arch/arm/tee/ffa_notif.c +3 -1
+
+    if ( notif_pending )
    {
        /* A pending global notification for the guest */
        ffa_set_regs(regs, FFA_SUCCESS_64, 0,
@@ -103,7 +101,7 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
    uint32_t w6 = 0;
    uint32_t w7 = 0;
 
-    if ( !notif_enabled )
+    if ( !IS_ENABLED(CONFIG_FFA_VM_TO_VM) && !fw_notif_enabled )
    {
        ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
        return;
@@ -115,7 +113,8 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
        return;
    }
 
Quoted 5 lines

xen/arch/arm/tee/ffa_notif.c void ffa_handle_notification_get(struct cpu_user_regs *regs) · line 117

Quoted 3 lines
xen/arch/arm/tee/ffa_notif.c +2 -1
-    if ( flags & ( FFA_NOTIF_FLAG_BITMAP_SP | FFA_NOTIF_FLAG_BITMAP_SPM ) )
+    if ( fw_notif_enabled && (flags & ( FFA_NOTIF_FLAG_BITMAP_SP
+                                        | FFA_NOTIF_FLAG_BITMAP_SPM )) )

Please end the previous line with the '|' operator.

ack

Quoted 28 lines
Quoted 28 lines
xen/arch/arm/tee/ffa_notif.c +5 -6
    {
        struct arm_smccc_1_2_regs arg = {
            .a0 = FFA_NOTIFICATION_GET,
@@ -170,15 +169,14 @@ int ffa_handle_notification_set(struct cpu_user_regs *regs)
    uint32_t bitmap_lo = get_user_reg(regs, 3);
    uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
    if ( (src_dst >> 16) != ffa_get_vm_id(d) )
        return FFA_RET_INVALID_PARAMETERS;
 
-    /* Let the SPMC check the destination of the notification */
-    return ffa_simple_call(FFA_NOTIFICATION_SET, src_dst, flags, bitmap_lo,
-                           bitmap_hi);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )
+        return ffa_simple_call(FFA_NOTIFICATION_SET, src_dst, flags, bitmap_lo,
+                               bitmap_hi);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 #ifdef CONFIG_FFA_VM_TO_VM
@@ -190,7 +188,7 @@ void ffa_raise_rx_buffer_full(struct domain *d)
        return;
 
Quoted 6 lines

xen/arch/arm/tee/ffa_notif.c void ffa_raise_rx_buffer_full(struct domain *d) · line 191

Quoted 3 lines
xen/arch/arm/tee/ffa_notif.c +1 -1
    if ( !test_and_set_bool(ctx->notif.buff_full_pending) )
-        vgic_inject_irq(d, d->vcpu[0], notif_sri_irq, true);
+        vgic_inject_irq(d, d->vcpu[0], GUEST_FFA_NOTIF_PEND_INTR_ID, true);

Please move this to the patch "xen/arm: ffa: Add buffer full notification support"

Ack

Quoted 97 lines
Quoted 97 lines
xen/arch/arm/tee/ffa_notif.c +28 -28
 }
 #endif
 
@@ -363,7 +361,7 @@ void ffa_notif_init_interrupt(void)
 {
    int ret;
 
-    if ( notif_enabled && notif_sri_irq < NR_GIC_SGI )
+    if ( fw_notif_enabled && notif_sri_irq < NR_GIC_SGI )
    {
        /*
         * An error here is unlikely since the primary CPU has already
@@ -394,41 +392,41 @@ void ffa_notif_init(void)
    int ret;
 
    /* Only enable fw notification if all ABIs we need are supported */
-    if ( !(ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_CREATE) &&
-           ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_DESTROY) &&
-           ffa_fw_supports_fid(FFA_NOTIFICATION_GET) &&
-           ffa_fw_supports_fid(FFA_NOTIFICATION_INFO_GET_64)) )
-        return;
-
-    arm_smccc_1_2_smc(&arg, &resp);
-    if ( resp.a0 != FFA_SUCCESS_32 )
-        return;
-
-    irq = resp.a2;
-    notif_sri_irq = irq;
-    if ( irq >= NR_GIC_SGI )
-        irq_set_type(irq, IRQ_TYPE_EDGE_RISING);
-    ret = request_irq(irq, 0, notif_irq_handler, "FF-A notif", NULL);
-    if ( ret )
+    if ( ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_CREATE) &&
+         ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_DESTROY) &&
+         ffa_fw_supports_fid(FFA_NOTIFICATION_GET) &&
+         ffa_fw_supports_fid(FFA_NOTIFICATION_INFO_GET_64) )
    {
-        printk(XENLOG_ERR "ffa: request_irq irq %u failed: error %d\n",
-               irq, ret);
-        return;
-    }
+        arm_smccc_1_2_smc(&arg, &resp);
+        if ( resp.a0 != FFA_SUCCESS_32 )
+            return;
 
-    notif_enabled = true;
+        irq = resp.a2;
+        notif_sri_irq = irq;
+        if ( irq >= NR_GIC_SGI )
+            irq_set_type(irq, IRQ_TYPE_EDGE_RISING);
+        ret = request_irq(irq, 0, notif_irq_handler, "FF-A notif", NULL);
+        if ( ret )
+        {
+            printk(XENLOG_ERR "ffa: request_irq irq %u failed: error %d\n",
+                   irq, ret);
+            return;
+        }
+        fw_notif_enabled = true;
+    }
 }
 
 int ffa_notif_domain_init(struct domain *d)
 {
    int32_t res;
 
-    if ( !notif_enabled )
-        return 0;
+    if ( fw_notif_enabled )
+    {
 
-    res = ffa_notification_bitmap_create(ffa_get_vm_id(d), d->max_vcpus);
-    if ( res )
-        return -ENOMEM;
+        res = ffa_notification_bitmap_create(ffa_get_vm_id(d), d->max_vcpus);
+        if ( res )
+            return -ENOMEM;
+    }
 
    return 0;
 }
@@ -439,6 +437,6 @@ void ffa_notif_domain_destroy(struct domain *d)
     * Call bitmap_destroy even if bitmap create failed as the SPMC will
     * return a DENIED error that we will ignore.
     */
-    if ( notif_enabled )
+    if ( fw_notif_enabled )
        ffa_notification_bitmap_destroy(ffa_get_vm_id(d));
 }
xen/arch/arm/tee/ffa_partinfo.c +0 -0
index 7af1eca2d0c4..291396c8f635 100644
@@ -130,7 +130,8 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
        goto out;
    }
 
Quoted 5 lines

xen/arch/arm/tee/ffa_partinfo.c void ffa_handle_partition_info_get(struct cpu_user_regs *regs) · line 133

Quoted 2 lines
xen/arch/arm/tee/ffa_partinfo.c +1 -1
-    if ( !ffa_fw_supports_fid(FFA_PARTITION_INFO_GET) )
+    if ( !IS_ENABLED(CONFIG_FFA_VM_TO_VM) &&

Doesn't this belong in the patch "xen/arm: ffa: Introduce VM to VM support"? And wouldn't ffa_vm_count make more sense?

ack. I will modify that and fold it into the VM to VM support patch.

Cheers
Bertrand

Quoted 10 lines

Cheers,
Jens

Quoted 6 lines
xen/arch/arm/tee/ffa_partinfo.c +1 -1
+         !ffa_fw_supports_fid(FFA_PARTITION_INFO_GET) )
    {
        /* Just give an empty partition list to the caller */
        ret = FFA_RET_OK;
--

2.47.1

#19 lore raw

Re: [PATCH v2 5/5] xen/arm: ffa: Enable VM to VM without firmware

Hi Jens,

Quoted 179 lines · Jens Wiklander

Hi,

Quoted 173 lines · Bertrand Marquis

Hi Jens,

Quoted 7 lines · Jens Wiklander

Hi Bertrand,

Quoted 1 line · Bertrand Marquis
Quoted 6 lines
Quoted 4 lines

When VM to VM support is activated and there is no suitable FF-A support in the firmware, enable FF-A support for VMs to allow using it for VM to VM communications.
If there is Optee running in the secure world and using the non FF-A

It's spelled OP-TEE in text, and optee or OPTEE in identifiers.

ack

Quoted 9 lines
Quoted 7 lines

communication system, having CONFIG_FFA_VM_TO_VM could be non functional (if optee is probed first) or Optee could be non functional (if FF-A is probed first) so it is not recommended to activate the configuration option for such systems.

To make buffer full notification work between VMs when there is not

s/not/no/

ack

Quoted 74 lines
Quoted 74 lines

firmware, rework the notification handling and modify the global flag to only be used as check for firmware notification support instead.

Modify part_info_get to return the list of VMs when there is no firmware support.

---
Changes in v2:
- replace ifdef with IS_ENABLED when possible
---

xen/arch/arm/tee/ffa.c          |  12 +++-
xen/arch/arm/tee/ffa_notif.c    | 114 ++++++++++++++++----------------
xen/arch/arm/tee/ffa_partinfo.c |   3 +-
3 files changed, 69 insertions(+), 60 deletions(-)
xen/arch/arm/tee/ffa.c +11 -1
index 3bbdd7168a6b..f6582d2e855a 100644
@@ -324,8 +324,9 @@ static int ffa_domain_init(struct domain *d)
   struct ffa_ctx *ctx;
   int ret;
 
-    if ( !ffa_fw_version )
+    if ( !IS_ENABLED(CONFIG_FFA_VM_TO_VM) && !ffa_fw_version )
       return -ENODEV;
+
   /*
    * We are using the domain_id + 1 as the FF-A ID for VMs as FF-A ID 0 is
    * reserved for the hypervisor and we only support secure endpoints using
@@ -549,6 +550,15 @@ err_no_fw:
   bitmap_zero(ffa_fw_abi_supported, FFA_ABI_BITMAP_SIZE);
   printk(XENLOG_WARNING "ARM FF-A No firmware support\n");
 
+    if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )
+    {
+        INIT_LIST_HEAD(&ffa_teardown_head);
+        init_timer(&ffa_teardown_timer, ffa_teardown_timer_callback, NULL, 0);
+
+        printk(XENLOG_INFO "ARM FF-A only available between VMs\n");
+        return true;
+    }
+
   return false;
 }
 
xen/arch/arm/tee/ffa_notif.c +1 -4
index d19aa5c5bef6..0673e53a9def 100644
@@ -16,7 +16,7 @@
 
 #include "ffa_private.h"
 
-static bool __ro_after_init notif_enabled;
+static bool __ro_after_init fw_notif_enabled;
 static unsigned int __ro_after_init notif_sri_irq;
 
 int ffa_handle_notification_bind(struct cpu_user_regs *regs)
@@ -27,21 +27,17 @@ int ffa_handle_notification_bind(struct cpu_user_regs *regs)
   uint32_t bitmap_lo = get_user_reg(regs, 3);
   uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
   if ( (src_dst & 0xFFFFU) != ffa_get_vm_id(d) )
       return FFA_RET_INVALID_PARAMETERS;
 
   if ( flags )    /* Only global notifications are supported */
       return FFA_RET_DENIED;
 
Quoted 10 lines

xen/arch/arm/tee/ffa_notif.c int ffa_handle_notification_bind(struct cpu_user_regs *regs) · line 36

Quoted 7 lines
xen/arch/arm/tee/ffa_notif.c +1 -6
-    /*
-     * We only support notifications from SP so no need to check the sender
-     * endpoint ID, the SPMC will take care of that for us.
-     */
-    return ffa_simple_call(FFA_NOTIFICATION_BIND, src_dst, flags, bitmap_lo,
-                           bitmap_hi);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )

Please add space before and after '>>', here and in the function below in this patch.

ack

Quoted 54 lines
Quoted 39 lines
xen/arch/arm/tee/ffa_notif.c +11 -10
+        return ffa_simple_call(FFA_NOTIFICATION_BIND, src_dst, flags,
+                               bitmap_lo, bitmap_hi);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 int ffa_handle_notification_unbind(struct cpu_user_regs *regs)
@@ -51,32 +47,34 @@ int ffa_handle_notification_unbind(struct cpu_user_regs *regs)
   uint32_t bitmap_lo = get_user_reg(regs, 3);
   uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
   if ( (src_dst & 0xFFFFU) != ffa_get_vm_id(d) )
       return FFA_RET_INVALID_PARAMETERS;
 
-    /*
-     * We only support notifications from SP so no need to check the
-     * destination endpoint ID, the SPMC will take care of that for us.
-     */
-    return  ffa_simple_call(FFA_NOTIFICATION_UNBIND, src_dst, 0, bitmap_lo,
-                            bitmap_hi);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )
+        return  ffa_simple_call(FFA_NOTIFICATION_UNBIND, src_dst, 0, bitmap_lo,
+                                bitmap_hi);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 void ffa_handle_notification_info_get(struct cpu_user_regs *regs)
 {
   struct domain *d = current->domain;
   struct ffa_ctx *ctx = d->arch.tee;
+    bool notif_pending = false;
 
-    if ( !notif_enabled )
+    if ( !IS_ENABLED(CONFIG_FFA_VM_TO_VM) && !fw_notif_enabled )

xen/arch/arm/tee/ffa_notif.c int ffa_handle_notification_unbind(struct cpu_user_regs *regs) · line 75

Quoted 10 lines
xen/arch/arm/tee/ffa_notif.c +4 -1
   {
       ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
       return;
   }
 
-    if ( test_and_clear_bool(ctx->notif.secure_pending) )
+    notif_pending = ctx->notif.secure_pending;
+#ifdef CONFIG_FFA_VM_TO_VM
+    notif_pending |= ctx->notif.buff_full_pending;
+#endif

Shouldn't ctx->notif.secure_pending and ctx->notif.secure_pending be cleared also, like:
notif_pending = test_and_clear_bool(ctx->notif.secure_pending) ||

              test_and_clear_bool(ctx->notif.buff_full_pending);
Quoted 8 lines
Quoted 2 lines

Notification info get is returning who has pending notification but only notification get should erase pending notifications.

FFA_NOTIFICATION_INFO_GET can return a "More pending notifications flag" in w2/x2 to inform the caller that it should call FFA_NOTIFICATION_INFO_GET again to get the remaining receiver endpoints. How can the ABI know where to resume the next time if the previous pending receivers aren't cleared?

I just checked the specification and you are right.
It is explicitly saying that "Information about pending notifications is returned only once".

Quoted 7 lines

The more pending notifications flag will not be needed here as we're dealing with a single endpoint at a time so it might be more of a philosophical question. I don't think it causes problems for the guest to keep secure_pending unchanged for FFA_NOTIFICATION_INFO_GET, but we should mention the changed behaviour in the commit message.

I agree I should discard the secure_pending flag in the implementation but I need to find a solution for the buffer full notification as I still need to signal it in notification get even if notification info get was called.

I will do the following:
- change secure_pending into pending_notif.
- set pending_notif when current secure_pending is set
- set pending_notif and buff_full_pending on indirect message
- clean pending_notif in notif_info_get
- clean pending_notif and buff_full in notif_get

Do you agree this is the right path ?

Cheers
Bertrand

Quoted 186 lines

Cheers,
Jens

Quoted 183 lines
Quoted 20 lines
Quoted 20 lines
xen/arch/arm/tee/ffa_notif.c +3 -1
+
+    if ( notif_pending )
   {
       /* A pending global notification for the guest */
       ffa_set_regs(regs, FFA_SUCCESS_64, 0,
@@ -103,7 +101,7 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
   uint32_t w6 = 0;
   uint32_t w7 = 0;
 
-    if ( !notif_enabled )
+    if ( !IS_ENABLED(CONFIG_FFA_VM_TO_VM) && !fw_notif_enabled )
   {
       ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
       return;
@@ -115,7 +113,8 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
       return;
   }
 
Quoted 5 lines

xen/arch/arm/tee/ffa_notif.c void ffa_handle_notification_get(struct cpu_user_regs *regs) · line 117

Quoted 3 lines
xen/arch/arm/tee/ffa_notif.c +2 -1
-    if ( flags & ( FFA_NOTIF_FLAG_BITMAP_SP | FFA_NOTIF_FLAG_BITMAP_SPM ) )
+    if ( fw_notif_enabled && (flags & ( FFA_NOTIF_FLAG_BITMAP_SP
+                                        | FFA_NOTIF_FLAG_BITMAP_SPM )) )

Please end the previous line with the '|' operator.

ack

Quoted 28 lines
Quoted 28 lines
xen/arch/arm/tee/ffa_notif.c +5 -6
   {
       struct arm_smccc_1_2_regs arg = {
           .a0 = FFA_NOTIFICATION_GET,
@@ -170,15 +169,14 @@ int ffa_handle_notification_set(struct cpu_user_regs *regs)
   uint32_t bitmap_lo = get_user_reg(regs, 3);
   uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
   if ( (src_dst >> 16) != ffa_get_vm_id(d) )
       return FFA_RET_INVALID_PARAMETERS;
 
-    /* Let the SPMC check the destination of the notification */
-    return ffa_simple_call(FFA_NOTIFICATION_SET, src_dst, flags, bitmap_lo,
-                           bitmap_hi);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )
+        return ffa_simple_call(FFA_NOTIFICATION_SET, src_dst, flags, bitmap_lo,
+                               bitmap_hi);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 #ifdef CONFIG_FFA_VM_TO_VM
@@ -190,7 +188,7 @@ void ffa_raise_rx_buffer_full(struct domain *d)
       return;
 
Quoted 6 lines

xen/arch/arm/tee/ffa_notif.c void ffa_raise_rx_buffer_full(struct domain *d) · line 191

Quoted 3 lines
xen/arch/arm/tee/ffa_notif.c +1 -1
   if ( !test_and_set_bool(ctx->notif.buff_full_pending) )
-        vgic_inject_irq(d, d->vcpu[0], notif_sri_irq, true);
+        vgic_inject_irq(d, d->vcpu[0], GUEST_FFA_NOTIF_PEND_INTR_ID, true);

Please move this to the patch "xen/arm: ffa: Add buffer full notification support"

Ack

Quoted 97 lines
Quoted 97 lines
xen/arch/arm/tee/ffa_notif.c +28 -28
 }
 #endif
 
@@ -363,7 +361,7 @@ void ffa_notif_init_interrupt(void)
 {
   int ret;
 
-    if ( notif_enabled && notif_sri_irq < NR_GIC_SGI )
+    if ( fw_notif_enabled && notif_sri_irq < NR_GIC_SGI )
   {
       /*
        * An error here is unlikely since the primary CPU has already
@@ -394,41 +392,41 @@ void ffa_notif_init(void)
   int ret;
 
   /* Only enable fw notification if all ABIs we need are supported */
-    if ( !(ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_CREATE) &&
-           ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_DESTROY) &&
-           ffa_fw_supports_fid(FFA_NOTIFICATION_GET) &&
-           ffa_fw_supports_fid(FFA_NOTIFICATION_INFO_GET_64)) )
-        return;
-
-    arm_smccc_1_2_smc(&arg, &resp);
-    if ( resp.a0 != FFA_SUCCESS_32 )
-        return;
-
-    irq = resp.a2;
-    notif_sri_irq = irq;
-    if ( irq >= NR_GIC_SGI )
-        irq_set_type(irq, IRQ_TYPE_EDGE_RISING);
-    ret = request_irq(irq, 0, notif_irq_handler, "FF-A notif", NULL);
-    if ( ret )
+    if ( ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_CREATE) &&
+         ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_DESTROY) &&
+         ffa_fw_supports_fid(FFA_NOTIFICATION_GET) &&
+         ffa_fw_supports_fid(FFA_NOTIFICATION_INFO_GET_64) )
   {
-        printk(XENLOG_ERR "ffa: request_irq irq %u failed: error %d\n",
-               irq, ret);
-        return;
-    }
+        arm_smccc_1_2_smc(&arg, &resp);
+        if ( resp.a0 != FFA_SUCCESS_32 )
+            return;
 
-    notif_enabled = true;
+        irq = resp.a2;
+        notif_sri_irq = irq;
+        if ( irq >= NR_GIC_SGI )
+            irq_set_type(irq, IRQ_TYPE_EDGE_RISING);
+        ret = request_irq(irq, 0, notif_irq_handler, "FF-A notif", NULL);
+        if ( ret )
+        {
+            printk(XENLOG_ERR "ffa: request_irq irq %u failed: error %d\n",
+                   irq, ret);
+            return;
+        }
+        fw_notif_enabled = true;
+    }
 }
 
 int ffa_notif_domain_init(struct domain *d)
 {
   int32_t res;
 
-    if ( !notif_enabled )
-        return 0;
+    if ( fw_notif_enabled )
+    {
 
-    res = ffa_notification_bitmap_create(ffa_get_vm_id(d), d->max_vcpus);
-    if ( res )
-        return -ENOMEM;
+        res = ffa_notification_bitmap_create(ffa_get_vm_id(d), d->max_vcpus);
+        if ( res )
+            return -ENOMEM;
+    }
 
   return 0;
 }
@@ -439,6 +437,6 @@ void ffa_notif_domain_destroy(struct domain *d)
    * Call bitmap_destroy even if bitmap create failed as the SPMC will
    * return a DENIED error that we will ignore.
    */
-    if ( notif_enabled )
+    if ( fw_notif_enabled )
       ffa_notification_bitmap_destroy(ffa_get_vm_id(d));
 }
xen/arch/arm/tee/ffa_partinfo.c +0 -0
index 7af1eca2d0c4..291396c8f635 100644
@@ -130,7 +130,8 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
       goto out;
   }
 
Quoted 5 lines

xen/arch/arm/tee/ffa_partinfo.c void ffa_handle_partition_info_get(struct cpu_user_regs *regs) · line 133

Quoted 2 lines
xen/arch/arm/tee/ffa_partinfo.c +1 -1
-    if ( !ffa_fw_supports_fid(FFA_PARTITION_INFO_GET) )
+    if ( !IS_ENABLED(CONFIG_FFA_VM_TO_VM) &&

Doesn't this belong in the patch "xen/arm: ffa: Introduce VM to VM support"? And wouldn't ffa_vm_count make more sense?

ack. I will modify that and fold it into the VM to VM support patch.

Cheers
Bertrand

Quoted 10 lines

Cheers,
Jens

Quoted 6 lines
xen/arch/arm/tee/ffa_partinfo.c +1 -1
+         !ffa_fw_supports_fid(FFA_PARTITION_INFO_GET) )
   {
       /* Just give an empty partition list to the caller */
       ret = FFA_RET_OK;
--

2.47.1

#20 lore raw

Re: [PATCH v2 5/5] xen/arm: ffa: Enable VM to VM without firmware

Hi Bertrand,

Quoted 207 lines · Bertrand Marquis

Hi Jens,

Quoted 179 lines · Jens Wiklander

Hi,

Quoted 173 lines · Bertrand Marquis

Hi Jens,

Quoted 7 lines · Jens Wiklander

Hi Bertrand,

Quoted 1 line · Bertrand Marquis
Quoted 6 lines
Quoted 4 lines

When VM to VM support is activated and there is no suitable FF-A support in the firmware, enable FF-A support for VMs to allow using it for VM to VM communications.
If there is Optee running in the secure world and using the non FF-A

It's spelled OP-TEE in text, and optee or OPTEE in identifiers.

ack

Quoted 9 lines
Quoted 7 lines

communication system, having CONFIG_FFA_VM_TO_VM could be non functional (if optee is probed first) or Optee could be non functional (if FF-A is probed first) so it is not recommended to activate the configuration option for such systems.

To make buffer full notification work between VMs when there is not

s/not/no/

ack

Quoted 74 lines
Quoted 74 lines

firmware, rework the notification handling and modify the global flag to only be used as check for firmware notification support instead.

Modify part_info_get to return the list of VMs when there is no firmware support.

---
Changes in v2:
- replace ifdef with IS_ENABLED when possible
---

xen/arch/arm/tee/ffa.c          |  12 +++-
xen/arch/arm/tee/ffa_notif.c    | 114 ++++++++++++++++----------------
xen/arch/arm/tee/ffa_partinfo.c |   3 +-
3 files changed, 69 insertions(+), 60 deletions(-)
xen/arch/arm/tee/ffa.c +11 -1
index 3bbdd7168a6b..f6582d2e855a 100644
@@ -324,8 +324,9 @@ static int ffa_domain_init(struct domain *d)
   struct ffa_ctx *ctx;
   int ret;
 
-    if ( !ffa_fw_version )
+    if ( !IS_ENABLED(CONFIG_FFA_VM_TO_VM) && !ffa_fw_version )
       return -ENODEV;
+
   /*
    * We are using the domain_id + 1 as the FF-A ID for VMs as FF-A ID 0 is
    * reserved for the hypervisor and we only support secure endpoints using
@@ -549,6 +550,15 @@ err_no_fw:
   bitmap_zero(ffa_fw_abi_supported, FFA_ABI_BITMAP_SIZE);
   printk(XENLOG_WARNING "ARM FF-A No firmware support\n");
 
+    if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )
+    {
+        INIT_LIST_HEAD(&ffa_teardown_head);
+        init_timer(&ffa_teardown_timer, ffa_teardown_timer_callback, NULL, 0);
+
+        printk(XENLOG_INFO "ARM FF-A only available between VMs\n");
+        return true;
+    }
+
   return false;
 }
 
xen/arch/arm/tee/ffa_notif.c +1 -4
index d19aa5c5bef6..0673e53a9def 100644
@@ -16,7 +16,7 @@
 
 #include "ffa_private.h"
 
-static bool __ro_after_init notif_enabled;
+static bool __ro_after_init fw_notif_enabled;
 static unsigned int __ro_after_init notif_sri_irq;
 
 int ffa_handle_notification_bind(struct cpu_user_regs *regs)
@@ -27,21 +27,17 @@ int ffa_handle_notification_bind(struct cpu_user_regs *regs)
   uint32_t bitmap_lo = get_user_reg(regs, 3);
   uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
   if ( (src_dst & 0xFFFFU) != ffa_get_vm_id(d) )
       return FFA_RET_INVALID_PARAMETERS;
 
   if ( flags )    /* Only global notifications are supported */
       return FFA_RET_DENIED;
 
Quoted 10 lines

xen/arch/arm/tee/ffa_notif.c int ffa_handle_notification_bind(struct cpu_user_regs *regs) · line 36

Quoted 7 lines
xen/arch/arm/tee/ffa_notif.c +1 -6
-    /*
-     * We only support notifications from SP so no need to check the sender
-     * endpoint ID, the SPMC will take care of that for us.
-     */
-    return ffa_simple_call(FFA_NOTIFICATION_BIND, src_dst, flags, bitmap_lo,
-                           bitmap_hi);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )

Please add space before and after '>>', here and in the function below in this patch.

ack

Quoted 54 lines
Quoted 39 lines
xen/arch/arm/tee/ffa_notif.c +11 -10
+        return ffa_simple_call(FFA_NOTIFICATION_BIND, src_dst, flags,
+                               bitmap_lo, bitmap_hi);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 int ffa_handle_notification_unbind(struct cpu_user_regs *regs)
@@ -51,32 +47,34 @@ int ffa_handle_notification_unbind(struct cpu_user_regs *regs)
   uint32_t bitmap_lo = get_user_reg(regs, 3);
   uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
   if ( (src_dst & 0xFFFFU) != ffa_get_vm_id(d) )
       return FFA_RET_INVALID_PARAMETERS;
 
-    /*
-     * We only support notifications from SP so no need to check the
-     * destination endpoint ID, the SPMC will take care of that for us.
-     */
-    return  ffa_simple_call(FFA_NOTIFICATION_UNBIND, src_dst, 0, bitmap_lo,
-                            bitmap_hi);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )
+        return  ffa_simple_call(FFA_NOTIFICATION_UNBIND, src_dst, 0, bitmap_lo,
+                                bitmap_hi);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 void ffa_handle_notification_info_get(struct cpu_user_regs *regs)
 {
   struct domain *d = current->domain;
   struct ffa_ctx *ctx = d->arch.tee;
+    bool notif_pending = false;
 
-    if ( !notif_enabled )
+    if ( !IS_ENABLED(CONFIG_FFA_VM_TO_VM) && !fw_notif_enabled )

xen/arch/arm/tee/ffa_notif.c int ffa_handle_notification_unbind(struct cpu_user_regs *regs) · line 75

Quoted 10 lines
xen/arch/arm/tee/ffa_notif.c +4 -1
   {
       ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
       return;
   }
 
-    if ( test_and_clear_bool(ctx->notif.secure_pending) )
+    notif_pending = ctx->notif.secure_pending;
+#ifdef CONFIG_FFA_VM_TO_VM
+    notif_pending |= ctx->notif.buff_full_pending;
+#endif

Shouldn't ctx->notif.secure_pending and ctx->notif.secure_pending be cleared also, like:
notif_pending = test_and_clear_bool(ctx->notif.secure_pending) ||

              test_and_clear_bool(ctx->notif.buff_full_pending);
Quoted 8 lines
Quoted 2 lines

Notification info get is returning who has pending notification but only notification get should erase pending notifications.

FFA_NOTIFICATION_INFO_GET can return a "More pending notifications flag" in w2/x2 to inform the caller that it should call FFA_NOTIFICATION_INFO_GET again to get the remaining receiver endpoints. How can the ABI know where to resume the next time if the previous pending receivers aren't cleared?

I just checked the specification and you are right.
It is explicitly saying that "Information about pending notifications is returned only once".

Quoted 7 lines

The more pending notifications flag will not be needed here as we're dealing with a single endpoint at a time so it might be more of a philosophical question. I don't think it causes problems for the guest to keep secure_pending unchanged for FFA_NOTIFICATION_INFO_GET, but we should mention the changed behaviour in the commit message.

I agree I should discard the secure_pending flag in the implementation but I need to find a solution for the buffer full notification as I still need to signal it in notification get even if notification info get was called.

Quoted 8 lines

I will do the following:
- change secure_pending into pending_notif.
- set pending_notif when current secure_pending is set
- set pending_notif and buff_full_pending on indirect message
- clean pending_notif in notif_info_get
- clean pending_notif and buff_full in notif_get

Do you agree this is the right path ?

Yes, this is the way. :-)

Cheers,
Jens

Quoted 192 lines

Cheers
Bertrand

Quoted 186 lines

Cheers,
Jens

Quoted 183 lines
Quoted 20 lines
Quoted 20 lines
xen/arch/arm/tee/ffa_notif.c +3 -1
+
+    if ( notif_pending )
   {
       /* A pending global notification for the guest */
       ffa_set_regs(regs, FFA_SUCCESS_64, 0,
@@ -103,7 +101,7 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
   uint32_t w6 = 0;
   uint32_t w7 = 0;
 
-    if ( !notif_enabled )
+    if ( !IS_ENABLED(CONFIG_FFA_VM_TO_VM) && !fw_notif_enabled )
   {
       ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
       return;
@@ -115,7 +113,8 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
       return;
   }
 
Quoted 5 lines

xen/arch/arm/tee/ffa_notif.c void ffa_handle_notification_get(struct cpu_user_regs *regs) · line 117

Quoted 3 lines
xen/arch/arm/tee/ffa_notif.c +2 -1
-    if ( flags & ( FFA_NOTIF_FLAG_BITMAP_SP | FFA_NOTIF_FLAG_BITMAP_SPM ) )
+    if ( fw_notif_enabled && (flags & ( FFA_NOTIF_FLAG_BITMAP_SP
+                                        | FFA_NOTIF_FLAG_BITMAP_SPM )) )

Please end the previous line with the '|' operator.

ack

Quoted 28 lines
Quoted 28 lines
xen/arch/arm/tee/ffa_notif.c +5 -6
   {
       struct arm_smccc_1_2_regs arg = {
           .a0 = FFA_NOTIFICATION_GET,
@@ -170,15 +169,14 @@ int ffa_handle_notification_set(struct cpu_user_regs *regs)
   uint32_t bitmap_lo = get_user_reg(regs, 3);
   uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
   if ( (src_dst >> 16) != ffa_get_vm_id(d) )
       return FFA_RET_INVALID_PARAMETERS;
 
-    /* Let the SPMC check the destination of the notification */
-    return ffa_simple_call(FFA_NOTIFICATION_SET, src_dst, flags, bitmap_lo,
-                           bitmap_hi);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )
+        return ffa_simple_call(FFA_NOTIFICATION_SET, src_dst, flags, bitmap_lo,
+                               bitmap_hi);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 #ifdef CONFIG_FFA_VM_TO_VM
@@ -190,7 +188,7 @@ void ffa_raise_rx_buffer_full(struct domain *d)
       return;
 
Quoted 6 lines

xen/arch/arm/tee/ffa_notif.c void ffa_raise_rx_buffer_full(struct domain *d) · line 191

Quoted 3 lines
xen/arch/arm/tee/ffa_notif.c +1 -1
   if ( !test_and_set_bool(ctx->notif.buff_full_pending) )
-        vgic_inject_irq(d, d->vcpu[0], notif_sri_irq, true);
+        vgic_inject_irq(d, d->vcpu[0], GUEST_FFA_NOTIF_PEND_INTR_ID, true);

Please move this to the patch "xen/arm: ffa: Add buffer full notification support"

Ack

Quoted 97 lines
Quoted 97 lines
xen/arch/arm/tee/ffa_notif.c +28 -28
 }
 #endif
 
@@ -363,7 +361,7 @@ void ffa_notif_init_interrupt(void)
 {
   int ret;
 
-    if ( notif_enabled && notif_sri_irq < NR_GIC_SGI )
+    if ( fw_notif_enabled && notif_sri_irq < NR_GIC_SGI )
   {
       /*
        * An error here is unlikely since the primary CPU has already
@@ -394,41 +392,41 @@ void ffa_notif_init(void)
   int ret;
 
   /* Only enable fw notification if all ABIs we need are supported */
-    if ( !(ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_CREATE) &&
-           ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_DESTROY) &&
-           ffa_fw_supports_fid(FFA_NOTIFICATION_GET) &&
-           ffa_fw_supports_fid(FFA_NOTIFICATION_INFO_GET_64)) )
-        return;
-
-    arm_smccc_1_2_smc(&arg, &resp);
-    if ( resp.a0 != FFA_SUCCESS_32 )
-        return;
-
-    irq = resp.a2;
-    notif_sri_irq = irq;
-    if ( irq >= NR_GIC_SGI )
-        irq_set_type(irq, IRQ_TYPE_EDGE_RISING);
-    ret = request_irq(irq, 0, notif_irq_handler, "FF-A notif", NULL);
-    if ( ret )
+    if ( ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_CREATE) &&
+         ffa_fw_supports_fid(FFA_NOTIFICATION_BITMAP_DESTROY) &&
+         ffa_fw_supports_fid(FFA_NOTIFICATION_GET) &&
+         ffa_fw_supports_fid(FFA_NOTIFICATION_INFO_GET_64) )
   {
-        printk(XENLOG_ERR "ffa: request_irq irq %u failed: error %d\n",
-               irq, ret);
-        return;
-    }
+        arm_smccc_1_2_smc(&arg, &resp);
+        if ( resp.a0 != FFA_SUCCESS_32 )
+            return;
 
-    notif_enabled = true;
+        irq = resp.a2;
+        notif_sri_irq = irq;
+        if ( irq >= NR_GIC_SGI )
+            irq_set_type(irq, IRQ_TYPE_EDGE_RISING);
+        ret = request_irq(irq, 0, notif_irq_handler, "FF-A notif", NULL);
+        if ( ret )
+        {
+            printk(XENLOG_ERR "ffa: request_irq irq %u failed: error %d\n",
+                   irq, ret);
+            return;
+        }
+        fw_notif_enabled = true;
+    }
 }
 
 int ffa_notif_domain_init(struct domain *d)
 {
   int32_t res;
 
-    if ( !notif_enabled )
-        return 0;
+    if ( fw_notif_enabled )
+    {
 
-    res = ffa_notification_bitmap_create(ffa_get_vm_id(d), d->max_vcpus);
-    if ( res )
-        return -ENOMEM;
+        res = ffa_notification_bitmap_create(ffa_get_vm_id(d), d->max_vcpus);
+        if ( res )
+            return -ENOMEM;
+    }
 
   return 0;
 }
@@ -439,6 +437,6 @@ void ffa_notif_domain_destroy(struct domain *d)
    * Call bitmap_destroy even if bitmap create failed as the SPMC will
    * return a DENIED error that we will ignore.
    */
-    if ( notif_enabled )
+    if ( fw_notif_enabled )
       ffa_notification_bitmap_destroy(ffa_get_vm_id(d));
 }
xen/arch/arm/tee/ffa_partinfo.c +0 -0
index 7af1eca2d0c4..291396c8f635 100644
@@ -130,7 +130,8 @@ void ffa_handle_partition_info_get(struct cpu_user_regs *regs)
       goto out;
   }
 
Quoted 5 lines

xen/arch/arm/tee/ffa_partinfo.c void ffa_handle_partition_info_get(struct cpu_user_regs *regs) · line 133

Quoted 2 lines
xen/arch/arm/tee/ffa_partinfo.c +1 -1
-    if ( !ffa_fw_supports_fid(FFA_PARTITION_INFO_GET) )
+    if ( !IS_ENABLED(CONFIG_FFA_VM_TO_VM) &&

Doesn't this belong in the patch "xen/arm: ffa: Introduce VM to VM support"? And wouldn't ffa_vm_count make more sense?

ack. I will modify that and fold it into the VM to VM support patch.

Cheers
Bertrand

Quoted 10 lines

Cheers,
Jens

Quoted 6 lines
xen/arch/arm/tee/ffa_partinfo.c +1 -1
+         !ffa_fw_supports_fid(FFA_PARTITION_INFO_GET) )
   {
       /* Just give an empty partition list to the caller */
       ret = FFA_RET_OK;
--

2.47.1

#21 lore raw

Re: [PATCH v2 5/5] xen/arm: ffa: Enable VM to VM without firmware

Hi Jens,

Quoted 220 lines · Jens Wiklander

Hi Bertrand,

Quoted 214 lines · Bertrand Marquis

Hi Jens,

Quoted 179 lines · Jens Wiklander

Hi,

Quoted 173 lines · Bertrand Marquis

Hi Jens,

Quoted 7 lines · Jens Wiklander

Hi Bertrand,

Quoted 1 line · Bertrand Marquis
Quoted 6 lines
Quoted 4 lines

When VM to VM support is activated and there is no suitable FF-A support in the firmware, enable FF-A support for VMs to allow using it for VM to VM communications.
If there is Optee running in the secure world and using the non FF-A

It's spelled OP-TEE in text, and optee or OPTEE in identifiers.

ack

Quoted 9 lines
Quoted 7 lines

communication system, having CONFIG_FFA_VM_TO_VM could be non functional (if optee is probed first) or Optee could be non functional (if FF-A is probed first) so it is not recommended to activate the configuration option for such systems.

To make buffer full notification work between VMs when there is not

s/not/no/

ack

Quoted 74 lines
Quoted 74 lines

firmware, rework the notification handling and modify the global flag to only be used as check for firmware notification support instead.

Modify part_info_get to return the list of VMs when there is no firmware support.

---
Changes in v2:
- replace ifdef with IS_ENABLED when possible
---

xen/arch/arm/tee/ffa.c          |  12 +++-
xen/arch/arm/tee/ffa_notif.c    | 114 ++++++++++++++++----------------
xen/arch/arm/tee/ffa_partinfo.c |   3 +-
3 files changed, 69 insertions(+), 60 deletions(-)
xen/arch/arm/tee/ffa.c +11 -1
index 3bbdd7168a6b..f6582d2e855a 100644
@@ -324,8 +324,9 @@ static int ffa_domain_init(struct domain *d)
  struct ffa_ctx *ctx;
  int ret;
 
-    if ( !ffa_fw_version )
+    if ( !IS_ENABLED(CONFIG_FFA_VM_TO_VM) && !ffa_fw_version )
      return -ENODEV;
+
  /*
   * We are using the domain_id + 1 as the FF-A ID for VMs as FF-A ID 0 is
   * reserved for the hypervisor and we only support secure endpoints using
@@ -549,6 +550,15 @@ err_no_fw:
  bitmap_zero(ffa_fw_abi_supported, FFA_ABI_BITMAP_SIZE);
  printk(XENLOG_WARNING "ARM FF-A No firmware support\n");
 
+    if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )
+    {
+        INIT_LIST_HEAD(&ffa_teardown_head);
+        init_timer(&ffa_teardown_timer, ffa_teardown_timer_callback, NULL, 0);
+
+        printk(XENLOG_INFO "ARM FF-A only available between VMs\n");
+        return true;
+    }
+
  return false;
 }
 
xen/arch/arm/tee/ffa_notif.c +1 -4
index d19aa5c5bef6..0673e53a9def 100644
@@ -16,7 +16,7 @@
 
 #include "ffa_private.h"
 
-static bool __ro_after_init notif_enabled;
+static bool __ro_after_init fw_notif_enabled;
 static unsigned int __ro_after_init notif_sri_irq;
 
 int ffa_handle_notification_bind(struct cpu_user_regs *regs)
@@ -27,21 +27,17 @@ int ffa_handle_notification_bind(struct cpu_user_regs *regs)
  uint32_t bitmap_lo = get_user_reg(regs, 3);
  uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
  if ( (src_dst & 0xFFFFU) != ffa_get_vm_id(d) )
      return FFA_RET_INVALID_PARAMETERS;
 
  if ( flags )    /* Only global notifications are supported */
      return FFA_RET_DENIED;
 
Quoted 10 lines

xen/arch/arm/tee/ffa_notif.c int ffa_handle_notification_bind(struct cpu_user_regs *regs) · line 36

Quoted 7 lines
xen/arch/arm/tee/ffa_notif.c +1 -6
-    /*
-     * We only support notifications from SP so no need to check the sender
-     * endpoint ID, the SPMC will take care of that for us.
-     */
-    return ffa_simple_call(FFA_NOTIFICATION_BIND, src_dst, flags, bitmap_lo,
-                           bitmap_hi);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )

Please add space before and after '>>', here and in the function below in this patch.

ack

Quoted 54 lines
Quoted 39 lines
xen/arch/arm/tee/ffa_notif.c +11 -10
+        return ffa_simple_call(FFA_NOTIFICATION_BIND, src_dst, flags,
+                               bitmap_lo, bitmap_hi);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 int ffa_handle_notification_unbind(struct cpu_user_regs *regs)
@@ -51,32 +47,34 @@ int ffa_handle_notification_unbind(struct cpu_user_regs *regs)
  uint32_t bitmap_lo = get_user_reg(regs, 3);
  uint32_t bitmap_hi = get_user_reg(regs, 4);
 
-    if ( !notif_enabled )
-        return FFA_RET_NOT_SUPPORTED;
-
  if ( (src_dst & 0xFFFFU) != ffa_get_vm_id(d) )
      return FFA_RET_INVALID_PARAMETERS;
 
-    /*
-     * We only support notifications from SP so no need to check the
-     * destination endpoint ID, the SPMC will take care of that for us.
-     */
-    return  ffa_simple_call(FFA_NOTIFICATION_UNBIND, src_dst, 0, bitmap_lo,
-                            bitmap_hi);
+    if ( FFA_ID_IS_SECURE(src_dst>>16) && fw_notif_enabled )
+        return  ffa_simple_call(FFA_NOTIFICATION_UNBIND, src_dst, 0, bitmap_lo,
+                                bitmap_hi);
+
+    return FFA_RET_NOT_SUPPORTED;
 }
 
 void ffa_handle_notification_info_get(struct cpu_user_regs *regs)
 {
  struct domain *d = current->domain;
  struct ffa_ctx *ctx = d->arch.tee;
+    bool notif_pending = false;
 
-    if ( !notif_enabled )
+    if ( !IS_ENABLED(CONFIG_FFA_VM_TO_VM) && !fw_notif_enabled )

xen/arch/arm/tee/ffa_notif.c int ffa_handle_notification_unbind(struct cpu_user_regs *regs) · line 75

Quoted 10 lines
xen/arch/arm/tee/ffa_notif.c +4 -1
  {
      ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
      return;
  }
 
-    if ( test_and_clear_bool(ctx->notif.secure_pending) )
+    notif_pending = ctx->notif.secure_pending;
+#ifdef CONFIG_FFA_VM_TO_VM
+    notif_pending |= ctx->notif.buff_full_pending;
+#endif

Shouldn't ctx->notif.secure_pending and ctx->notif.secure_pending be cleared also, like:
notif_pending = test_and_clear_bool(ctx->notif.secure_pending) ||

             test_and_clear_bool(ctx->notif.buff_full_pending);
Quoted 8 lines
Quoted 2 lines

Notification info get is returning who has pending notification but only notification get should erase pending notifications.

FFA_NOTIFICATION_INFO_GET can return a "More pending notifications flag" in w2/x2 to inform the caller that it should call FFA_NOTIFICATION_INFO_GET again to get the remaining receiver endpoints. How can the ABI know where to resume the next time if the previous pending receivers aren't cleared?

I just checked the specification and you are right.
It is explicitly saying that "Information about pending notifications is returned only once".

Quoted 7 lines

The more pending notifications flag will not be needed here as we're dealing with a single endpoint at a time so it might be more of a philosophical question. I don't think it causes problems for the guest to keep secure_pending unchanged for FFA_NOTIFICATION_INFO_GET, but we should mention the changed behaviour in the commit message.

I agree I should discard the secure_pending flag in the implementation but I need to find a solution for the buffer full notification as I still need to signal it in notification get even if notification info get was called.

I will do the following:
- change secure_pending into pending_notif.
- set pending_notif when current secure_pending is set
- set pending_notif and buff_full_pending on indirect message
- clean pending_notif in notif_info_get
- clean pending_notif and buff_full in notif_get

Quoted 3 lines
Quoted 1 line

Do you agree this is the right path ?

Yes, this is the way. :-)

Well in fact there is a mistake in this way and I had to do something a bit different.

When we have a notification get, we can only clear secure_pending if FFA_NOTIF_FLAG_BITMAP_SP(M) are passed.

So i think i have to do the following:
- secure_pending: set and clean as they are now
- vm_pending: set when raising buf full and clean in info_get or get with FFA_NOTIF_FLAG_BITMAP_HYP set
- buff_full_pending only cleaned in get with FFA_NOTIF_FLAG_BITMAP_HYP set

This way info_get would still return something if a get is done but with only flags to retrieve secure and there is a buff full notif pending or with only HYP and secure ones pending.

Giving something like that:

xen/arch/arm/tee/ffa_notif.c +13 -5
@@ -69,6 +69,7 @@ void ffa_handle_notification_info_get(struct cpu_user_regs *regs)
 {
     struct domain *d = current->domain;
     struct ffa_ctx *ctx = d->arch.tee;
+    bool notif_pending;
 
     if ( !notif_enabled )
     {
@@ -76,7 +77,11 @@ void ffa_handle_notification_info_get(struct cpu_user_regs *regs)
         return;
     }
 
-    if ( test_and_clear_bool(ctx->notif.secure_pending) )
+    notif_pending = test_and_clear_bool(ctx->notif.secure_pending);
+    if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )
+        notif_pending |= test_and_clear_bool(ctx->notif.vm_pending)
+
+    if ( notif_pending )
     {
         /* A pending global notification for the guest */
         ffa_set_regs(regs, FFA_SUCCESS_64, 0,
@@ -153,11 +158,13 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
             w6 = resp.a6;
     }
 
-#ifdef CONFIG_FFA_VM_TO_VM
-    if ( flags & FFA_NOTIF_FLAG_BITMAP_HYP &&
+    if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) &&
+          flags & FFA_NOTIF_FLAG_BITMAP_HYP &&
           test_and_clear_bool(ctx->notif.buff_full_pending) )
+    {
+        ACCESS_ONCE(ctx->notif.vm_pending) = false;
         w7 = FFA_NOTIF_RX_BUFFER_FULL;
-#endif
+    }
 
     ffa_set_regs(regs, FFA_SUCCESS_32, 0, w2, w3, w4, w5, w6, w7);
 }
@@ -189,7 +196,8 @@ void ffa_raise_rx_buffer_full(struct domain *d)
     if ( !ctx )
         return;
 
-    if ( !test_and_set_bool(ctx->notif.buff_full_pending) )
+    ACCESS_ONCE(ctx->notif.buff_full_pending) = true
+    if ( !test_and_set_bool(ctx->notif.vm_pending) )
         vgic_inject_irq(d, d->vcpu[0], notif_sri_irq, true);
 }
 

Cheers
Bertrand