FPGA DFL updates
Subsystems (4)
v3 cover FPGA DFL updates
Hi Greg
This is v3 patchset which adds more features to FPGA DFL. This patchset is made on top of patch[1] and 5.3-rc1 tree. Documentation patch for DFL is added back to this patchset
Main changes from v2:
- update kernel version/date in sysfs doc (patch #4, #5, #8, #10, #11).
- add back Documentation patch (patch #12).
Main changes from v1:
- remove DRV/MODULE_VERSION modifications. (patch #1, #3, #4, #6)
- remove argsz from new ioctls. (patch #2)
- replace sysfs_create/remove_* with device_add/remove_* for sysfs entries.
(patch #5, #8, #11)
[1] [PATCH] fpga: dfl: use driver core functions, not sysfs ones.
https://lkml.org/lkml/2019/7/4/36
Wu Hao (12):
fpga: dfl: fme: support 512bit data width PR
fpga: dfl: fme: add DFL_FPGA_FME_PORT_RELEASE/ASSIGN ioctl support.
fpga: dfl: pci: enable SRIOV support.
fpga: dfl: afu: add AFU state related sysfs interfaces
fpga: dfl: afu: add userclock sysfs interfaces.
fpga: dfl: add id_table for dfl private feature driver
fpga: dfl: afu: export __port_enable/disable function.
fpga: dfl: afu: add error reporting support.
fpga: dfl: afu: add STP (SignalTap) support
fpga: dfl: fme: add capability sysfs interfaces
fpga: dfl: fme: add global error reporting support
Documentation: fpga: dfl: add descriptions for virtualization and new
interfaces.
Documentation/ABI/testing/sysfs-platform-dfl-fme | 98 ++++++ Documentation/ABI/testing/sysfs-platform-dfl-port | 104 ++++++ Documentation/fpga/dfl.rst | 105 ++++++ drivers/fpga/Makefile | 3 +- drivers/fpga/dfl-afu-error.c | 225 +++++++++++++ drivers/fpga/dfl-afu-main.c | 328 +++++++++++++++++- drivers/fpga/dfl-afu.h | 7 + drivers/fpga/dfl-fme-error.c | 385 ++++++++++++++++++++++ drivers/fpga/dfl-fme-main.c | 93 +++++- drivers/fpga/dfl-fme-mgr.c | 110 ++++++- drivers/fpga/dfl-fme-pr.c | 50 ++- drivers/fpga/dfl-fme.h | 7 +- drivers/fpga/dfl-pci.c | 39 +++ drivers/fpga/dfl.c | 166 +++++++++- drivers/fpga/dfl.h | 54 ++- include/uapi/linux/fpga-dfl.h | 19 ++ 16 files changed, 1722 insertions(+), 71 deletions(-) create mode 100644 drivers/fpga/dfl-afu-error.c create mode 100644 drivers/fpga/dfl-fme-error.c
Signature · 1 line
1.8.3.1
PATCH v3 1/12 fpga: dfl: fme: support 512bit data width PR
In early partial reconfiguration private feature, it only supports 32bit data width when writing data to hardware for PR. 512bit data width PR support is an important optimization for some specific solutions (e.g. XEON with FPGA integrated), it allows driver to use AVX512 instruction to improve the performance of partial reconfiguration. e.g. programming one 100MB bitstream image via this 512bit data width PR hardware only takes ~300ms, but 32bit revision requires ~3s per test result.
Please note now this optimization is only done on revision 2 of this PR private feature which is only used in integrated solution that AVX512 is always supported. This revision 2 hardware doesn't support 32bit PR.
---
v2: remove DRV/MODULE_VERSION modifications
--- drivers/fpga/dfl-fme-mgr.c | 110 ++++++++++++++++++++++++++++++++++++++------- drivers/fpga/dfl-fme-pr.c | 43 +++++++++++------- drivers/fpga/dfl-fme.h | 2 + drivers/fpga/dfl.h | 5 +++ 4 files changed, 129 insertions(+), 31 deletions(-)
drivers/fpga/dfl-fme-mgr.c +94 -16
@@ -22,6 +22,7 @@#include <linux/io-64-nonatomic-lo-hi.h> #include <linux/fpga/fpga-mgr.h> +#include "dfl.h" #include "dfl-fme-pr.h" /* FME Partial Reconfiguration Sub Feature Register Set */
@@ -30,6 +31,7 @@#define FME_PR_STS 0x10#define FME_PR_DATA 0x18#define FME_PR_ERR 0x20 +#define FME_PR_512_DATA 0x40/* Data Register for 512bit datawidth PR */ #define FME_PR_INTFC_ID_L 0xA8#define FME_PR_INTFC_ID_H 0xB0
@@ -67,8 +69,43 @@#define PR_WAIT_TIMEOUT 8000000#define PR_HOST_STATUS_IDLE 0 +#if defined(CONFIG_X86 ) && defined(CONFIG_AS_AVX512 ) + +#include <linux/cpufeature.h> +#include <asm/fpu/api.h> + +static inline int is_cpu_avx512_enabled (void ) +{ +return cpu_feature_enabled (X86_FEATURE_AVX512F ); +} + +static inline void copy512 (const void *src,void __iomem *dst) +{ +kernel_fpu_begin (); + + asmvolatile ("vmovdqu64 (%0), %%zmm0;" +"vmovntdq %%zmm0, (%1);" + : + :"r" (src),"r" (dst) + :"memory" ); + +kernel_fpu_end (); +} +#else +static inline int is_cpu_avx512_enabled (void ) +{ +return 0 ; +} + +static inline void copy512 (const void *src,void __iomem *dst) +{ +WARN_ON_ONCE (1 ); +} +#endif +struct fme_mgr_priv {void __iomem *ioaddr; +unsigned int pr_datawidth;u64 pr_error; };
@@ -169,7 +206,7 @@ static int fme_mgr_write(struct fpga_manager *mgr,struct fme_mgr_priv *priv = mgr->priv;void __iomem *fme_pr = priv->ioaddr;u64 pr_ctrl, pr_status, pr_data; -int delay =0 , pr_credit, i =0 ; +int ret =0 , delay =0 , pr_credit;dev_dbg (dev,"start request\n" );
@@ -181,9 +218,9 @@ static int fme_mgr_write(struct fpga_manager *mgr,/* * driver can push data to PR hardware using PR_DATA register once HW -* has enough pr_credit (> 1), pr_credit reduces one for every 32bit -* pr data write to PR_DATA register. If pr_credit <= 1, driver needs -* to wait for enough pr_credit from hardware by polling. +* has enough pr_credit (> 1), pr_credit reduces one for every pr data +* width write to PR_DATA register. If pr_credit <= 1, driver needs to +* wait for enough pr_credit from hardware by polling. */ pr_status =readq (fme_pr +FME_PR_STS ); pr_credit =FIELD_GET (FME_PR_STS_PR_CREDIT , pr_status);
@@ -192,7 +229,8 @@ static int fme_mgr_write(struct fpga_manager *mgr,while (pr_credit <=1 ) {if (delay++ >PR_WAIT_TIMEOUT ) {dev_err (dev,"PR_CREDIT timeout\n" ); -return -ETIMEDOUT ; + ret = -ETIMEDOUT ; + goto done; }udelay (1 );
@@ -200,21 +238,27 @@ static int fme_mgr_write(struct fpga_manager *mgr, pr_credit =FIELD_GET (FME_PR_STS_PR_CREDIT , pr_status); } -if (count <4 ) { -dev_err (dev,"Invalid PR bitstream size\n" ); -return -EINVAL ; +WARN_ON (count < priv->pr_datawidth); + +switch (priv->pr_datawidth) { +case 4 : + pr_data =FIELD_PREP (FME_PR_DATA_PR_DATA_RAW , + *(u32 *)buf); +writeq (pr_data, fme_pr +FME_PR_DATA ); +break ; +case 64 : +copy512 (buf, fme_pr +FME_PR_512_DATA ); +break ; +default : +WARN_ON_ONCE (1 ); } - - pr_data =0 ; - pr_data |=FIELD_PREP (FME_PR_DATA_PR_DATA_RAW , - *(((u32 *)buf) + i)); -writeq (pr_data, fme_pr +FME_PR_DATA ); - count -=4 ; + buf += priv->pr_datawidth; + count -= priv->pr_datawidth; pr_credit--; - i++; } -return 0 ; +done: +return ret; } static int fme_mgr_write_complete(struct fpga_manager *mgr,
@@ -279,6 +323,36 @@ static void fme_mgr_get_compat_id(void __iomem *fme_pr, id->id_h =readq (fme_pr +FME_PR_INTFC_ID_H ); } +static u8 fme_mgr_get_pr_datawidth (struct device *dev,void __iomem *fme_pr) +{ +u8 revision =dfl_feature_revision (fme_pr); + +if (revision <2 ) { +/* +* revision 0 and 1 only support 32bit data width partial +* reconfiguration, so pr_datawidth is 4 (Byte). +*/ +return 4 ; + }else if (revision ==2 ) { +/* +* revision 2 hardware has optimization to support 512bit data +* width partial reconfiguration with AVX512 instructions. So +* pr_datawidth is 64 (Byte). As revision 2 hardware is only +* used in integrated solution, CPU supports AVX512 instructions +* for sure, but it still needs to check here as AVX512 could be +* disabled in kernel (e.g. using clearcpuid boot option). +*/ +if (is_cpu_avx512_enabled ()) +return 64 ; + +dev_err (dev,"revision 2: AVX512 is disabled\n" ); +return 0 ; + } + +dev_err (dev,"revision %d is not supported yet\n" , revision); +return 0 ; +} +static int fme_mgr_probe (struct platform_device *pdev) {struct dfl_fme_mgr_pdata *pdata =dev_get_platdata (&pdev->dev);
@@ -302,6 +376,10 @@ static int fme_mgr_probe(struct platform_device *pdev)return PTR_ERR (priv->ioaddr); } + priv->pr_datawidth =fme_mgr_get_pr_datawidth (dev, priv->ioaddr); +if (!priv->pr_datawidth) +return -ENODEV ; + compat_id =devm_kzalloc (dev,sizeof (*compat_id),GFP_KERNEL );if (!compat_id)return -ENOMEM ;
drivers/fpga/dfl-fme-pr.c +28 -15
84 lines not on this page · raw message
drivers/fpga/dfl-fme.h +2 -0
@@ -21,12 +21,14 @@/** * struct dfl_fme - dfl fme private data * +* @pr_datawidth: data width for partial reconfiguration. * @mgr: FME's FPGA manager platform device. * @region_list: linked list of FME's FPGA regions. * @bridge_list: linked list of FME's FPGA bridges. * @pdata: fme platform device's pdata. */ struct dfl_fme { +int pr_datawidth;struct platform_device *mgr;struct list_head region_list;struct list_head bridge_list;
drivers/fpga/dfl.h +5 -0
@@ -331,6 +331,11 @@ static inline bool dfl_feature_is_port(void __iomem *base) (FIELD_GET (DFH_ID , v) ==DFH_ID_FIU_PORT ); } +static inline u8 dfl_feature_revision (void __iomem *base) +{ +return (u8 )FIELD_GET (DFH_REVISION ,readq (base +DFH )); +} +/** * struct dfl_fpga_enum_info - DFL FPGA enumeration information *
Signature · 1 line
1.8.3.1
Re: [PATCH v3 01/12] fpga: dfl: fme: support 512bit data width PR
Quoted 52 lines · Wu Hao
In early partial reconfiguration private feature, it only supports 32bit data width when writing data to hardware for PR. 512bit data width PR support is an important optimization for some specific solutions (e.g. XEON with FPGA integrated), it allows driver to use AVX512 instruction to improve the performance of partial reconfiguration. e.g. programming one 100MB bitstream image via this 512bit data width PR hardware only takes ~300ms, but 32bit revision requires ~3s per test result.
Please note now this optimization is only done on revision 2 of this PR private feature which is only used in integrated solution that AVX512 is always supported. This revision 2 hardware doesn't support 32bit PR.
Signed-off-by Ananda Ravuri <[email protected]>Signed-off-by Xu Yilun <[email protected]>Signed-off-by Wu Hao <[email protected]>Acked-by Alan Tull <[email protected]>Signed-off-by Moritz Fischer <[email protected]>---
v2: remove DRV/MODULE_VERSION modifications--- drivers/fpga/dfl-fme-mgr.c | 110 ++++++++++++++++++++++++++++++++++++++------- drivers/fpga/dfl-fme-pr.c | 43 +++++++++++------- drivers/fpga/dfl-fme.h | 2 + drivers/fpga/dfl.h | 5 +++ 4 files changed, 129 insertions(+), 31 deletions(-)drivers/fpga/dfl-fme-mgr.c +2
-0@@ -22,6 +22,7 @@#include <linux/io-64-nonatomic-lo-hi.h> #include <linux/fpga/fpga-mgr.h> +#include "dfl.h" #include "dfl-fme-pr.h" /* FME Partial Reconfiguration Sub Feature Register Set */ @@ -30,6 +31,7 @@#define FME_PR_STS 0x10#define FME_PR_DATA 0x18#define FME_PR_ERR 0x20 +#define FME_PR_512_DATA 0x40/* Data Register for 512bit datawidth PR */ #define FME_PR_INTFC_ID_L 0xA8#define FME_PR_INTFC_ID_H 0xB0@@ -67,8 +69,43 @@#define PR_WAIT_TIMEOUT 8000000#define PR_HOST_STATUS_IDLE 0
drivers/fpga/dfl-fme-mgr.c line 80
Quoted 10 lines
drivers/fpga/dfl-fme-mgr.c +9
-0+#if defined(CONFIG_X86 ) && defined(CONFIG_AS_AVX512 ) + +#include <linux/cpufeature.h> +#include <asm/fpu/api.h> + +static inline int is_cpu_avx512_enabled (void ) +{ +return cpu_feature_enabled (X86_FEATURE_AVX512F ); +}
That's a very arch specific function, why would a driver ever care about this?
drivers/fpga/dfl-fme-mgr.c line 93
Quoted 13 lines
drivers/fpga/dfl-fme-mgr.c +13
-0+ +static inline void copy512 (const void *src,void __iomem *dst) +{ +kernel_fpu_begin (); + + asmvolatile ("vmovdqu64 (%0), %%zmm0;" +"vmovntdq %%zmm0, (%1);" + : + :"r" (src),"r" (dst) + :"memory" ); + +kernel_fpu_end (); +}
Shouldn't this be an arch-specific function somewhere? Burying this in a random driver is not ok. Please make this generic for all systems.
drivers/fpga/dfl-fme-mgr.c line 102
Quoted 9 lines
drivers/fpga/dfl-fme-mgr.c +9
-0+#else +static inline int is_cpu_avx512_enabled (void ) +{ +return 0 ; +} + +static inline void copy512 (const void *src,void __iomem *dst) +{ +WARN_ON_ONCE (1 );
Are you trying to get reports from syzbot? :)
Please fix this all up.
greg k-h
Re: [PATCH v3 01/12] fpga: dfl: fme: support 512bit data width PR
Quoted 54 lines · Greg KH
Quoted 53 lines · Wu Hao
In early partial reconfiguration private feature, it only supports 32bit data width when writing data to hardware for PR. 512bit data width PR support is an important optimization for some specific solutions (e.g. XEON with FPGA integrated), it allows driver to use AVX512 instruction to improve the performance of partial reconfiguration. e.g. programming one 100MB bitstream image via this 512bit data width PR hardware only takes ~300ms, but 32bit revision requires ~3s per test result.
Please note now this optimization is only done on revision 2 of this PR private feature which is only used in integrated solution that AVX512 is always supported. This revision 2 hardware doesn't support 32bit PR.
Signed-off-by Ananda Ravuri <[email protected]>Signed-off-by Xu Yilun <[email protected]>Signed-off-by Wu Hao <[email protected]>Acked-by Alan Tull <[email protected]>Signed-off-by Moritz Fischer <[email protected]>---
v2: remove DRV/MODULE_VERSION modifications--- drivers/fpga/dfl-fme-mgr.c | 110 ++++++++++++++++++++++++++++++++++++++------- drivers/fpga/dfl-fme-pr.c | 43 +++++++++++------- drivers/fpga/dfl-fme.h | 2 + drivers/fpga/dfl.h | 5 +++ 4 files changed, 129 insertions(+), 31 deletions(-)drivers/fpga/dfl-fme-mgr.c +2
-0@@ -22,6 +22,7 @@#include <linux/io-64-nonatomic-lo-hi.h> #include <linux/fpga/fpga-mgr.h> +#include "dfl.h" #include "dfl-fme-pr.h" /* FME Partial Reconfiguration Sub Feature Register Set */ @@ -30,6 +31,7 @@#define FME_PR_STS 0x10#define FME_PR_DATA 0x18#define FME_PR_ERR 0x20 +#define FME_PR_512_DATA 0x40/* Data Register for 512bit datawidth PR */ #define FME_PR_INTFC_ID_L 0xA8#define FME_PR_INTFC_ID_H 0xB0@@ -67,8 +69,43 @@#define PR_WAIT_TIMEOUT 8000000#define PR_HOST_STATUS_IDLE 0
Quoted 12 lines
drivers/fpga/dfl-fme-mgr.c line 80
Quoted 9 lines
drivers/fpga/dfl-fme-mgr.c +9
-0+#if defined(CONFIG_X86 ) && defined(CONFIG_AS_AVX512 ) + +#include <linux/cpufeature.h> +#include <asm/fpu/api.h> + +static inline int is_cpu_avx512_enabled (void ) +{ +return cpu_feature_enabled (X86_FEATURE_AVX512F ); +}That's a very arch specific function, why would a driver ever care about this?
Yes, this is only applied to a specific FPGA solution, which FPGA has been integrated with XEON. Hardware indicates this using register to software. As it's cpu integrated solution, so CPU always has this AVX512 capability. The only check we do, is make sure this is not manually disabled by kernel.
With this hardware, software could use AVX512 to accelerate the FPGA partial reconfiguration as mentioned in the patch commit message. It brings performance benifits to people who uses it. This is only one optimization (512 vs 32bit data write to hw) for a specific hardware.
For other discrete solutions, e.g. FPGA PCIe Card, this is not used at all as driver does check hardware register to avoid any AVX512 code.
Quoted 17 lines
Quoted 4 lines
drivers/fpga/dfl-fme-mgr.c +3
-0+ +static inline void copy512 (const void *src,void __iomem *dst) +{drivers/fpga/dfl-fme-mgr.c line 93
Quoted 10 lines
drivers/fpga/dfl-fme-mgr.c +10
-0+kernel_fpu_begin (); + + asmvolatile ("vmovdqu64 (%0), %%zmm0;" +"vmovntdq %%zmm0, (%1);" + : + :"r" (src),"r" (dst) + :"memory" ); + +kernel_fpu_end (); +}Shouldn't this be an arch-specific function somewhere? Burying this in a random driver is not ok. Please make this generic for all systems.
If more people need the same avx operation like this in kernel, then maybe this can be moved to some arch-specific lib code somewhere as some common functions to everybody, but i am not very sure if this is the case. Let me think about this more.
Quoted 12 lines
drivers/fpga/dfl-fme-mgr.c line 102
Quoted 10 lines
drivers/fpga/dfl-fme-mgr.c +9
-0+#else +static inline int is_cpu_avx512_enabled (void ) +{ +return 0 ; +} + +static inline void copy512 (const void *src,void __iomem *dst) +{ +WARN_ON_ONCE (1 );Are you trying to get reports from syzbot? :)
Oh.. no.. I will remove it. :)
Thank you very much!
Hao
Quoted 4 lines
Please fix this all up.
greg k-h
Re: [PATCH v3 01/12] fpga: dfl: fme: support 512bit data width PR
Quoted 26 lines · Wu Hao
Quoted 6 lines · Greg KH
Quoted 5 lines · Wu Hao
drivers/fpga/dfl-fme-mgr.c +0
-0@@ -67,8 +69,43 @@#define PR_WAIT_TIMEOUT 8000000#define PR_HOST_STATUS_IDLE 0Quoted 12 lines
drivers/fpga/dfl-fme-mgr.c line 80
Quoted 9 lines
drivers/fpga/dfl-fme-mgr.c +9
-0+#if defined(CONFIG_X86 ) && defined(CONFIG_AS_AVX512 ) + +#include <linux/cpufeature.h> +#include <asm/fpu/api.h> + +static inline int is_cpu_avx512_enabled (void ) +{ +return cpu_feature_enabled (X86_FEATURE_AVX512F ); +}That's a very arch specific function, why would a driver ever care about this?
Yes, this is only applied to a specific FPGA solution, which FPGA has been integrated with XEON. Hardware indicates this using register to software. As it's cpu integrated solution, so CPU always has this AVX512 capability. The only check we do, is make sure this is not manually disabled by kernel.
Quoted 4 lines
With this hardware, software could use AVX512 to accelerate the FPGA partial reconfiguration as mentioned in the patch commit message. It brings performance benifits to people who uses it. This is only one optimization (512 vs 32bit data write to hw) for a specific hardware.
I thought earlier you said that 512 bit accesses were required for this particular integrated-only version of the device, and not just an optimization?
Quoted 12 lines
Quoted 11 lines
drivers/fpga/dfl-fme-mgr.c line 102
Quoted 9 lines
drivers/fpga/dfl-fme-mgr.c +9
-0+#else +static inline int is_cpu_avx512_enabled (void ) +{ +return 0 ; +} + +static inline void copy512 (const void *src,void __iomem *dst) +{ +WARN_ON_ONCE (1 );Are you trying to get reports from syzbot? :)
Quoted 3 lines
Oh.. no.. I will remove it. :)
Thank you very much!
What's wrong with this? The driver should never call copy512() if is_cpu_avx512_enabled() returns 0, and if syzbot can somehow make the driver do so, then yes we do want a report.
-Scott
Re: [PATCH v3 01/12] fpga: dfl: fme: support 512bit data width PR
Quoted 27 lines · Scott Wood
Quoted 26 lines · Wu Hao
Quoted 6 lines · Greg KH
Quoted 5 lines · Wu Hao
drivers/fpga/dfl-fme-mgr.c +0
-0@@ -67,8 +69,43 @@#define PR_WAIT_TIMEOUT 8000000#define PR_HOST_STATUS_IDLE 0Quoted 12 lines
drivers/fpga/dfl-fme-mgr.c line 80
Quoted 9 lines
drivers/fpga/dfl-fme-mgr.c +9
-0+#if defined(CONFIG_X86 ) && defined(CONFIG_AS_AVX512 ) + +#include <linux/cpufeature.h> +#include <asm/fpu/api.h> + +static inline int is_cpu_avx512_enabled (void ) +{ +return cpu_feature_enabled (X86_FEATURE_AVX512F ); +}That's a very arch specific function, why would a driver ever care about this?
Yes, this is only applied to a specific FPGA solution, which FPGA has been integrated with XEON. Hardware indicates this using register to software. As it's cpu integrated solution, so CPU always has this AVX512 capability. The only check we do, is make sure this is not manually disabled by kernel.
Quoted 8 lines
Quoted 4 lines
With this hardware, software could use AVX512 to accelerate the FPGA partial reconfiguration as mentioned in the patch commit message. It brings performance benifits to people who uses it. This is only one optimization (512 vs 32bit data write to hw) for a specific hardware.
I thought earlier you said that 512 bit accesses were required for this particular integrated-only version of the device, and not just an optimization?
yes, some optimization implemented in a specific integrated-only version of hardware, this patch is used to support that particular hardware. This is also the reason you see code here to check hardware revision in this patch.
Quoted 13 lines
Quoted 13 lines
Quoted 12 lines
drivers/fpga/dfl-fme-mgr.c line 102
Quoted 10 lines
drivers/fpga/dfl-fme-mgr.c +9
-0+#else +static inline int is_cpu_avx512_enabled (void ) +{ +return 0 ; +} + +static inline void copy512 (const void *src,void __iomem *dst) +{ +WARN_ON_ONCE (1 );Are you trying to get reports from syzbot? :)
Quoted 7 lines
Quoted 3 lines
Oh.. no.. I will remove it. :)
Thank you very much!
What's wrong with this? The driver should never call
copy512()ifis_cpu_avx512_enabled()returns 0, and if syzbot can somehow make the driver do so, then yes we do want a report.
Yes, you are right, in previous version, it doesn't have avx512 enable check there, so it's possible to have false reporting, it should be fine after driver does early check on this during probe. As this patch has been dropped from main patchset, may rework it later and resubmit. Thanks for the comments.
Hao
Quoted 3 lines
-Scott
PATCH v3 2/12 fpga: dfl: fme: add DFL_FPGA_FME_PORT_RELEASE/ASSIGN ioctl support.
In reply to Wu HaoIn order to support virtualization usage via PCIe SRIOV, this patch adds two ioctls under FPGA Management Engine (FME) to release and assign back the port device. In order to safely turn Port from PF into VF and enable PCIe SRIOV, it requires user to invoke this PORT_RELEASE ioctl to release port firstly to remove userspace interfaces, and then configure the PF/VF access register in FME. After disable SRIOV, it requires user to invoke this PORT_ASSIGN ioctl to attach the port back to PF.
Ioctl interfaces:
* DFL_FPGA_FME_PORT_RELEASE
Release platform device of given port, it deletes port platform device to remove related userspace interfaces on PF, then configures PF/VF access mode to VF.
* DFL_FPGA_FME_PORT_ASSIGN
Assign platform device of given port back to PF, it configures PF/VF access mode to PF, then adds port platform device back to re-enable related userspace interfaces on PF.
---
v2: remove argsz from ioctls.
--- drivers/fpga/dfl-fme-main.c | 30 ++++++++++++ drivers/fpga/dfl.c | 107 +++++++++++++++++++++++++++++++++++++----- drivers/fpga/dfl.h | 10 ++++ include/uapi/linux/fpga-dfl.h | 19 ++++++++ 4 files changed, 154 insertions(+), 12 deletions(-)
drivers/fpga/dfl-fme-main.c +30 -0
@@ -16,6 +16,7 @@#include <linux/kernel.h> #include <linux/module.h> +#include <linux/uaccess.h> #include <linux/fpga-dfl.h> #include "dfl.h"
@@ -104,9 +105,38 @@ static void fme_hdr_uinit(struct platform_device *pdev,device_remove_groups (&pdev->dev, fme_hdr_groups); } +static long fme_hdr_ioctl_config_port (struct dfl_feature_platform_data *pdata, +unsigned long arg,bool release) +{ +struct dfl_fpga_cdev *cdev = pdata->dfl_cdev; +int port_id; + +if (get_user (port_id, (int __user *)arg)) +return -EFAULT ; + +return dfl_fpga_cdev_config_port (cdev, port_id, release); +} + +static long fme_hdr_ioctl (struct platform_device *pdev, +struct dfl_feature *feature, +unsigned int cmd,unsigned long arg) +{ +struct dfl_feature_platform_data *pdata =dev_get_platdata (&pdev->dev); + +switch (cmd) { +case DFL_FPGA_FME_PORT_RELEASE : +return fme_hdr_ioctl_config_port (pdata, arg, true); +case DFL_FPGA_FME_PORT_ASSIGN : +return fme_hdr_ioctl_config_port (pdata, arg, false); + } + +return -ENODEV ; +} +static const struct dfl_feature_ops fme_hdr_ops = { .init = fme_hdr_init, .uinit = fme_hdr_uinit, + .ioctl = fme_hdr_ioctl, };static struct dfl_feature_driver fme_feature_drvs[] = {
drivers/fpga/dfl.c +95 -12
148 lines not on this page · raw message
drivers/fpga/dfl.h +10 -0
@@ -183,6 +183,8 @@ struct dfl_feature {#define DEV_STATUS_IN_USE 0 +#define FEATURE_DEV_ID_UNUSED (-1) +/** * struct dfl_feature_platform_data - platform data for feature devices *
@@ -191,6 +193,7 @@ struct dfl_feature {* @cdev: cdev of feature dev. * @dev: ptr to platform device linked with this platform data. * @dfl_cdev: ptr to container device. +* @id: id used for this feature device. * @disable_count: count for port disable. * @num: number for sub features. * @dev_status: dev status (e.g. DEV_STATUS_IN_USE).
@@ -203,6 +206,7 @@ struct dfl_feature_platform_data {struct cdev cdev;struct platform_device *dev;struct dfl_fpga_cdev *dfl_cdev; +int id;unsigned int disable_count;unsigned long dev_status;void *private;
@@ -378,6 +382,7 @@ int dfl_fpga_enum_info_add_dfl(struct dfl_fpga_enum_info *info,* @fme_dev: FME feature device under this container device. * @lock: mutex lock to protect the port device list. * @port_dev_list: list of all port feature devices under this container device. +* @released_port_num: released port number under this container device. */ struct dfl_fpga_cdev {struct device *parent;
@@ -385,6 +390,7 @@ struct dfl_fpga_cdev {struct device *fme_dev;struct mutex lock;struct list_head port_dev_list; +int released_port_num; }; struct dfl_fpga_cdev *
@@ -412,4 +418,8 @@ struct platform_device * return pdev; } + +int dfl_fpga_cdev_config_port (struct dfl_fpga_cdev *cdev, +int port_id,bool release); +#endif /* __FPGA_DFL_H */
include/uapi/linux/fpga-dfl.h +19 -0
@@ -176,4 +176,23 @@ struct dfl_fpga_fme_port_pr {#define DFL_FPGA_FME_PORT_PR _IO(DFL_FPGA_MAGIC, DFL_FME_BASE + 0) +/** +* DFL_FPGA_FME_PORT_RELEASE - _IOW(DFL_FPGA_MAGIC, DFL_FME_BASE + 1, +* int port_id) +* +* Driver releases the port per Port ID provided by caller. +* Return: 0 on success, -errno on failure. +*/ +#define DFL_FPGA_FME_PORT_RELEASE _IOW(DFL_FPGA_MAGIC, DFL_FME_BASE + 1, int) + +/** +* DFL_FPGA_FME_PORT_ASSIGN - _IOW(DFL_FPGA_MAGIC, DFL_FME_BASE + 2, +* int port_id) +* +* Driver assigns the port back per Port ID provided by caller. +* Return: 0 on success, -errno on failure. +*/ + +#define DFL_FPGA_FME_PORT_ASSIGN _IOW(DFL_FPGA_MAGIC, DFL_FME_BASE + 2, int) +#endif /* _UAPI_LINUX_FPGA_DFL_H */
Signature · 1 line
1.8.3.1
Re: [PATCH v3 02/12] fpga: dfl: fme: add DFL_FPGA_FME_PORT_RELEASE/ASSIGN ioctl support.
Quoted 7 lines · Wu Hao
drivers/fpga/dfl.c +7
-0+/** +* dfl_fpga_cdev_config_port - configure a port feature dev +* @cdev: parent container device. +* @port_id: id of the port feature device. +* @release: release port or assign port back. +* +* This function allows user to release port platform device or assign it back.
drivers/fpga/dfl.c struct platform_device * · line 1113
Quoted 10 lines
drivers/fpga/dfl.c +10
-0+* e.g. to safely turn one port from PF into VF for PCI device SRIOV support, +* release port platform device is one necessary step. +*/ +int dfl_fpga_cdev_config_port (struct dfl_fpga_cdev *cdev,int port_id, +bool release) +{ +return release ?detach_port_dev (cdev, port_id) : +attach_port_dev (cdev, port_id); +} +EXPORT_SYMBOL_GPL (dfl_fpga_cdev_config_port);
That's a horrible api. Every time you see this call in code, you have to go and look up what "bool" means here. There's no reason for it.
Just have 2 different functions, one that attaches a port, and one that detaches it. That way when you read the code that calls this function, you know what it does instantly without having to go look up some api function somewhere else.
Write code for people to read first. And you are saving nothing here by trying to do two different things in the same exact function.
thanks,
greg k-h
Re: [PATCH v3 02/12] fpga: dfl: fme: add DFL_FPGA_FME_PORT_RELEASE/ASSIGN ioctl support.
Quoted 22 lines · Greg KH
Quoted 7 lines · Wu Hao
drivers/fpga/dfl.c +7
-0+/** +* dfl_fpga_cdev_config_port - configure a port feature dev +* @cdev: parent container device. +* @port_id: id of the port feature device. +* @release: release port or assign port back. +* +* This function allows user to release port platform device or assign it back. drivers/fpga/dfl.c struct platform_device * · line 1113
Quoted 10 lines
drivers/fpga/dfl.c +10
-0+* e.g. to safely turn one port from PF into VF for PCI device SRIOV support, +* release port platform device is one necessary step. +*/ +int dfl_fpga_cdev_config_port (struct dfl_fpga_cdev *cdev,int port_id, +bool release) +{ +return release ?detach_port_dev (cdev, port_id) : +attach_port_dev (cdev, port_id); +} +EXPORT_SYMBOL_GPL (dfl_fpga_cdev_config_port);That's a horrible api. Every time you see this call in code, you have to go and look up what "bool" means here. There's no reason for it.
Quoted 7 lines
Just have 2 different functions, one that attaches a port, and one that detaches it. That way when you read the code that calls this function, you know what it does instantly without having to go look up some api function somewhere else.
Write code for people to read first. And you are saving nothing here by trying to do two different things in the same exact function.
I see, you're right, it saves everybody's time on reading, very important. I will fix this and keep it in mind. Thank you.
Hao
Quoted 4 lines
thanks,
greg k-h
PATCH v3 3/12 fpga: dfl: pci: enable SRIOV support.
In reply to Wu HaoThis patch enables the standard sriov support. It allows user to enable SRIOV (and VFs), then user could pass through accelerators (VFs) into virtual machine or use VFs directly in host.
---
v2: remove DRV/MODULE_VERSION modifications.
--- drivers/fpga/dfl-pci.c | 39 +++++++++++++++++++++++++++++++++++++++ drivers/fpga/dfl.c | 41 +++++++++++++++++++++++++++++++++++++++++ drivers/fpga/dfl.h | 1 + 3 files changed, 81 insertions(+)
drivers/fpga/dfl-pci.c +39 -0
@@ -223,8 +223,46 @@ int cci_pci_probe(struct pci_dev *pcidev, const struct pci_device_id *pcidevid)return ret; } +static int cci_pci_sriov_configure (struct pci_dev *pcidev,int num_vfs) +{ +struct cci_drvdata *drvdata =pci_get_drvdata (pcidev); +struct dfl_fpga_cdev *cdev = drvdata->cdev; +int ret =0 ; + +mutex_lock (&cdev->lock); + +if (!num_vfs) { +/* +* disable SRIOV and then put released ports back to default +* PF access mode. +*/ +pci_disable_sriov (pcidev); + +__dfl_fpga_cdev_config_port_vf (cdev, false); + + }else if (cdev->released_port_num == num_vfs) { +/* +* only enable SRIOV if cdev has matched released ports, put +* released ports into VF access mode firstly. +*/ +__dfl_fpga_cdev_config_port_vf (cdev, true); + + ret =pci_enable_sriov (pcidev, num_vfs); +if (ret) +__dfl_fpga_cdev_config_port_vf (cdev, false); + }else { + ret = -EINVAL ; + } + +mutex_unlock (&cdev->lock); +return ret; +} +static void cci_pci_remove (struct pci_dev *pcidev) { +if (dev_is_pf (&pcidev->dev)) +cci_pci_sriov_configure (pcidev,0 ); +cci_remove_feature_devs (pcidev);pci_disable_pcie_error_reporting (pcidev); }
@@ -234,6 +272,7 @@ static void cci_pci_remove(struct pci_dev *pcidev) .id_table = cci_pcie_id_tbl, .probe = cci_pci_probe, .remove = cci_pci_remove, + .sriov_configure = cci_pci_sriov_configure, };module_pci_driver (cci_pci_driver);
drivers/fpga/dfl.c +41 -0
47 lines not on this page · raw message
drivers/fpga/dfl.h +1 -0
@@ -421,5 +421,6 @@ struct platform_device *int dfl_fpga_cdev_config_port (struct dfl_fpga_cdev *cdev,int port_id,bool release); +void __dfl_fpga_cdev_config_port_vf (struct dfl_fpga_cdev *cdev,bool is_vf);#endif /* __FPGA_DFL_H */
Signature · 1 line
1.8.3.1
Re: [PATCH v3 03/12] fpga: dfl: pci: enable SRIOV support.
Quoted 115 lines · Wu Hao
This patch enables the standard sriov support. It allows user to enable SRIOV (and VFs), then user could pass through accelerators (VFs) into virtual machine or use VFs directly in host.
Signed-off-by Zhang Yi Z <[email protected]>Signed-off-by Xu Yilun <[email protected]>Signed-off-by Wu Hao <[email protected]>Acked-by Alan Tull <[email protected]>Acked-by Moritz Fischer <[email protected]>Signed-off-by Moritz Fischer <[email protected]>---
v2: remove DRV/MODULE_VERSION modifications.--- drivers/fpga/dfl-pci.c | 39 +++++++++++++++++++++++++++++++++++++++ drivers/fpga/dfl.c | 41 +++++++++++++++++++++++++++++++++++++++++ drivers/fpga/dfl.h | 1 + 3 files changed, 81 insertions(+)drivers/fpga/dfl-pci.c +39
-0@@ -223,8 +223,46 @@ int cci_pci_probe(struct pci_dev *pcidev, const struct pci_device_id *pcidevid)return ret; } +static int cci_pci_sriov_configure (struct pci_dev *pcidev,int num_vfs) +{ +struct cci_drvdata *drvdata =pci_get_drvdata (pcidev); +struct dfl_fpga_cdev *cdev = drvdata->cdev; +int ret =0 ; + +mutex_lock (&cdev->lock); + +if (!num_vfs) { +/* +* disable SRIOV and then put released ports back to default +* PF access mode. +*/ +pci_disable_sriov (pcidev); + +__dfl_fpga_cdev_config_port_vf (cdev, false); + + }else if (cdev->released_port_num == num_vfs) { +/* +* only enable SRIOV if cdev has matched released ports, put +* released ports into VF access mode firstly. +*/ +__dfl_fpga_cdev_config_port_vf (cdev, true); + + ret =pci_enable_sriov (pcidev, num_vfs); +if (ret) +__dfl_fpga_cdev_config_port_vf (cdev, false); + }else { + ret = -EINVAL ; + } + +mutex_unlock (&cdev->lock); +return ret; +} +static void cci_pci_remove (struct pci_dev *pcidev) { +if (dev_is_pf (&pcidev->dev)) +cci_pci_sriov_configure (pcidev,0 ); +cci_remove_feature_devs (pcidev);pci_disable_pcie_error_reporting (pcidev); }@@ -234,6 +272,7 @@ static void cci_pci_remove(struct pci_dev *pcidev) .id_table = cci_pcie_id_tbl, .probe = cci_pci_probe, .remove = cci_pci_remove, + .sriov_configure = cci_pci_sriov_configure, };module_pci_driver (cci_pci_driver);drivers/fpga/dfl.c +30
-0@@ -1112,6 +1112,47 @@ int dfl_fpga_cdev_config_port(struct dfl_fpga_cdev *cdev, int port_id, }EXPORT_SYMBOL_GPL (dfl_fpga_cdev_config_port); +static void config_port_vf (struct device *fme_dev,int port_id,bool is_vf) +{ +void __iomem *base; +u64 v; + + base =dfl_get_feature_ioaddr_by_id (fme_dev,FME_FEATURE_ID_HEADER ); + + v =readq (base +FME_HDR_PORT_OFST (port_id)); + + v &= ~FME_PORT_OFST_ACC_CTRL ; + v |=FIELD_PREP (FME_PORT_OFST_ACC_CTRL , + is_vf ?FME_PORT_OFST_ACC_VF :FME_PORT_OFST_ACC_PF ); + +writeq (v, base +FME_HDR_PORT_OFST (port_id)); +} + +/** +* __dfl_fpga_cdev_config_port_vf - configure port to VF access mode +* +* @cdev: parent container device. +* @if_vf: true for VF access mode, and false for PF access mode +* +* Return: 0 on success, negative error code otherwise. +* +* This function is needed in sriov configuration routine. It could be used to +* configures the released ports access mode to VF or PF. +* The caller needs to hold lock for protection. +*/ +void __dfl_fpga_cdev_config_port_vf (struct dfl_fpga_cdev *cdev,bool is_vf) +{
drivers/fpga/dfl.c int dfl_fpga_cdev_config_port(struct dfl_fpga_cdev *cdev, int port_id, · line 1154
Quoted 10 lines
drivers/fpga/dfl.c +10
-0+struct dfl_feature_platform_data *pdata; + +list_for_each_entry (pdata, &cdev->port_dev_list, node) { +if (device_is_registered (&pdata->dev->dev)) +continue ; + +config_port_vf (cdev->fme_dev, pdata->id, is_vf); + } +} +EXPORT_SYMBOL_GPL (__dfl_fpga_cdev_config_port_vf);
Why are you exporting a function with a leading __?
You are expecting someone else, in who knows what code, to do locking correctly? If so, and the caller always has to have a local lock, then it's not a big deal, just drop the '__', otherwise if you have to have a specific lock for a specific device, then you have a really complex and probably broken api here :(
thanks,
greg k-h
Re: [PATCH v3 03/12] fpga: dfl: pci: enable SRIOV support.
Quoted 129 lines · Greg KH
Quoted 115 lines · Wu Hao
This patch enables the standard sriov support. It allows user to enable SRIOV (and VFs), then user could pass through accelerators (VFs) into virtual machine or use VFs directly in host.
Signed-off-by Zhang Yi Z <[email protected]>Signed-off-by Xu Yilun <[email protected]>Signed-off-by Wu Hao <[email protected]>Acked-by Alan Tull <[email protected]>Acked-by Moritz Fischer <[email protected]>Signed-off-by Moritz Fischer <[email protected]>---
v2: remove DRV/MODULE_VERSION modifications.--- drivers/fpga/dfl-pci.c | 39 +++++++++++++++++++++++++++++++++++++++ drivers/fpga/dfl.c | 41 +++++++++++++++++++++++++++++++++++++++++ drivers/fpga/dfl.h | 1 + 3 files changed, 81 insertions(+)drivers/fpga/dfl-pci.c +39
-0@@ -223,8 +223,46 @@ int cci_pci_probe(struct pci_dev *pcidev, const struct pci_device_id *pcidevid)return ret; } +static int cci_pci_sriov_configure (struct pci_dev *pcidev,int num_vfs) +{ +struct cci_drvdata *drvdata =pci_get_drvdata (pcidev); +struct dfl_fpga_cdev *cdev = drvdata->cdev; +int ret =0 ; + +mutex_lock (&cdev->lock); + +if (!num_vfs) { +/* +* disable SRIOV and then put released ports back to default +* PF access mode. +*/ +pci_disable_sriov (pcidev); + +__dfl_fpga_cdev_config_port_vf (cdev, false); + + }else if (cdev->released_port_num == num_vfs) { +/* +* only enable SRIOV if cdev has matched released ports, put +* released ports into VF access mode firstly. +*/ +__dfl_fpga_cdev_config_port_vf (cdev, true); + + ret =pci_enable_sriov (pcidev, num_vfs); +if (ret) +__dfl_fpga_cdev_config_port_vf (cdev, false); + }else { + ret = -EINVAL ; + } + +mutex_unlock (&cdev->lock); +return ret; +} +static void cci_pci_remove (struct pci_dev *pcidev) { +if (dev_is_pf (&pcidev->dev)) +cci_pci_sriov_configure (pcidev,0 ); +cci_remove_feature_devs (pcidev);pci_disable_pcie_error_reporting (pcidev); }@@ -234,6 +272,7 @@ static void cci_pci_remove(struct pci_dev *pcidev) .id_table = cci_pcie_id_tbl, .probe = cci_pci_probe, .remove = cci_pci_remove, + .sriov_configure = cci_pci_sriov_configure, };module_pci_driver (cci_pci_driver);drivers/fpga/dfl.c +30
-0@@ -1112,6 +1112,47 @@ int dfl_fpga_cdev_config_port(struct dfl_fpga_cdev *cdev, int port_id, }EXPORT_SYMBOL_GPL (dfl_fpga_cdev_config_port); +static void config_port_vf (struct device *fme_dev,int port_id,bool is_vf) +{ +void __iomem *base; +u64 v; + + base =dfl_get_feature_ioaddr_by_id (fme_dev,FME_FEATURE_ID_HEADER ); + + v =readq (base +FME_HDR_PORT_OFST (port_id)); + + v &= ~FME_PORT_OFST_ACC_CTRL ; + v |=FIELD_PREP (FME_PORT_OFST_ACC_CTRL , + is_vf ?FME_PORT_OFST_ACC_VF :FME_PORT_OFST_ACC_PF ); + +writeq (v, base +FME_HDR_PORT_OFST (port_id)); +} + +/** +* __dfl_fpga_cdev_config_port_vf - configure port to VF access mode +* +* @cdev: parent container device. +* @if_vf: true for VF access mode, and false for PF access mode +* +* Return: 0 on success, negative error code otherwise. +* +* This function is needed in sriov configuration routine. It could be used to +* configures the released ports access mode to VF or PF. +* The caller needs to hold lock for protection. +*/ +void __dfl_fpga_cdev_config_port_vf (struct dfl_fpga_cdev *cdev,bool is_vf) +{drivers/fpga/dfl.c int dfl_fpga_cdev_config_port(struct dfl_fpga_cdev *cdev, int port_id, · line 1154
Quoted 10 lines
drivers/fpga/dfl.c +10
-0+struct dfl_feature_platform_data *pdata; + +list_for_each_entry (pdata, &cdev->port_dev_list, node) { +if (device_is_registered (&pdata->dev->dev)) +continue ; + +config_port_vf (cdev->fme_dev, pdata->id, is_vf); + } +} +EXPORT_SYMBOL_GPL (__dfl_fpga_cdev_config_port_vf);Why are you exporting a function with a leading __?
Quoted 5 lines
You are expecting someone else, in who knows what code, to do locking correctly? If so, and the caller always has to have a local lock, then it's not a big deal, just drop the '__', otherwise if you have to have a specific lock for a specific device, then you have a really complex and probably broken api here :(
Yes, I just want to remind the user of this API, caller needs to hold the lock to protect the list. I fully agree, it does make sense to make the APIs easy to use. I will try to improve this, maybe move the lock inside this function, then API user doesn't need to know the details of locking.
Thanks a lot for the comments, it really helps.
Hao
Quoted 4 lines
thanks,
greg k-h
PATCH v3 4/12 fpga: dfl: afu: add AFU state related sysfs interfaces
In reply to Wu HaoThis patch introduces more sysfs interfaces for Accelerated Function Unit (AFU). These interfaces allow users to read current AFU Power State (APx), read / clear AFU Power (APx) events which are sticky to identify transient APx state, and manage AFU's LTR (latency tolerance reporting).
---
v2: rebased, and remove DRV/MODULE_VERSION modifications
v3: update kernel version and date in sysfs doc
--- Documentation/ABI/testing/sysfs-platform-dfl-port | 30 +++++ drivers/fpga/dfl-afu-main.c | 137 ++++++++++++++++++++++ drivers/fpga/dfl.h | 11 ++ 3 files changed, 178 insertions(+)
Documentation/ABI/testing/sysfs-platform-dfl-port +30 -0
@@ -14,3 +14,33 @@ Description: Read-only. User can program different PR bitstreams to FPGA Accelerator Function Unit (AFU) for different functions. It returns uuid which could be used to identify which PR bitstream is programmed in this AFU. + +What: /sys/bus/platform/devices/dfl-port.0/power_state +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-only. It reports the APx (AFU Power) state, different APx + means different throttling level. When reading this file, it + returns "0" - Normal / "1" - AP1 / "2" - AP2 / "6" - AP6. + +What: /sys/bus/platform/devices/dfl-port.0/ap1_event +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-write. Read or set 1 to clear AP1 (AFU Power State 1) + event. It's used to indicate transient AP1 state. + +What: /sys/bus/platform/devices/dfl-port.0/ap2_event +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-write. Read or set 1 to clear AP2 (AFU Power State 2) + event. It's used to indicate transient AP2 state. + +What: /sys/bus/platform/devices/dfl-port.0/ltr +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-write. Read and set AFU latency tolerance reporting value. + Set ltr to 1 if the AFU can tolerate latency >= 40us or set it + to 0 if it is latency sensitive.
drivers/fpga/dfl-afu-main.c +137 -0
145 lines not on this page · raw message
drivers/fpga/dfl.h +11 -0
@@ -119,6 +119,7 @@#define PORT_HDR_NEXT_AFU NEXT_AFU#define PORT_HDR_CAP 0x30#define PORT_HDR_CTRL 0x38 +#define PORT_HDR_STS 0x40/* Port Capability Register Bitfield */ #define PORT_CAP_PORT_NUM GENMASK_ULL(1, 0)/* ID of this port */
@@ -130,6 +131,16 @@/* Latency tolerance reporting. '1' >= 40us, '0' < 40us.*/ #define PORT_CTRL_LATENCY BIT_ULL(2)#define PORT_CTRL_SFTRST_ACK BIT_ULL(4)/* HW ack for reset */ + +/* Port Status Register Bitfield */ +#define PORT_STS_AP2_EVT BIT_ULL(13)/* AP2 event detected */ +#define PORT_STS_AP1_EVT BIT_ULL(12)/* AP1 event detected */ +#define PORT_STS_PWR_STATE GENMASK_ULL(11, 8)/* AFU power states */ +#define PORT_STS_PWR_STATE_NORM 0 +#define PORT_STS_PWR_STATE_AP1 1/* 50% throttling */ +#define PORT_STS_PWR_STATE_AP2 2/* 90% throttling */ +#define PORT_STS_PWR_STATE_AP6 6/* 100% throttling */ +/** * struct dfl_fpga_port_ops - port ops *
Signature · 1 line
1.8.3.1
Re: [PATCH v3 04/12] fpga: dfl: afu: add AFU state related sysfs interfaces
Quoted 33 lines · Wu Hao
This patch introduces more sysfs interfaces for Accelerated Function Unit (AFU). These interfaces allow users to read current AFU Power State (APx), read / clear AFU Power (APx) events which are sticky to identify transient APx state, and manage AFU's LTR (latency tolerance reporting).
Signed-off-by Ananda Ravuri <[email protected]>Signed-off-by Xu Yilun <[email protected]>Signed-off-by Wu Hao <[email protected]>Acked-by Alan Tull <[email protected]>Signed-off-by Moritz Fischer <[email protected]>---
v2: rebased, and remove DRV/MODULE_VERSION modifications
v3: update kernel version and date in sysfs doc--- Documentation/ABI/testing/sysfs-platform-dfl-port | 30 +++++ drivers/fpga/dfl-afu-main.c | 137 ++++++++++++++++++++++ drivers/fpga/dfl.h | 11 ++ 3 files changed, 178 insertions(+)Documentation/ABI/testing/sysfs-platform-dfl-port +5
-0@@ -14,3 +14,33 @@ Description: Read-only. User can program different PR bitstreams to FPGA Accelerator Function Unit (AFU) for different functions. It returns uuid which could be used to identify which PR bitstream is programmed in this AFU. + +What: /sys/bus/platform/devices/dfl-port.0/power_state +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]>
Documentation/ABI/testing/sysfs-platform-dfl-port Description: Read-only. User can program different PR bitstreams to FPGA · line 31
Quoted 10 lines
Documentation/ABI/testing/sysfs-platform-dfl-port +10
-0+Description: Read-only. It reports the APx (AFU Power) state, different APx + means different throttling level. When reading this file, it + returns "0" - Normal / "1" - AP1 / "2" - AP2 / "6" - AP6. + +What: /sys/bus/platform/devices/dfl-port.0/ap1_event +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-write. Read or set 1 to clear AP1 (AFU Power State 1) + event. It's used to indicate transient AP1 state.
So reading the value changes the state of the system? That's almost always never a good idea.
Force userspace to write the value to change something. Otherwise all libraries that use sysfs will be accidentally changing the state of your system without you ever knowing it.
Quoted 25 lines
Documentation/ABI/testing/sysfs-platform-dfl-port +15
-0+ +What: /sys/bus/platform/devices/dfl-port.0/ap2_event +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-write. Read or set 1 to clear AP2 (AFU Power State 2) + event. It's used to indicate transient AP2 state. + +What: /sys/bus/platform/devices/dfl-port.0/ltr +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-write. Read and set AFU latency tolerance reporting value. + Set ltr to 1 if the AFU can tolerate latency >= 40us or set it + to 0 if it is latency sensitive.drivers/fpga/dfl-afu-main.c +2
-0@@ -141,8 +141,145 @@ static int port_get_id(struct platform_device *pdev) }static DEVICE_ATTR_RO (id); +static ssize_t +ltr_show (struct device *dev,struct device_attribute *attr,char *buf)
drivers/fpga/dfl-afu-main.c static int port_get_id(struct platform_device *pdev) · line 155
Quoted 10 lines
drivers/fpga/dfl-afu-main.c +10
-0+{ +struct dfl_feature_platform_data *pdata =dev_get_platdata (dev); +void __iomem *base; +u64 v; + + base =dfl_get_feature_ioaddr_by_id (dev,PORT_FEATURE_ID_HEADER ); + +mutex_lock (&pdata->lock); + v =readq (base +PORT_HDR_CTRL ); +mutex_unlock (&pdata->lock);
Why do you need a lock to call readq()? What are you protecting here?
Quoted 5 lines
drivers/fpga/dfl-afu-main.c +5
-0+ +return sprintf (buf,"%x\n" , (u8 )FIELD_GET (PORT_CTRL_LATENCY , v)); +} + +static ssize_t
drivers/fpga/dfl-afu-main.c static int port_get_id(struct platform_device *pdev) · line 170
Quoted 10 lines
drivers/fpga/dfl-afu-main.c +10
-0+ltr_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct dfl_feature_platform_data *pdata = dev_get_platdata(dev); +void __iomem *base; +u8 ltr; +u64 v; + +if (kstrtou8 (buf,0 , <r) || ltr >1 ) +return -EINVAL ;
Are you doing anything with this value? If not, how about just using the sysfs boolean read function and if it is 1, then do the clearing?
Same for all other show/store functions in here.
thanks,
greg k-h
Re: [PATCH v3 04/12] fpga: dfl: afu: add AFU state related sysfs interfaces
Quoted 48 lines · Greg KH
Quoted 33 lines · Wu Hao
This patch introduces more sysfs interfaces for Accelerated Function Unit (AFU). These interfaces allow users to read current AFU Power State (APx), read / clear AFU Power (APx) events which are sticky to identify transient APx state, and manage AFU's LTR (latency tolerance reporting).
Signed-off-by Ananda Ravuri <[email protected]>Signed-off-by Xu Yilun <[email protected]>Signed-off-by Wu Hao <[email protected]>Acked-by Alan Tull <[email protected]>Signed-off-by Moritz Fischer <[email protected]>---
v2: rebased, and remove DRV/MODULE_VERSION modifications
v3: update kernel version and date in sysfs doc--- Documentation/ABI/testing/sysfs-platform-dfl-port | 30 +++++ drivers/fpga/dfl-afu-main.c | 137 ++++++++++++++++++++++ drivers/fpga/dfl.h | 11 ++ 3 files changed, 178 insertions(+)Documentation/ABI/testing/sysfs-platform-dfl-port +5
-0@@ -14,3 +14,33 @@ Description: Read-only. User can program different PR bitstreams to FPGA Accelerator Function Unit (AFU) for different functions. It returns uuid which could be used to identify which PR bitstream is programmed in this AFU. + +What: /sys/bus/platform/devices/dfl-port.0/power_state +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]>Documentation/ABI/testing/sysfs-platform-dfl-port Description: Read-only. User can program different PR bitstreams to FPGA · line 31
Quoted 10 lines
Documentation/ABI/testing/sysfs-platform-dfl-port +10
-0+Description: Read-only. It reports the APx (AFU Power) state, different APx + means different throttling level. When reading this file, it + returns "0" - Normal / "1" - AP1 / "2" - AP2 / "6" - AP6. + +What: /sys/bus/platform/devices/dfl-port.0/ap1_event +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-write. Read or set 1 to clear AP1 (AFU Power State 1) + event. It's used to indicate transient AP1 state.So reading the value changes the state of the system? That's almost always never a good idea.
Quoted 3 lines
Force userspace to write the value to change something. Otherwise all libraries that use sysfs will be accidentally changing the state of your system without you ever knowing it.
Oh.. I think the description makes some misunderstanding here, will fix it in the next version. This AP1/AP2 event will only be cleared by write 1 to it, read will not change the state.
Quoted 24 lines
Quoted 24 lines
Documentation/ABI/testing/sysfs-platform-dfl-port +15
-0+ +What: /sys/bus/platform/devices/dfl-port.0/ap2_event +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-write. Read or set 1 to clear AP2 (AFU Power State 2) + event. It's used to indicate transient AP2 state. + +What: /sys/bus/platform/devices/dfl-port.0/ltr +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-write. Read and set AFU latency tolerance reporting value. + Set ltr to 1 if the AFU can tolerate latency >= 40us or set it + to 0 if it is latency sensitive.drivers/fpga/dfl-afu-main.c +0
-0@@ -141,8 +141,145 @@ static int port_get_id(struct platform_device *pdev) }static DEVICE_ATTR_RO (id);
Quoted 14 lines
drivers/fpga/dfl-afu-main.c static int port_get_id(struct platform_device *pdev) · line 155
Quoted 12 lines
drivers/fpga/dfl-afu-main.c +12
-0+static ssize_t +ltr_show (struct device *dev,struct device_attribute *attr,char *buf) +{ +struct dfl_feature_platform_data *pdata =dev_get_platdata (dev); +void __iomem *base; +u64 v; + + base =dfl_get_feature_ioaddr_by_id (dev,PORT_FEATURE_ID_HEADER ); + +mutex_lock (&pdata->lock); + v =readq (base +PORT_HDR_CTRL ); +mutex_unlock (&pdata->lock);Why do you need a lock to call
readq()? What are you protecting here?
If this code is running on 32bit machine, readq will be replaced with 2 readl operation. If that is the case, should we protect the code against it?
Quoted 21 lines
Quoted 7 lines
drivers/fpga/dfl-afu-main.c +5
-0+ +return sprintf (buf,"%x\n" , (u8 )FIELD_GET (PORT_CTRL_LATENCY , v)); +} + +static ssize_tdrivers/fpga/dfl-afu-main.c static int port_get_id(struct platform_device *pdev) · line 170
Quoted 10 lines
drivers/fpga/dfl-afu-main.c +10
-0+ltr_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct dfl_feature_platform_data *pdata = dev_get_platdata(dev); +void __iomem *base; +u8 ltr; +u64 v; + +if (kstrtou8 (buf,0 , <r) || ltr >1 ) +return -EINVAL ;Are you doing anything with this value? If not, how about just using the sysfs boolean read function and if it is 1, then do the clearing?
Quoted 1 line
Same for all other show/store functions in here.
Sure, will fix this in the next version.
Thanks a lot for the comments.
Hao
Quoted 4 lines
thanks,
greg k-h
PATCH v3 5/12 fpga: dfl: afu: add userclock sysfs interfaces.
In reply to Wu HaoThis patch introduces userclock sysfs interfaces for AFU, user could use these interfaces for clock setting to AFU.
Please note that, this is only working for port header feature with revision 0, for later revisions, userclock setting is moved to a separated private feature, so one revision sysfs interface is exposed to userspace application for this purpose too.
---
v2: rebased, and switched to use device_add/remove_groups for sysfs
v3: update kernel version and date in sysfs doc
--- Documentation/ABI/testing/sysfs-platform-dfl-port | 35 +++++++ drivers/fpga/dfl-afu-main.c | 114 +++++++++++++++++++++- drivers/fpga/dfl.h | 4 + 3 files changed, 152 insertions(+), 1 deletion(-)
Documentation/ABI/testing/sysfs-platform-dfl-port +35 -0
@@ -44,3 +44,38 @@ Contact: Wu Hao <[email protected]> Description: Read-write. Read and set AFU latency tolerance reporting value. Set ltr to 1 if the AFU can tolerate latency >= 40us or set it to 0 if it is latency sensitive. + +What: /sys/bus/platform/devices/dfl-port.0/revision +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-only. Read this file to get the revision of port header + feature. + +What: /sys/bus/platform/devices/dfl-port.0/userclk_freqcmd +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Write-only. User writes command to this interface to set + userclock to AFU. + +What: /sys/bus/platform/devices/dfl-port.0/userclk_freqsts +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-only. Read this file to get the status of issued command + to userclck_freqcmd. + +What: /sys/bus/platform/devices/dfl-port.0/userclk_freqcntrcmd +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Write-only. User writes command to this interface to set + userclock counter. + +What: /sys/bus/platform/devices/dfl-port.0/userclk_freqcntrsts +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-only. Read this file to get the status of issued command + to userclck_freqcntrcmd.
drivers/fpga/dfl-afu-main.c +113 -1
145 lines not on this page · raw message
drivers/fpga/dfl.h +4 -0
@@ -120,6 +120,10 @@#define PORT_HDR_CAP 0x30#define PORT_HDR_CTRL 0x38#define PORT_HDR_STS 0x40 +#define PORT_HDR_USRCLK_CMD0 0x50 +#define PORT_HDR_USRCLK_CMD1 0x58 +#define PORT_HDR_USRCLK_STS0 0x60 +#define PORT_HDR_USRCLK_STS1 0x68/* Port Capability Register Bitfield */ #define PORT_CAP_PORT_NUM GENMASK_ULL(1, 0)/* ID of this port */
Signature · 1 line
1.8.3.1
PATCH v3 6/12 fpga: dfl: add id_table for dfl private feature driver
In reply to Wu HaoThis patch adds id_table for each dfl private feature driver, it allows to reuse same private feature driver to match and support multiple dfl private features.
---
v2: rebased, remove DRV/MODULE_VERSION modifications
--- drivers/fpga/dfl-afu-main.c | 14 ++++++++++++-- drivers/fpga/dfl-fme-main.c | 11 ++++++++--- drivers/fpga/dfl-fme-pr.c | 7 ++++++- drivers/fpga/dfl-fme.h | 3 ++- drivers/fpga/dfl.c | 18 ++++++++++++++++-- drivers/fpga/dfl.h | 21 +++++++++++++++------ 6 files changed, 59 insertions(+), 15 deletions(-)
drivers/fpga/dfl-afu-main.c +12 -2
@@ -435,6 +435,11 @@ static void port_hdr_uinit(struct platform_device *pdev,return ret; } +static const struct dfl_feature_id port_hdr_id_table[] = { + {.id =PORT_FEATURE_ID_HEADER ,}, + {0 ,} +}; +static const struct dfl_feature_ops port_hdr_ops = { .init = port_hdr_init, .uinit = port_hdr_uinit,
@@ -496,6 +501,11 @@ static void port_afu_uinit(struct platform_device *pdev,device_remove_groups (&pdev->dev, port_afu_groups); } +static const struct dfl_feature_id port_afu_id_table[] = { + {.id =PORT_FEATURE_ID_AFU ,}, + {0 ,} +}; +static const struct dfl_feature_ops port_afu_ops = { .init = port_afu_init, .uinit = port_afu_uinit,
@@ -503,11 +513,11 @@ static void port_afu_uinit(struct platform_device *pdev,static struct dfl_feature_driver port_feature_drvs[] = { { - .id =PORT_FEATURE_ID_HEADER , + .id_table = port_hdr_id_table, .ops = &port_hdr_ops, }, { - .id =PORT_FEATURE_ID_AFU , + .id_table = port_afu_id_table, .ops = &port_afu_ops, }, {
drivers/fpga/dfl-fme-main.c +8 -3
@@ -133,6 +133,11 @@ static long fme_hdr_ioctl(struct platform_device *pdev,return -ENODEV ; } +static const struct dfl_feature_id fme_hdr_id_table[] = { + {.id =FME_FEATURE_ID_HEADER ,}, + {0 ,} +}; +static const struct dfl_feature_ops fme_hdr_ops = { .init = fme_hdr_init, .uinit = fme_hdr_uinit,
@@ -141,12 +146,12 @@ static long fme_hdr_ioctl(struct platform_device *pdev,static struct dfl_feature_driver fme_feature_drvs[] = { { - .id =FME_FEATURE_ID_HEADER , + .id_table = fme_hdr_id_table, .ops = &fme_hdr_ops, }, { - .id =FME_FEATURE_ID_PR_MGMT , - .ops = &pr_mgmt_ops, + .id_table = fme_pr_mgmt_id_table, + .ops = &fme_pr_mgmt_ops, }, { .ops =NULL ,
drivers/fpga/dfl-fme-pr.c +6 -1
@@ -483,7 +483,12 @@ static long fme_pr_ioctl(struct platform_device *pdev,return ret; } -const struct dfl_feature_ops pr_mgmt_ops = { +const struct dfl_feature_id fme_pr_mgmt_id_table[] = { + {.id =FME_FEATURE_ID_PR_MGMT ,}, + {0 } +}; + +const struct dfl_feature_ops fme_pr_mgmt_ops = { .init = pr_mgmt_init, .uinit = pr_mgmt_uinit, .ioctl = fme_pr_ioctl,
drivers/fpga/dfl-fme.h +2 -1
@@ -35,6 +35,7 @@ struct dfl_fme { struct dfl_feature_platform_data *pdata; }; -extern const struct dfl_feature_ops pr_mgmt_ops; +extern const struct dfl_feature_ops fme_pr_mgmt_ops; +extern const struct dfl_feature_id fme_pr_mgmt_id_table[];#endif /* __DFL_FME_H */
drivers/fpga/dfl.c +16 -2
@@ -281,6 +281,21 @@ static int dfl_feature_instance_init(struct platform_device *pdev,return ret; } +static bool dfl_feature_drv_match (struct dfl_feature *feature, +struct dfl_feature_driver *driver) +{ +const struct dfl_feature_id *ids = driver->id_table; + +if (ids) { +while (ids->id) { +if (ids->id == feature->id) +return true; + ids++; + } + } +return false; +} +/** * dfl_fpga_dev_feature_init - init for sub features of dfl feature device * @pdev: feature device.
@@ -301,8 +316,7 @@ int dfl_fpga_dev_feature_init(struct platform_device *pdev,while (drv->ops) {dfl_fpga_dev_for_each_feature (pdata, feature) { -/* match feature and drv using id */ -if (feature->id == drv->id) { +if (dfl_feature_drv_match (feature, drv)) { ret =dfl_feature_instance_init (pdev, pdata, feature, drv);if (ret)
drivers/fpga/dfl.h +15 -6
@@ -30,8 +30,8 @@/* plus one for fme device */ #define MAX_DFL_FEATURE_DEV_NUM (MAX_DFL_FPGA_PORT_NUM + 1) -/* Reserved 0x0 for Header Group Register and 0xff for AFU */ -#define FEATURE_ID_FIU_HEADER 0x0 +/* Reserved 0xfe for Header Group Register and 0xff for AFU */ +#define FEATURE_ID_FIU_HEADER 0xfe#define FEATURE_ID_AFU 0xff#define FME_FEATURE_ID_HEADER FEATURE_ID_FIU_HEADER
@@ -169,13 +169,22 @@ struct dfl_fpga_port_ops {int dfl_fpga_check_port_id (struct platform_device *pdev,void *pport_id);/** -* struct dfl_feature_driver - sub feature's driver +* struct dfl_feature_id - dfl private feature id * -* @id: sub feature id. -* @ops: ops of this sub feature. +* @id: unique dfl private feature id. */ -struct dfl_feature_driver { +struct dfl_feature_id {u64 id; +}; + +/** +* struct dfl_feature_driver - dfl private feature driver +* +* @id_table: id_table for dfl private features supported by this driver. +* @ops: ops of this dfl private feature driver. +*/ +struct dfl_feature_driver { +const struct dfl_feature_id *id_table;const struct dfl_feature_ops *ops; };
Signature · 1 line
1.8.3.1
PATCH v3 7/12 fpga: dfl: afu: export __port_enable/disable function.
In reply to Wu HaoAs these two functions are used by other private features. e.g. in error reporting private feature, it requires to check port status and reset port for error clearing.
---
v2: rebased.
--- drivers/fpga/dfl-afu-main.c | 25 ++++++++++++++----------- drivers/fpga/dfl-afu.h | 3 +++ 2 files changed, 17 insertions(+), 11 deletions(-)
drivers/fpga/dfl-afu-main.c +14 -11
@@ -22,14 +22,16 @@#include "dfl-afu.h" /** -* port_enable - enable a port +* __port_enable - enable a port * @pdev: port platform device. * * Enable Port by clear the port soft reset bit, which is set by default. * The AFU is unable to respond to any MMIO access while in reset. -* port_enable function should only be used after port_disable function. +* __port_enable function should only be used after __port_disable function. +* +* The caller needs to hold lock for protection. */ -static void port_enable (struct platform_device *pdev) +void __port_enable (struct platform_device *pdev) {struct dfl_feature_platform_data *pdata =dev_get_platdata (&pdev->dev);void __iomem *base;
@@ -52,13 +54,14 @@ static void port_enable(struct platform_device *pdev)#define RST_POLL_TIMEOUT 1000/* us */ /** -* port_disable - disable a port +* __port_disable - disable a port * @pdev: port platform device. * -* Disable Port by setting the port soft reset bit, it puts the port into -* reset. +* Disable Port by setting the port soft reset bit, it puts the port into reset. +* +* The caller needs to hold lock for protection. */ -static int port_disable (struct platform_device *pdev) +int __port_disable (struct platform_device *pdev) {struct dfl_feature_platform_data *pdata =dev_get_platdata (&pdev->dev);void __iomem *base;
@@ -104,9 +107,9 @@ static int __port_reset(struct platform_device *pdev) {int ret; - ret =port_disable (pdev); + ret =__port_disable (pdev);if (!ret) -port_enable (pdev); +__port_enable (pdev);return ret; }
@@ -806,9 +809,9 @@ static int port_enable_set(struct platform_device *pdev, bool enable)mutex_lock (&pdata->lock);if (enable) -port_enable (pdev); +__port_enable (pdev);else - ret =port_disable (pdev); + ret =__port_disable (pdev);mutex_unlock (&pdata->lock);return ret;
drivers/fpga/dfl-afu.h +3 -0
@@ -79,6 +79,9 @@ struct dfl_afu {struct dfl_feature_platform_data *pdata; }; +void __port_enable (struct platform_device *pdev); +int __port_disable (struct platform_device *pdev); +void afu_mmio_region_init (struct dfl_feature_platform_data *pdata);int afu_mmio_region_add (struct dfl_feature_platform_data *pdata,u32 region_index,u64 region_size,u64 phys,u32 flags);
Signature · 1 line
1.8.3.1
PATCH v3 8/12 fpga: dfl: afu: add error reporting support.
In reply to Wu HaoError reporting is one important private feature, it reports error detected on port and accelerated function unit (AFU). It introduces several sysfs interfaces to allow userspace to check and clear errors detected by hardware.
---
v2: switch to device_add/remove_group for sysfs.
v3: update kernel version and date in sysfs doc
--- Documentation/ABI/testing/sysfs-platform-dfl-port | 39 ++++ drivers/fpga/Makefile | 1 + drivers/fpga/dfl-afu-error.c | 225 ++++++++++++++++++++++ drivers/fpga/dfl-afu-main.c | 4 + drivers/fpga/dfl-afu.h | 4 + 5 files changed, 273 insertions(+) create mode 100644 drivers/fpga/dfl-afu-error.c
Documentation/ABI/testing/sysfs-platform-dfl-port +39 -0
@@ -79,3 +79,42 @@ KernelVersion: 5.4 Contact: Wu Hao <[email protected]> Description: Read-only. Read this file to get the status of issued command to userclck_freqcntrcmd. + +What: /sys/bus/platform/devices/dfl-port.0/errors/revision +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-only. Read this file to get the revision of this error + reporting private feature. + +What: /sys/bus/platform/devices/dfl-port.0/errors/errors +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-only. Read this file to get errors detected on port and + Accelerated Function Unit (AFU). + +What: /sys/bus/platform/devices/dfl-port.0/errors/first_error +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-only. Read this file to get the first error detected by + hardware. + +What: /sys/bus/platform/devices/dfl-port.0/errors/first_malformed_req +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-only. Read this file to get the first malformed request + captured by hardware. + +What: /sys/bus/platform/devices/dfl-port.0/errors/clear +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Write-only. Write error code to this file to clear errors. + Write fails with -EINVAL if input parsing fails or input error + code doesn't match. + Write fails with -EBUSY or -ETIMEDOUT if error can't be cleared + as hardware is in low power state (-EBUSY) or not responding + (-ETIMEDOUT).
drivers/fpga/Makefile +1 -0
@@ -41,6 +41,7 @@ obj-$(CONFIG_FPGA_DFL_AFU) += dfl-afu.o dfl-fme-objs := dfl-fme-main.o dfl-fme-pr.o dfl-afu-objs := dfl-afu-main.o dfl-afu-region.o dfl-afu-dma-region.o +dfl-afu-objs += dfl-afu-error.o # Drivers for FPGAs which implement DFL obj-$(CONFIG_FPGA_DFL_PCI) += dfl-pci.o
drivers/fpga/dfl-afu-error.c new file +225 -0
225 lines not on this page · raw message
drivers/fpga/dfl-afu-main.c +4 -0
@@ -524,6 +524,10 @@ static void port_afu_uinit(struct platform_device *pdev, .ops = &port_afu_ops, }, { + .id_table = port_err_id_table, + .ops = &port_err_ops, + }, + { .ops = NULL, } };
drivers/fpga/dfl-afu.h +4 -0
@@ -100,4 +100,8 @@ int afu_dma_map_region(struct dfl_feature_platform_data *pdata,struct dfl_afu_dma_region *afu_dma_region_find (struct dfl_feature_platform_data *pdata,u64 iova,u64 size); + +extern const struct dfl_feature_ops port_err_ops; +extern const struct dfl_feature_id port_err_id_table[]; +#endif /* __DFL_AFU_H */
Signature · 1 line
1.8.3.1
PATCH v3 9/12 fpga: dfl: afu: add STP (SignalTap) support
In reply to Wu HaoSTP (SignalTap) is one of the private features under the port for debugging. This patch adds private feature driver support for it to allow userspace applications to mmap related mmio region and provide STP service.
--- drivers/fpga/dfl-afu-main.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+)
drivers/fpga/dfl-afu-main.c +34 -0
@@ -514,6 +514,36 @@ static void port_afu_uinit(struct platform_device *pdev, .uinit = port_afu_uinit, }; +static int port_stp_init (struct platform_device *pdev, +struct dfl_feature *feature) +{ +struct resource *res = &pdev->resource[feature->resource_index]; + +dev_dbg (&pdev->dev,"PORT STP Init.\n" ); + +return afu_mmio_region_add (dev_get_platdata (&pdev->dev), +DFL_PORT_REGION_INDEX_STP , +resource_size (res), res->start, +DFL_PORT_REGION_MMAP |DFL_PORT_REGION_READ | +DFL_PORT_REGION_WRITE ); +} + +static void port_stp_uinit (struct platform_device *pdev, +struct dfl_feature *feature) +{ +dev_dbg (&pdev->dev,"PORT STP UInit.\n" ); +} + +static const struct dfl_feature_id port_stp_id_table[] = { + {.id =PORT_FEATURE_ID_STP ,}, + {0 ,} +}; + +static const struct dfl_feature_ops port_stp_ops = { + .init = port_stp_init, + .uinit = port_stp_uinit, +}; +static struct dfl_feature_driver port_feature_drvs[] = { { .id_table = port_hdr_id_table,
@@ -528,6 +558,10 @@ static void port_afu_uinit(struct platform_device *pdev, .ops = &port_err_ops, }, { + .id_table = port_stp_id_table, + .ops = &port_stp_ops, + }, + { .ops = NULL, } };
Signature · 1 line
1.8.3.1
Re: [PATCH v3 09/12] fpga: dfl: afu: add STP (SignalTap) support
Quoted 19 lines · Wu Hao
STP (SignalTap) is one of the private features under the port for debugging. This patch adds private feature driver support for it to allow userspace applications to mmap related mmio region and provide STP service.
Signed-off-by Xu Yilun <[email protected]>Signed-off-by Wu Hao <[email protected]>Acked-by Moritz Fischer <[email protected]>Acked-by Alan Tull <[email protected]>Signed-off-by Moritz Fischer <[email protected]>--- drivers/fpga/dfl-afu-main.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+)drivers/fpga/dfl-afu-main.c +0
-0
drivers/fpga/dfl-afu-main.c static void port_afu_uinit(struct platform_device *pdev, · line 522
Quoted 9 lines
drivers/fpga/dfl-afu-main.c +6
-0@@ -514,6 +514,36 @@ static void port_afu_uinit(struct platform_device *pdev, .uinit = port_afu_uinit, }; +static int port_stp_init (struct platform_device *pdev, +struct dfl_feature *feature) +{ +struct resource *res = &pdev->resource[feature->resource_index]; + +dev_dbg (&pdev->dev,"PORT STP Init.\n" );
ftrace is your friend, no need to do a lot of "look I am here!" messages.
drivers/fpga/dfl-afu-main.c static void port_afu_uinit(struct platform_device *pdev, · line 534
Quoted 12 lines
drivers/fpga/dfl-afu-main.c +12
-0+ +return afu_mmio_region_add (dev_get_platdata (&pdev->dev), +DFL_PORT_REGION_INDEX_STP , +resource_size (res), res->start, +DFL_PORT_REGION_MMAP |DFL_PORT_REGION_READ | +DFL_PORT_REGION_WRITE ); +} + +static void port_stp_uinit (struct platform_device *pdev, +struct dfl_feature *feature) +{ +dev_dbg (&pdev->dev,"PORT STP UInit.\n" );
Same here.
Why have this function at all if it does not do anything?
thanks,
greg k-h
Re: [PATCH v3 09/12] fpga: dfl: afu: add STP (SignalTap) support
Quoted 23 lines · Greg KH
Quoted 22 lines · Wu Hao
STP (SignalTap) is one of the private features under the port for debugging. This patch adds private feature driver support for it to allow userspace applications to mmap related mmio region and provide STP service.
Signed-off-by Xu Yilun <[email protected]>Signed-off-by Wu Hao <[email protected]>Acked-by Moritz Fischer <[email protected]>Acked-by Alan Tull <[email protected]>Signed-off-by Moritz Fischer <[email protected]>--- drivers/fpga/dfl-afu-main.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+)drivers/fpga/dfl-afu-main.c +0
-0@@ -514,6 +514,36 @@ static void port_afu_uinit(struct platform_device *pdev, .uinit = port_afu_uinit, };
Quoted 9 lines
drivers/fpga/dfl-afu-main.c static void port_afu_uinit(struct platform_device *pdev, · line 522
Quoted 6 lines
drivers/fpga/dfl-afu-main.c +6
-0+static int port_stp_init (struct platform_device *pdev, +struct dfl_feature *feature) +{ +struct resource *res = &pdev->resource[feature->resource_index]; + +dev_dbg (&pdev->dev,"PORT STP Init.\n" );ftrace is your friend, no need to do a lot of "look I am here!" messages.
Hi Greg,
Thanks for the code review!
Sure, let me remove them.
Quoted 16 lines
drivers/fpga/dfl-afu-main.c static void port_afu_uinit(struct platform_device *pdev, · line 534
Quoted 13 lines
drivers/fpga/dfl-afu-main.c +12
-0+ +return afu_mmio_region_add (dev_get_platdata (&pdev->dev), +DFL_PORT_REGION_INDEX_STP , +resource_size (res), res->start, +DFL_PORT_REGION_MMAP |DFL_PORT_REGION_READ | +DFL_PORT_REGION_WRITE ); +} + +static void port_stp_uinit (struct platform_device *pdev, +struct dfl_feature *feature) +{ +dev_dbg (&pdev->dev,"PORT STP UInit.\n" );Same here.
Quoted 1 line
Why have this function at all if it does not do anything?
Let me remove them in the next version. actually uinit callback is always required in current code, i will add one more patch to change it, and remove all uinit functions who do nothing, it does save code.
Thanks for the comments.
Hao
Quoted 5 lines
thanks,
greg k-h
PATCH v3 10/12 fpga: dfl: fme: add capability sysfs interfaces
In reply to Wu HaoThis patch adds 3 read-only sysfs interfaces for FPGA Management Engine (FME) block for capabilities including cache_size, fabric_version and socket_id.
---
v2: rebased.
v3: update kernel version and date in sysfs doc
--- Documentation/ABI/testing/sysfs-platform-dfl-fme | 23 ++++++++++++ drivers/fpga/dfl-fme-main.c | 48 ++++++++++++++++++++++++ 2 files changed, 71 insertions(+)
Documentation/ABI/testing/sysfs-platform-dfl-fme +23 -0
@@ -21,3 +21,26 @@ Contact: Wu Hao <[email protected]> Description: Read-only. It returns Bitstream (static FPGA region) meta data, which includes the synthesis date, seed and other information of this static FPGA region. + +What: /sys/bus/platform/devices/dfl-fme.0/cache_size +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-only. It returns cache size of this FPGA device. + +What: /sys/bus/platform/devices/dfl-fme.0/fabric_version +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-only. It returns fabric version of this FPGA device. + Userspace applications need this information to select + best data channels per different fabric design. + +What: /sys/bus/platform/devices/dfl-fme.0/socket_id +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-only. It returns socket_id to indicate which socket + this FPGA belongs to, only valid for integrated solution. + User only needs this information, in case standard numa node + can't provide correct information.
drivers/fpga/dfl-fme-main.c +48 -0
58 lines not on this page · raw message
Signature · 1 line
1.8.3.1
PATCH v3 11/12 fpga: dfl: fme: add global error reporting support
In reply to Wu HaoThis patch adds support for global error reporting for FPGA Management Engine (FME), it introduces sysfs interfaces to report different error detected by the hardware, and allow user to clear errors or inject error for testing purpose.
---
v2: switch to device_add/remove_groups for sysfs.
v3: update kernel version and date in sysfs doc
--- Documentation/ABI/testing/sysfs-platform-dfl-fme | 75 +++++ drivers/fpga/Makefile | 2 +- drivers/fpga/dfl-fme-error.c | 385 +++++++++++++++++++++++ drivers/fpga/dfl-fme-main.c | 4 + drivers/fpga/dfl-fme.h | 2 + drivers/fpga/dfl.h | 2 + 6 files changed, 469 insertions(+), 1 deletion(-) create mode 100644 drivers/fpga/dfl-fme-error.c
Documentation/ABI/testing/sysfs-platform-dfl-fme +75 -0
@@ -44,3 +44,78 @@ Description: Read-only. It returns socket_id to indicate which socket this FPGA belongs to, only valid for integrated solution. User only needs this information, in case standard numa node can't provide correct information. + +What: /sys/bus/platform/devices/dfl-fme.0/errors/revision +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-only. Read this file to get the revision of this global + error reporting private feature. + +What: /sys/bus/platform/devices/dfl-fme.0/errors/pcie0_errors +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-Write. Read this file for errors detected on pcie0 link. + Write this file to clear errors logged in pcie0_errors. Write + fails with -EINVAL if input parsing fails or input error code + doesn't match. + +What: /sys/bus/platform/devices/dfl-fme.0/errors/pcie1_errors +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-Write. Read this file for errors detected on pcie1 link. + Write this file to clear errors logged in pcie1_errors. Write + fails with -EINVAL if input parsing fails or input error code + doesn't match. + +What: /sys/bus/platform/devices/dfl-fme.0/errors/nonfatal_errors +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-only. It returns non-fatal errors detected. + +What: /sys/bus/platform/devices/dfl-fme.0/errors/catfatal_errors +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-only. It returns catastrophic and fatal errors detected. + +What: /sys/bus/platform/devices/dfl-fme.0/errors/inject_error +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-Write. Read this file to check errors injected. Write this + file to inject errors for testing purpose. Write fails with + -EINVAL if input parsing fails or input inject error code isn't + supported. + +What: /sys/bus/platform/devices/dfl-fme.0/errors/fme-errors/errors +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-only. Read this file to get errors detected by hardware. + +What: /sys/bus/platform/devices/dfl-fme.0/errors/fme-errors/first_error +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-only. Read this file to get the first error detected by + hardware. + +What: /sys/bus/platform/devices/dfl-fme.0/errors/fme-errors/next_error +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Read-only. Read this file to get the second error detected by + hardware. + +What: /sys/bus/platform/devices/dfl-fme.0/errors/fme-errors/clear +Date: July 2019 +KernelVersion: 5.4 +Contact: Wu Hao <[email protected]> +Description: Write-only. Write error code to this file to clear all errors + logged in errors, first_error and next_error. Write fails with + -EINVAL if input parsing fails or input error code doesn't + match.
drivers/fpga/Makefile +1 -1
@@ -39,7 +39,7 @@ obj-$(CONFIG_FPGA_DFL_FME_BRIDGE) += dfl-fme-br.o obj-$(CONFIG_FPGA_DFL_FME_REGION) += dfl-fme-region.o obj-$(CONFIG_FPGA_DFL_AFU) += dfl-afu.o -dfl-fme-objs := dfl-fme-main.o dfl-fme-pr.o +dfl-fme-objs := dfl-fme-main.o dfl-fme-pr.o dfl-fme-error.o dfl-afu-objs := dfl-afu-main.o dfl-afu-region.o dfl-afu-dma-region.o dfl-afu-objs += dfl-afu-error.o
drivers/fpga/dfl-fme-error.c new file +385 -0
385 lines not on this page · raw message
drivers/fpga/dfl-fme-main.c +4 -0
@@ -202,6 +202,10 @@ static long fme_hdr_ioctl(struct platform_device *pdev, .ops = &fme_pr_mgmt_ops, }, { + .id_table = fme_global_err_id_table, + .ops = &fme_global_err_ops, + }, + { .ops = NULL, }, };
drivers/fpga/dfl-fme.h +2 -0
@@ -37,5 +37,7 @@ struct dfl_fme {extern const struct dfl_feature_ops fme_pr_mgmt_ops;extern const struct dfl_feature_id fme_pr_mgmt_id_table[]; +extern const struct dfl_feature_ops fme_global_err_ops; +extern const struct dfl_feature_id fme_global_err_id_table[];#endif /* __DFL_FME_H */
drivers/fpga/dfl.h +2 -0
@@ -197,12 +197,14 @@ struct dfl_feature_driver {* feature dev (platform device)'s reources. * @ioaddr: mapped mmio resource address. * @ops: ops of this sub feature. +* @priv: priv data of this feature. */ struct dfl_feature {u64 id;int resource_index;void __iomem *ioaddr;const struct dfl_feature_ops *ops; +void *priv; };#define DEV_STATUS_IN_USE 0
Signature · 1 line
1.8.3.1
PATCH v3 12/12 Documentation: fpga: dfl: add descriptions for virtualization and new interfaces.
In reply to Wu HaoThis patch adds virtualization support description for DFL based FPGA devices (based on PCIe SRIOV), and introductions to new interfaces added by new dfl private feature drivers.
--- Documentation/fpga/dfl.rst | 105 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+)
Documentation/fpga/dfl.rst +105 -0
@@ -87,6 +87,8 @@ The following functions are exposed through ioctls: - Get driver API version (DFL_FPGA_GET_API_VERSION) - Check for extensions (DFL_FPGA_CHECK_EXTENSION) - Program bitstream (DFL_FPGA_FME_PORT_PR) +- Assign port to PF (DFL_FPGA_FME_PORT_ASSIGN) +- Release port from PF (DFL_FPGA_FME_PORT_RELEASE) More functions are exposed through sysfs (/sys/class/fpga_region/regionX/dfl-fme.n/):
@@ -102,6 +104,10 @@ More functions are exposed through sysfs one FPGA device may have more than one port, this sysfs interface indicates how many ports the FPGA device has. + Global error reporting management (errors/) + error reporting sysfs interfaces allow user to read errors detected by the + hardware, and clear the logged errors. + FIU - PORT ==========
@@ -143,6 +149,10 @@ More functions are exposed through sysfs: Read Accelerator GUID (afu_id) afu_id indicates which PR bitstream is programmed to this AFU. + Error reporting (errors/) + error reporting sysfs interfaces allow user to read port/afu errors + detected by the hardware, and clear the logged errors. + DFL Framework Overview ======================
@@ -218,6 +228,101 @@ the compat_id exposed by the target FPGA region. This check is usually done by userspace before calling the reconfiguration IOCTL. +FPGA virtualization - PCIe SRIOV +================================ +This section describes the virtualization support on DFL based FPGA device to +enable accessing an accelerator from applications running in a virtual machine +(VM). This section only describes the PCIe based FPGA device with SRIOV support. + +Features supported by the particular FPGA device are exposed through Device +Feature Lists, as illustrated below: + +:: + + +-------------------------------+ +-------------+ + | PF | | VF | + +-------------------------------+ +-------------+ + ^ ^ ^ ^ + | | | | + +-----|------------|---------|--------------|-------+ + | | | | | | + | +-----+ +-------+ +-------+ +-------+ | + | | FME | | Port0 | | Port1 | | Port2 | | + | +-----+ +-------+ +-------+ +-------+ | + | ^ ^ ^ | + | | | | | + | +-------+ +------+ +-------+ | + | | AFU | | AFU | | AFU | | + | +-------+ +------+ +-------+ | + | | + | DFL based FPGA PCIe Device | + +---------------------------------------------------+ + +FME is always accessed through the physical function (PF). + +Ports (and related AFUs) are accessed via PF by default, but could be exposed +through virtual function (VF) devices via PCIe SRIOV. Each VF only contains +1 Port and 1 AFU for isolation. Users could assign individual VFs (accelerators) +created via PCIe SRIOV interface, to virtual machines. + +The driver organization in virtualization case is illustrated below: +:: + + +-------++------++------+ | + | FME || FME || FME | | + | FPGA || FPGA || FPGA | | + |Manager||Bridge||Region| | + +-------++------++------+ | + +-----------------------+ +--------+ | +--------+ + | FME | | AFU | | | AFU | + | Module | | Module | | | Module | + +-----------------------+ +--------+ | +--------+ + +-----------------------+ | +-----------------------+ + | FPGA Container Device | | | FPGA Container Device | + | (FPGA Base Region) | | | (FPGA Base Region) | + +-----------------------+ | +-----------------------+ + +------------------+ | +------------------+ + | FPGA PCIE Module | | Virtual | FPGA PCIE Module | + +------------------+ Host | Machine +------------------+ + -------------------------------------- | ------------------------------ + +---------------+ | +---------------+ + | PCI PF Device | | | PCI VF Device | + +---------------+ | +---------------+ + +FPGA PCIe device driver is always loaded first once a FPGA PCIe PF or VF device +is detected. It: + +* Finishes enumeration on both FPGA PCIe PF and VF device using common + interfaces from DFL framework. +* Supports SRIOV. + +The FME device driver plays a management role in this driver architecture, it +provides ioctls to release Port from PF and assign Port to PF. After release +a port from PF, then it's safe to expose this port through a VF via PCIe SRIOV +sysfs interface. + +To enable accessing an accelerator from applications running in a VM, the +respective AFU's port needs to be assigned to a VF using the following steps: + +#. The PF owns all AFU ports by default. Any port that needs to be + reassigned to a VF must first be released through the + DFL_FPGA_FME_PORT_RELEASE ioctl on the FME device. + +#. Once N ports are released from PF, then user can use command below + to enable SRIOV and VFs. Each VF owns only one Port with AFU. + + :: + + echo N > $PCI_DEVICE_PATH/sriov_numvfs + +#. Pass through the VFs to VMs + +#. The AFU under VF is accessible from applications in VM (using the + same driver inside the VF). + +Note that an FME can't be assigned to a VF, thus PR and other management +functions are only available via the PF. + Device enumeration ================== This section introduces how applications enumerate the fpga device from
Signature · 1 line
1.8.3.1