raw/ifpga: add extra OPAE APIs
v11 cover raw/ifpga: add extra OPAE APIs
Cyborg is part of OpenStack, it needs some OPAE APIs to manage devices with Intel FPGA. The first three patches implement extra APIs to meet Cyborg requirement. The last patch add an example to show how to use these APIs.
Main changes from v10:
- remove call of rte_eal_init() and rte_eal_cleanup()
- rename opae_init_eal() to opae_init()
- rename opae_cleanup_eal() to opae_cleanup()
Wei Huang (4):
raw/ifpga: add fpga rsu function
raw/ifpga: add fpga property get function
raw/ifpga: add OPAE API for OpenStack Cyborg
examples/ifpga: add example for opae ifpga API
MAINTAINERS | 1 + doc/guides/sample_app_ug/ifpga.rst | 435 +++++ doc/guides/sample_app_ug/index.rst | 1 + drivers/raw/ifpga/base/ifpga_api.c | 34 + drivers/raw/ifpga/base/ifpga_defines.h | 1 + drivers/raw/ifpga/base/ifpga_feature_dev.c | 21 + drivers/raw/ifpga/base/ifpga_feature_dev.h | 1 + drivers/raw/ifpga/base/ifpga_fme.c | 36 +- drivers/raw/ifpga/base/ifpga_fme_rsu.c | 435 +++++ drivers/raw/ifpga/base/ifpga_hw.h | 1 + drivers/raw/ifpga/base/ifpga_sec_mgr.c | 639 +++++++ drivers/raw/ifpga/base/ifpga_sec_mgr.h | 89 + drivers/raw/ifpga/base/meson.build | 2 + drivers/raw/ifpga/base/opae_hw_api.c | 77 + drivers/raw/ifpga/base/opae_hw_api.h | 13 + drivers/raw/ifpga/base/opae_ifpga_hw_api.h | 1 + drivers/raw/ifpga/base/opae_intel_max10.c | 48 + drivers/raw/ifpga/base/opae_intel_max10.h | 44 + drivers/raw/ifpga/ifpga_opae_api.c | 1793 ++++++++++++++++++++ drivers/raw/ifpga/ifpga_opae_api.h | 248 +++ drivers/raw/ifpga/ifpga_rawdev.c | 352 ++++ drivers/raw/ifpga/ifpga_rawdev.h | 50 +- drivers/raw/ifpga/meson.build | 4 +- examples/ifpga/Makefile | 45 + examples/ifpga/commands.c | 1321 ++++++++++++++ examples/ifpga/commands.h | 16 + examples/ifpga/main.c | 41 + examples/ifpga/meson.build | 19 + 28 files changed, 5764 insertions(+), 4 deletions(-) create mode 100644 doc/guides/sample_app_ug/ifpga.rst create mode 100644 drivers/raw/ifpga/base/ifpga_fme_rsu.c create mode 100644 drivers/raw/ifpga/base/ifpga_sec_mgr.c create mode 100644 drivers/raw/ifpga/base/ifpga_sec_mgr.h create mode 100644 drivers/raw/ifpga/ifpga_opae_api.c create mode 100644 drivers/raw/ifpga/ifpga_opae_api.h create mode 100644 examples/ifpga/Makefile create mode 100644 examples/ifpga/commands.c create mode 100644 examples/ifpga/commands.h create mode 100644 examples/ifpga/main.c create mode 100644 examples/ifpga/meson.build
Signature · 1 line
2.29.2
PATCH v11 1/4 raw/ifpga: add fpga rsu function
RSU (Remote System Update) depends on secure manager which may be different on various implementations, so a new secure manager device is implemented for adapting such difference.
There are three major functions added:
1. ifpga_rawdev_update_flash() updates flash with specific image file.
2. ifpga_rawdev_stop_flash_update() aborts flash update process.
3. ifpga_rawdev_reload() reloads FPGA from updated flash.
---
v2: fix coding style issue in ifpga_fme_rsu.c and ifpga_sec_mgr.c
---
v3: fix compilation issues in ifpga_fme_rsu.c
---
v4: fix compilation issues in opae_intel_max10.c
--- drivers/raw/ifpga/base/ifpga_api.c | 26 + drivers/raw/ifpga/base/ifpga_fme.c | 8 + drivers/raw/ifpga/base/ifpga_fme_rsu.c | 435 +++++++++++++++ drivers/raw/ifpga/base/ifpga_hw.h | 1 + drivers/raw/ifpga/base/ifpga_sec_mgr.c | 639 ++++++++++++++++++++++ drivers/raw/ifpga/base/ifpga_sec_mgr.h | 89 +++ drivers/raw/ifpga/base/meson.build | 2 + drivers/raw/ifpga/base/opae_hw_api.c | 59 ++ drivers/raw/ifpga/base/opae_hw_api.h | 11 + drivers/raw/ifpga/base/opae_intel_max10.c | 48 ++ drivers/raw/ifpga/base/opae_intel_max10.h | 44 ++ drivers/raw/ifpga/ifpga_rawdev.c | 55 ++ drivers/raw/ifpga/ifpga_rawdev.h | 7 +- 13 files changed, 1423 insertions(+), 1 deletion(-) create mode 100644 drivers/raw/ifpga/base/ifpga_fme_rsu.c create mode 100644 drivers/raw/ifpga/base/ifpga_sec_mgr.c create mode 100644 drivers/raw/ifpga/base/ifpga_sec_mgr.h
drivers/raw/ifpga/base/ifpga_api.c +26 -0
@@ -5,6 +5,7 @@#include "ifpga_api.h" #include "ifpga_enumerate.h" #include "ifpga_feature_dev.h" +#include "ifpga_sec_mgr.h" #include "opae_hw_api.h"
@@ -228,11 +229,36 @@ static int ifpga_mgr_get_board_info(struct opae_manager *mgr,return 0 ; } +static int ifpga_mgr_update_flash (struct opae_manager *mgr,const char *image, +u64 *status) +{ +struct ifpga_fme_hw *fme = mgr->data; + +return fpga_update_flash (fme, image, status); +} + +static int ifpga_mgr_stop_flash_update (struct opae_manager *mgr,int force) +{ +struct ifpga_fme_hw *fme = mgr->data; + +return fpga_stop_flash_update (fme, force); +} + +static int ifpga_mgr_reload (struct opae_manager *mgr,int type,int page) +{ +struct ifpga_fme_hw *fme = mgr->data; + +return fpga_reload (fme, type, page); +} +struct opae_manager_ops ifpga_mgr_ops = { .flash = ifpga_mgr_flash, .get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info, .get_sensor_value = ifpga_mgr_get_sensor_value, .get_board_info = ifpga_mgr_get_board_info, + .update_flash = ifpga_mgr_update_flash, + .stop_flash_update = ifpga_mgr_stop_flash_update, + .reload = ifpga_mgr_reload, }; static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
drivers/raw/ifpga/base/ifpga_fme.c +8 -0
@@ -7,6 +7,7 @@#include "opae_intel_max10.h" #include "opae_i2c.h" #include "opae_at24_eeprom.h" +#include "ifpga_sec_mgr.h" #define PWR_THRESHOLD_MAX 0x7F
@@ -1152,6 +1153,12 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)if (spi_self_checking (max10)) goto spi_fail; + ret =init_sec_mgr (fme); +if (ret) { +dev_err (fme,"security manager init fail\n" ); + goto spi_fail; + } +return ret; spi_fail:
@@ -1165,6 +1172,7 @@ static void fme_nios_spi_uinit(struct ifpga_feature *feature) {struct ifpga_fme_hw *fme = (struct ifpga_fme_hw *)feature->parent; +release_sec_mgr (fme);if (fme->max10_dev)intel_max10_device_remove (fme->max10_dev); }
drivers/raw/ifpga/base/ifpga_fme_rsu.c new file +435 -0
435 lines not on this page · raw message
drivers/raw/ifpga/base/ifpga_hw.h +1 -0
@@ -91,6 +91,7 @@ struct ifpga_fme_hw {struct opae_board_info board_info;int nums_eth_dev;unsigned int nums_acc_region; +void *sec_mgr; };enum ifpga_port_state {
drivers/raw/ifpga/base/ifpga_sec_mgr.c new file +639 -0
639 lines not on this page · raw message
drivers/raw/ifpga/base/ifpga_sec_mgr.h new file +89 -0
89 lines not on this page · raw message
drivers/raw/ifpga/base/meson.build +2 -0
@@ -12,6 +12,8 @@ sources = [ 'ifpga_port.c', 'ifpga_port_error.c', 'ifpga_fme_pr.c', + 'ifpga_fme_rsu.c', + 'ifpga_sec_mgr.c', 'opae_hw_api.c', 'opae_ifpga_hw_api.c', 'opae_debug.c',
drivers/raw/ifpga/base/opae_hw_api.c +59 -0
68 lines not on this page · raw message
drivers/raw/ifpga/base/opae_hw_api.h +11 -0
@@ -55,6 +55,10 @@ struct opae_manager_ops { unsigned int *value);int (*get_board_info)(struct opae_manager *mgr,struct opae_board_info **info); +int (*update_flash)(struct opae_manager *mgr,const char *image, +u64 *status); +int (*stop_flash_update)(struct opae_manager *mgr,int force); +int (*reload)(struct opae_manager *mgr,int type,int page); };/* networking management ops in FME */
@@ -276,6 +280,8 @@ typedef struct {pthread_mutex_t i2c_mutex;u32 ref_cnt;/* reference count of shared memory */ u32 dtb_size;/* actual length of DTB data in byte */ +u32 rsu_ctrl;/* used to control rsu */ +u32 rsu_stat;/* used to report status for rsu */ }; };u8 dtb[SHM_BLK_SIZE ];/* DTB data */
@@ -354,4 +360,9 @@ int opae_manager_eth_group_read_reg(struct opae_manager *mgr, u8 group_id, u8 type, u8 index, u16 addr, u32 *data);int opae_mgr_get_board_info (struct opae_manager *mgr,struct opae_board_info **info); +int opae_mgr_update_flash (struct opae_manager *mgr,const char *image, +uint64_t *status); +int opae_mgr_stop_flash_update (struct opae_manager *mgr,int force); +int opae_mgr_reload (struct opae_manager *mgr,int type,int page); +#endif /* _OPAE_HW_API_H_*/
drivers/raw/ifpga/base/opae_intel_max10.c +48 -0
66 lines not on this page · raw message
drivers/raw/ifpga/base/opae_intel_max10.h +44 -0
76 lines not on this page · raw message
drivers/raw/ifpga/ifpga_rawdev.c +55 -0
58 lines not on this page · raw message
drivers/raw/ifpga/ifpga_rawdev.h +6 -1
@@ -43,7 +43,7 @@ enum ifpga_rawdev_device_state {static inline struct opae_adapter *ifpga_rawdev_get_priv (const struct rte_rawdev *rawdev) { -return rawdev->dev_private; +return (struct opae_adapter *)rawdev->dev_private; }#define IFPGA_RAWDEV_MSIX_IRQ_NUM 7
@@ -76,4 +76,9 @@ int ifpga_unregister_msix_irq(enum ifpga_irq_type type, int vec_start, rte_intr_callback_fn handler, void *arg); +int ifpga_rawdev_update_flash (struct rte_rawdev *dev,const char *image, +uint64_t *status); +int ifpga_rawdev_stop_flash_update (struct rte_rawdev *dev,int force); +int ifpga_rawdev_reload (struct rte_rawdev *dev,int type,int page); +#endif /* _IFPGA_RAWDEV_H_ */
Signature · 1 line
2.29.2
Re: [dpdk-dev] [dpdk-stable] [PATCH v11 1/4] raw/ifpga: add fpga rsu function
Quoted 11 lines · Wei Huang
RSU (Remote System Update) depends on secure manager which may be different on various implementations, so a new secure manager device is implemented for adapting such difference.
There are three major functions added:
1.ifpga_rawdev_update_flash()updates flash with specific image file.
2.ifpga_rawdev_stop_flash_update()aborts flash update process.
3.ifpga_rawdev_reload()reloads FPGA from updated flash.Signed-off-by Wei Huang <[email protected]>Acked-by Tianfei Zhang <[email protected]>Acked-by Rosen Xu <[email protected]>
<...>
drivers/raw/ifpga/ifpga_rawdev.h int · line 84
Quoted 11 lines
drivers/raw/ifpga/ifpga_rawdev.h +5
-0@@ -76,4 +76,9 @@ int ifpga_unregister_msix_irq(enum ifpga_irq_type type, int vec_start, rte_intr_callback_fn handler, void *arg); +int ifpga_rawdev_update_flash (struct rte_rawdev *dev,const char *image, +uint64_t *status); +int ifpga_rawdev_stop_flash_update (struct rte_rawdev *dev,int force); +int ifpga_rawdev_reload (struct rte_rawdev *dev,int type,int page); +#endif /* _IFPGA_RAWDEV_H_ */
Hi Wei,
Please help me understand the rawdev, who should be calling the above newly added functions?
Re: [dpdk-dev] [dpdk-stable] [PATCH v11 1/4] raw/ifpga: add fpga rsu function
Hi Ferruh,
Cyborg is an OpenStack project that aims to provide a general purpose management framework for acceleration resources (i.e. various types of accelerators such as GPU, FPGA, NP, ODP, DPDK/SPDK and so on).
To update the FPGA flash is one of requirements from Cyborg. Originally there are no such interfaces, so I added them.
These interfaces use rte_rawdev to identify which FPGA to access, they will be called in opae_update_flash(), opae_cancel_flash_update() and opae_reboot_device() in ifpga_opae_api.c .
These opae_xxx function use PCI address to identify FPGA instead of rte_rawdev, so the caller has no need to know the existence of rte_rawdev. In fact, Cyborg is Python application, these opae_xxx functions will be eventually wrapped in a Python module for Cyborg to call.
Thanks,
Wei
-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Friday, January 22, 2021 00:30
To: Huang, Wei <[email protected]>; [email protected]; Xu, Rosen <[email protected]>; Zhang, Qi Z <[email protected]>
Cc: [email protected]; Zhang, Tianfei <[email protected]>; Ray Kinsella <[email protected]>
Subject: Re: [dpdk-stable] [PATCH v11 1/4] raw/ifpga: add fpga rsu function
Quoted 11 lines · Wei Huang
RSU (Remote System Update) depends on secure manager which may be different on various implementations, so a new secure manager device is implemented for adapting such difference.
There are three major functions added:
1.ifpga_rawdev_update_flash()updates flash with specific image file.
2.ifpga_rawdev_stop_flash_update()aborts flash update process.
3.ifpga_rawdev_reload()reloads FPGA from updated flash.Signed-off-by Wei Huang <[email protected]>Acked-by Tianfei Zhang <[email protected]>Acked-by Rosen Xu <[email protected]>
<...>
drivers/raw/ifpga/ifpga_rawdev.h int · line 84
Quoted 12 lines
drivers/raw/ifpga/ifpga_rawdev.h +6
-0@@ -76,4 +76,9 @@ int ifpga_unregister_msix_irq(enum ifpga_irq_type type, int vec_start, rte_intr_callback_fn handler, void *arg); +int ifpga_rawdev_update_flash (struct rte_rawdev *dev,const char *image, +uint64_t *status); +int ifpga_rawdev_stop_flash_update (struct rte_rawdev *dev,int +force);int ifpga_rawdev_reload (struct rte_rawdev *dev,int type,int +page); +#endif /* _IFPGA_RAWDEV_H_ */
Hi Wei,
Please help me understand the rawdev, who should be calling the above newly added functions?
Re: [dpdk-dev] [dpdk-stable] [PATCH v11 1/4] raw/ifpga: add fpga rsu function
On 1/22/2021 2:18 AM, Huang, Wei wrote:
<Please do not top post, your comment moved below>
Quoted 45 lines
Quoted 38 lines
-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Friday, January 22, 2021 00:30
To: Huang, Wei <[email protected]>; [email protected]; Xu, Rosen <[email protected]>; Zhang, Qi Z <[email protected]>
Cc: [email protected]; Zhang, Tianfei <[email protected]>; Ray Kinsella <[email protected]>
Subject: Re: [dpdk-stable] [PATCH v11 1/4] raw/ifpga: add fpga rsu functionQuoted 11 lines · Wei Huang
RSU (Remote System Update) depends on secure manager which may be different on various implementations, so a new secure manager device is implemented for adapting such difference.
There are three major functions added:
1.ifpga_rawdev_update_flash()updates flash with specific image file.
2.ifpga_rawdev_stop_flash_update()aborts flash update process.
3.ifpga_rawdev_reload()reloads FPGA from updated flash.Signed-off-by Wei Huang <[email protected]>Acked-by Tianfei Zhang <[email protected]>Acked-by Rosen Xu <[email protected]><...>
drivers/raw/ifpga/ifpga_rawdev.h int · line 84
Quoted 12 lines
drivers/raw/ifpga/ifpga_rawdev.h +6
-0@@ -76,4 +76,9 @@ int ifpga_unregister_msix_irq(enum ifpga_irq_type type, int vec_start, rte_intr_callback_fn handler, void *arg); +int ifpga_rawdev_update_flash (struct rte_rawdev *dev,const char *image, +uint64_t *status); +int ifpga_rawdev_stop_flash_update (struct rte_rawdev *dev,int +force);int ifpga_rawdev_reload (struct rte_rawdev *dev,int type,int +page); +#endif /* _IFPGA_RAWDEV_H_ */ Hi Wei,
Quoted 2 lines
Please help me understand the rawdev, who should be calling the above newly added functions?
Hi Ferruh,
Cyborg is an OpenStack project that aims to provide a general purpose management framework for acceleration resources (i.e. various types of accelerators such as GPU, FPGA, NP, ODP, DPDK/SPDK and so on).
Quoted 8 lines
To update the FPGA flash is one of requirements from Cyborg. Originally there are no such interfaces, so I added them.
These interfaces use
rte_rawdevto identify which FPGA to access, they will be called inopae_update_flash(),opae_cancel_flash_update()andopae_reboot_device()inifpga_opae_api.c.These
opae_xxxfunction use PCI address to identify FPGA instead ofrte_rawdev, so the caller has no need to know the existence ofrte_rawdev. In fact, Cyborg is Python application, theseopae_xxxfunctions will be eventually wrapped in a Python module for Cyborg to call.
Thanks for clarification, I see what you are doing, but still I think these APIs are not belong to a driver, they look like more application level.
@Wei, @Rosen, what do you think to keep only generic raw/ipfga APIs in the driver and move all 'ifpga_opae_api.c/h' to the sample application?
In that case raw/ifpga APIs still can get the port_id as parameter, and the 'opae' layer in the sample can do the conversion?
Re: [dpdk-dev] [dpdk-stable] [PATCH v11 1/4] raw/ifpga: add fpga rsu function
Hi,
Quoted 74 lines
-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Monday, January 25, 2021 18:01
To: Huang, Wei <[email protected]>; [email protected]; Xu, Rosen <[email protected]>; Zhang, Qi Z <[email protected]>
Cc: [email protected]; Zhang, Tianfei <[email protected]>; Ray Kinsella <[email protected]>
Subject: Re: [dpdk-stable] [PATCH v11 1/4] raw/ifpga: add fpga rsu functionOn 1/22/2021 2:18 AM, Huang, Wei wrote:
<Please do not top post, your comment moved below>
Quoted 41 lines
Quoted 41 lines
-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Friday, January 22, 2021 00:30
To: Huang, Wei <[email protected]>; [email protected]; Xu, Rosen <[email protected]>; Zhang, Qi Z <[email protected]>
Cc: [email protected]; Zhang, Tianfei <[email protected]>; Ray Kinsella <[email protected]>
Subject: Re: [dpdk-stable] [PATCH v11 1/4] raw/ifpga: add fpga rsu functionQuoted 11 lines · Wei Huang
RSU (Remote System Update) depends on secure manager which may be different on various implementations, so a new secure manager device is implemented for adapting such difference.
There are three major functions added:
1.ifpga_rawdev_update_flash()updates flash with specific image file.
2.ifpga_rawdev_stop_flash_update()aborts flash update process.
3.ifpga_rawdev_reload()reloads FPGA from updated flash.Signed-off-by Wei Huang <[email protected]>Acked-by Tianfei Zhang <[email protected]>Acked-by Rosen Xu <[email protected]><...>
drivers/raw/ifpga/ifpga_rawdev.h int · line 84
Quoted 12 lines
drivers/raw/ifpga/ifpga_rawdev.h +6
-0@@ -76,4 +76,9 @@ int ifpga_unregister_msix_irq(enum ifpga_irq_type type, int vec_start, rte_intr_callback_fn handler, void *arg); +int ifpga_rawdev_update_flash (struct rte_rawdev *dev,const char +*image,uint64_t *status);int +ifpga_rawdev_stop_flash_update (struct rte_rawdev *dev,int force); +int ifpga_rawdev_reload (struct rte_rawdev *dev,int type,int +page); +#endif /* _IFPGA_RAWDEV_H_ */ Hi Wei,
Quoted 1 line
Quoted 1 line
Please help me understand the rawdev, who should be calling the above
newly added functions?
Quoted 5 lines
Quoted 1 line
Hi Ferruh,
Cyborg is an OpenStack project that aims to provide a general purpose
management framework for acceleration resources (i.e. various types of accelerators such as GPU, FPGA, NP, ODP, DPDK/SPDK and so on).
Quoted 2 lines
To update the FPGA flash is one of requirements from Cyborg. Originally
there are no such interfaces, so I added them.
Quoted 2 lines
These interfaces use
rte_rawdevto identify which FPGA to access, they willbe called in
opae_update_flash(), >opae_cancel_flash_update()andopae_reboot_device()inifpga_opae_api.c.Quoted 2 lines
These
opae_xxxfunction use PCI address to identify FPGA instead of
rte_rawdev, so the caller has no need to know the existence ofrte_rawdev.Quoted 1 line
In fact, Cyborg is Python application, these
opae_xxxfunctions will be
Quoted 8 lines
eventually wrapped in a Python module for Cyborg to call.
Quoted 1 line
Thanks for clarification, I see what you are doing, but still I think these APIs are not belong to a driver, they look like more application level.
@Wei, @Rosen, what do you think to keep only generic raw/ipfga APIs in the driver and move all 'ifpga_opae_api.c/h' to the sample application?
@Huang, Wei, could you classify which functions are only used in sample application?
Quoted 2 lines
In that case raw/ifpga APIs still can get the
port_idas parameter, and the 'opae' layer in the sample can do the conversion?
Re: [dpdk-dev] [dpdk-stable] [PATCH v11 1/4] raw/ifpga: add fpga rsu function
Hi,
-----Original Message-----
From: Xu, Rosen <[email protected]>
Sent: Monday, January 25, 2021 18:33
To: Yigit, Ferruh <[email protected]>; Huang, Wei <[email protected]>; [email protected]; Zhang, Qi Z <[email protected]>
Cc: [email protected]; Zhang, Tianfei <[email protected]>; Ray Kinsella <[email protected]>
Subject: RE: [dpdk-stable] [PATCH v11 1/4] raw/ifpga: add fpga rsu function
Hi,
Quoted 77 lines
-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Monday, January 25, 2021 18:01
To: Huang, Wei <[email protected]>; [email protected]; Xu, Rosen <[email protected]>; Zhang, Qi Z <[email protected]>
Cc: [email protected]; Zhang, Tianfei <[email protected]>; Ray Kinsella <[email protected]>
Subject: Re: [dpdk-stable] [PATCH v11 1/4] raw/ifpga: add fpga rsu functionOn 1/22/2021 2:18 AM, Huang, Wei wrote:
<Please do not top post, your comment moved below>
Quoted 41 lines
Quoted 41 lines
-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Friday, January 22, 2021 00:30
To: Huang, Wei <[email protected]>; [email protected]; Xu, Rosen <[email protected]>; Zhang, Qi Z <[email protected]>
Cc: [email protected]; Zhang, Tianfei <[email protected]>; Ray Kinsella <[email protected]>
Subject: Re: [dpdk-stable] [PATCH v11 1/4] raw/ifpga: add fpga rsu functionQuoted 11 lines · Wei Huang
RSU (Remote System Update) depends on secure manager which may be different on various implementations, so a new secure manager device is implemented for adapting such difference.
There are three major functions added:
1.ifpga_rawdev_update_flash()updates flash with specific image file.
2.ifpga_rawdev_stop_flash_update()aborts flash update process.
3.ifpga_rawdev_reload()reloads FPGA from updated flash.Signed-off-by Wei Huang <[email protected]>Acked-by Tianfei Zhang <[email protected]>Acked-by Rosen Xu <[email protected]><...>
drivers/raw/ifpga/ifpga_rawdev.h int · line 84
Quoted 12 lines
drivers/raw/ifpga/ifpga_rawdev.h +6
-0@@ -76,4 +76,9 @@ int ifpga_unregister_msix_irq(enum ifpga_irq_type type, int vec_start, rte_intr_callback_fn handler, void *arg); +int ifpga_rawdev_update_flash (struct rte_rawdev *dev,const char +*image,uint64_t *status);int +ifpga_rawdev_stop_flash_update (struct rte_rawdev *dev,int +force);int ifpga_rawdev_reload (struct rte_rawdev *dev,int type, +int page); +#endif /* _IFPGA_RAWDEV_H_ */ Hi Wei,
Quoted 2 lines
Quoted 2 lines
Please help me understand the rawdev, who should be calling the above
newly added functions?
Quoted 5 lines
Quoted 1 line
Hi Ferruh,
Cyborg is an OpenStack project that aims to provide a general
purpose management framework for acceleration resources (i.e. various types of accelerators such as GPU, FPGA, NP, ODP, DPDK/SPDK and so on).
Quoted 2 lines
To update the FPGA flash is one of requirements from Cyborg.
Originally there are no such interfaces, so I added them.
Quoted 2 lines
These interfaces use
rte_rawdevto identify which FPGA to access,they will be called in
opae_update_flash(), >opae_cancel_flash_update()andopae_reboot_device()inifpga_opae_api.c.Quoted 2 lines
These
opae_xxxfunction use PCI address to identify FPGA instead of
rte_rawdev, so the caller has no need to know the existence ofrte_rawdev.Quoted 1 line
In fact, Cyborg is Python application, these
opae_xxxfunctions
Quoted 8 lines
will be eventually wrapped in a Python module for Cyborg to call.
Quoted 1 line
Thanks for clarification, I see what you are doing, but still I think these APIs are not belong to a driver, they look like more application level.
@Wei, @Rosen, what do you think to keep only generic raw/ipfga APIs in the driver and move all 'ifpga_opae_api.c/h' to the sample application?
@Huang, Wei, could you classify which functions are only used in sample application?
All the opae_xxx functions used in sample application will be used by Cyborg. I will try to move ifpga_opae_api.c/h to sample according to Ferruh's comments.
Quoted 2 lines
In that case raw/ifpga APIs still can get the
port_idas parameter, and the 'opae' layer in the sample can do the conversion?
PATCH v11 2/4 raw/ifpga: add fpga property get function
In reply to Wei HuangThere are three types of property can be got from FPGA, they are implemented in below functions:
1. ifpga_rawdev_get_fme_property() get property of FME (FPGA Management Engine).
2. ifpga_rawdev_get_port_property() get property of FPGA port.
3. ifpga_rawdev_get_bmc_property() get property of BMC (Board Management Controller).
--- drivers/raw/ifpga/base/ifpga_api.c | 8 ++ drivers/raw/ifpga/base/ifpga_defines.h | 1 + drivers/raw/ifpga/base/ifpga_feature_dev.c | 21 +++ drivers/raw/ifpga/base/ifpga_feature_dev.h | 1 + drivers/raw/ifpga/base/ifpga_fme.c | 28 +++- drivers/raw/ifpga/base/opae_hw_api.c | 18 +++ drivers/raw/ifpga/base/opae_hw_api.h | 2 + drivers/raw/ifpga/base/opae_ifpga_hw_api.h | 1 + drivers/raw/ifpga/ifpga_rawdev.c | 157 +++++++++++++++++++++ drivers/raw/ifpga/ifpga_rawdev.h | 28 ++++ 10 files changed, 263 insertions(+), 2 deletions(-)
drivers/raw/ifpga/base/ifpga_api.c +8 -0
@@ -229,6 +229,13 @@ static int ifpga_mgr_get_board_info(struct opae_manager *mgr,return 0 ; } +static int ifpga_mgr_get_uuid (struct opae_manager *mgr,struct uuid *uuid) +{ +struct ifpga_fme_hw *fme = mgr->data; + +return fpga_get_pr_uuid (fme, uuid); +} +static int ifpga_mgr_update_flash (struct opae_manager *mgr,const char *image,u64 *status) {
@@ -256,6 +263,7 @@ struct opae_manager_ops ifpga_mgr_ops = { .get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info, .get_sensor_value = ifpga_mgr_get_sensor_value, .get_board_info = ifpga_mgr_get_board_info, + .get_uuid = ifpga_mgr_get_uuid, .update_flash = ifpga_mgr_update_flash, .stop_flash_update = ifpga_mgr_stop_flash_update, .reload = ifpga_mgr_reload,
drivers/raw/ifpga/base/ifpga_defines.h +1 -0
@@ -1727,6 +1727,7 @@ struct opae_board_info {u8 seu;u8 ptp; +u32 boot_page;u32 max10_version;u32 nios_fw_version;u32 nums_of_retimer;
drivers/raw/ifpga/base/ifpga_feature_dev.c +21 -0
@@ -87,6 +87,27 @@ int fpga_get_afu_uuid(struct ifpga_port_hw *port, struct uuid *uuid)return 0 ; } +int fpga_get_pr_uuid (struct ifpga_fme_hw *fme,struct uuid *uuid) +{ +struct feature_fme_pr *fme_pr; +u64 guidl, guidh; + +if (!fme || !uuid) +return -EINVAL ; + + fme_pr =get_fme_feature_ioaddr_by_index (fme,FME_FEATURE_ID_PR_MGMT ); + +spinlock_lock (&fme->lock); + guidl =readq (&fme_pr->fme_pr_intfc_id_l); + guidh =readq (&fme_pr->fme_pr_intfc_id_h); +spinlock_unlock (&fme->lock); + +opae_memcpy (uuid->b, &guidl,sizeof (u64)); +opae_memcpy (uuid->b +8 , &guidh,sizeof (u64)); + +return 0 ; +} +/* Mask / Unmask Port Errors by the Error Mask register. */ void port_err_mask (struct ifpga_port_hw *port,bool mask) {
drivers/raw/ifpga/base/ifpga_feature_dev.h +1 -0
@@ -103,6 +103,7 @@ is_port_feature_present(struct ifpga_port_hw *port, int index) }int fpga_get_afu_uuid (struct ifpga_port_hw *port,struct uuid *uuid); +int fpga_get_pr_uuid (struct ifpga_fme_hw *fme,struct uuid *uuid);int __fpga_port_disable (struct ifpga_port_hw *port);void __fpga_port_enable (struct ifpga_port_hw *port);
drivers/raw/ifpga/base/ifpga_fme.c +26 -2
@@ -101,6 +101,24 @@ static int fme_hdr_get_ports_num(struct ifpga_fme_hw *fme, u64 *ports_num)return 0 ; } +static int fme_hdr_get_port_type (struct ifpga_fme_hw *fme,u64 *port_type) +{ +struct feature_fme_header *fme_hdr + =get_fme_feature_ioaddr_by_index (fme,FME_FEATURE_ID_HEADER ); +struct feature_fme_port pt; +u32 port = (u32)((*port_type >>32 ) &0xffffffff ); + + pt.csr =readq (&fme_hdr->port[port]); +if (!pt.port_implemented) +return -ENODEV ; +if (pt.afu_access_control) + *port_type |=0x1 ; +else + *port_type &= ~0x1 ; + +return 0 ; +} +static int fme_hdr_get_cache_size (struct ifpga_fme_hw *fme,u64 *cache_size) {struct feature_fme_header *fme_hdr
@@ -179,6 +197,8 @@ fme_hdr_get_prop(struct ifpga_feature *feature, struct feature_prop *prop)return fme_hdr_get_bitstream_id (fme, &prop->data);case FME_HDR_PROP_BITSTREAM_METADATA :return fme_hdr_get_bitstream_metadata (fme, &prop->data); +case FME_HDR_PROP_PORT_TYPE : +return fme_hdr_get_port_type (fme, &prop->data); }return -ENOENT ;
@@ -891,13 +911,17 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme) fme->board_info.nums_of_fvl, fme->board_info.ports_per_fvl); +if (max10_sys_read (fme->max10_dev,FPGA_PAGE_INFO , &val)) +return -EINVAL ; + fme->board_info.boot_page = val &0x7 ; +if (max10_sys_read (fme->max10_dev,MAX10_BUILD_VER , &val))return -EINVAL ; - fme->board_info.max10_version = val &0xffffff ; + fme->board_info.max10_version = val;if (max10_sys_read (fme->max10_dev,NIOS2_FW_VERSION , &val))return -EINVAL ; - fme->board_info.nios_fw_version = val &0xffffff ; + fme->board_info.nios_fw_version = val; dev_info(fme, "max10 version 0x%x, nios fw version 0x%x\n", fme->board_info.max10_version,
drivers/raw/ifpga/base/opae_hw_api.c +18 -0
@@ -967,6 +967,24 @@ opae_mgr_get_board_info(struct opae_manager *mgr,return -ENOENT ; } +/** +* opae_mgr_get_uuid - get manager's UUID. +* @mgr: targeted manager +* @uuid: a pointer to UUID +* +* Return: 0 on success, otherwise error code. +*/ +int opae_mgr_get_uuid (struct opae_manager *mgr,struct uuid *uuid) +{ +if (!mgr || !uuid) +return -EINVAL ; + +if (mgr->ops && mgr->ops->get_uuid) +return mgr->ops->get_uuid (mgr, uuid); + +return -ENOENT ; +} +/** * opae_mgr_update_flash - update image in flash. * @mgr: targeted manager
drivers/raw/ifpga/base/opae_hw_api.h +2 -0
@@ -55,6 +55,7 @@ struct opae_manager_ops { unsigned int *value);int (*get_board_info)(struct opae_manager *mgr,struct opae_board_info **info); +int (*get_uuid)(struct opae_manager *mgr,struct uuid *uuid);int (*update_flash)(struct opae_manager *mgr,const char *image,u64 *status);int (*stop_flash_update)(struct opae_manager *mgr,int force);
@@ -360,6 +361,7 @@ int opae_manager_eth_group_read_reg(struct opae_manager *mgr, u8 group_id, u8 type, u8 index, u16 addr, u32 *data);int opae_mgr_get_board_info (struct opae_manager *mgr,struct opae_board_info **info); +int opae_mgr_get_uuid (struct opae_manager *mgr,struct uuid *uuid);int opae_mgr_update_flash (struct opae_manager *mgr,const char *image,uint64_t *status);int opae_mgr_stop_flash_update (struct opae_manager *mgr,int force);
drivers/raw/ifpga/base/opae_ifpga_hw_api.h +1 -0
@@ -61,6 +61,7 @@ struct feature_prop {#define FME_HDR_PROP_SOCKET_ID 0x5/* RDONLY */ #define FME_HDR_PROP_BITSTREAM_ID 0x6/* RDONLY */ #define FME_HDR_PROP_BITSTREAM_METADATA 0x7/* RDONLY */ +#define FME_HDR_PROP_PORT_TYPE 0x8/* RDWR */ /* FME error reporting feature's properties */ /* FME error reporting properties format */
drivers/raw/ifpga/ifpga_rawdev.c +157 -0
163 lines not on this page · raw message
drivers/raw/ifpga/ifpga_rawdev.h +28 -0
@@ -67,6 +67,28 @@ enum ifpga_irq_type { IFPGA_AFU_IRQ = 1, }; +typedef struct { +uint8_t b[16 ]; +}ifpga_uuid ; + +typedef struct { +uint32_t boot_page; +uint32_t num_ports; +uint64_t bitstream_id; +uint64_t bitstream_metadata; +ifpga_uuid pr_id; +}ifpga_fme_property ; + +typedef struct { +ifpga_uuid afu_id; +uint32_t type;/* AFU memory access control type */ +}ifpga_port_property ; + +typedef struct { +uint32_t bmc_version; +uint32_t fw_version; +}ifpga_bmc_property ; + int ifpga_register_msix_irq(struct rte_rawdev *dev, int port_id, enum ifpga_irq_type type, int vec_start, int count,
@@ -76,6 +98,12 @@ int ifpga_unregister_msix_irq(enum ifpga_irq_type type, int vec_start, rte_intr_callback_fn handler, void *arg); +int ifpga_rawdev_get_fme_property (struct rte_rawdev *dev, +ifpga_fme_property *prop); +int ifpga_rawdev_get_port_property (struct rte_rawdev *dev,uint32_t port, +ifpga_port_property *prop); +int ifpga_rawdev_get_bmc_property (struct rte_rawdev *dev, +ifpga_bmc_property *prop);int ifpga_rawdev_update_flash (struct rte_rawdev *dev,const char *image,uint64_t *status);int ifpga_rawdev_stop_flash_update (struct rte_rawdev *dev,int force);
Signature · 1 line
2.29.2
PATCH v11 3/4 raw/ifpga: add OPAE API for OpenStack Cyborg
In reply to Wei HuangCyborg is an OpenStack project that aims to provide a general purpose management framework for acceleration resources (i.e. various types of accelerators such as GPU, FPGA, NP, ODP, DPDK/SPDK and so on). It needs some OPAE type APIs to manage PACs (Programmable Acceleration Card) with Intel FPGA. Below major functions are added to meets Cyborg requirements.
1. opae_init() set up OPAE environment.
2. opae_cleanup() clean up OPAE environment.
3. opae_enumerate() searches PAC with specific FPGA.
4. opae_get_property() gets properties of FPGA.
5. opae_partial_reconfigure() perform partial configuration on FPGA.
6. opae_get_image_info() gets information of image file.
7. opae_update_flash() updates FPGA flash with specific image file.
8. opae_cancel_flash_update() cancel process of FPGA flash update.
9. opae_probe_device() manually probe specific FPGA with ifpga driver.
10. opae_remove_device() manually remove specific FPGA from ifpga driver.
11. opae_bind_driver() binds specific FPGA with specified kernel driver.
12. opae_unbind_driver() unbinds specific FPGA from kernel driver.
13. opae_reboot_device() reboots specific FPGA (do reconfiguration).
---
v2: fix typo in commit log and ifpga_opae_api.h
---
v3: fix coding style issue in ifpga_opae_api.c
---
v4: enclose macro PCI_EXT_CAP_ID in parentheses
---
v5: fix icc compiler build error
---
v6: fix compilation issues in ifpga_opae_api.c
---
v7: remove call of rte_eal_init() and rte_eal_cleanup()
--- drivers/raw/ifpga/ifpga_opae_api.c | 1793 ++++++++++++++++++++++++++++ drivers/raw/ifpga/ifpga_opae_api.h | 248 ++++ drivers/raw/ifpga/ifpga_rawdev.c | 152 ++- drivers/raw/ifpga/ifpga_rawdev.h | 15 + drivers/raw/ifpga/meson.build | 4 +- 5 files changed, 2205 insertions(+), 7 deletions(-) create mode 100644 drivers/raw/ifpga/ifpga_opae_api.c create mode 100644 drivers/raw/ifpga/ifpga_opae_api.h
drivers/raw/ifpga/ifpga_opae_api.c new file +1793 -0
@@ -0,0 +1,1793 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#include <stdlib.h> +#include <string.h> +#include <dirent.h> +#include <fcntl.h> +#include <glob.h> +#include <unistd.h> +#include <sys/stat.h> +#include <rte_eal.h> +#include <rte_bus_pci.h> +#include <rte_rawdev_pmd.h> +#include "base/opae_hw_api.h" +#include "base/ifpga_sec_mgr.h" +#include "ifpga_rawdev.h" +#include "ifpga_opae_api.h" + + +int opae_log_level; +FILE *opae_log_file; + +static opae_api_version api_ver = {21 ,2 ,0 }; +static int eal_inited; +static uint32_t dev_aer[2 ] = {0 }; + +static const char *const log_level_name[] = {"CRITICAL" ,"ERROR" , +"WARNING" ,"INFORMATION" ,"DEBUG" }; +static const char *const proc_type_name[] = {"NON-DPDK" ,"PRIMARY" , +"SECONDARY" }; +static const char *const platform_name[] = {"Vista Creek" ,"Rush Creek" , +"Darby Creek" ,"Lightning Creek" }; +static const char *const release_name[] = {"Pre-Alpha" ,"Alpha" ,"Beta" ,"PV" }; +static const char *const interface_type[] = {"8x10G" ,"4x25G" ,"2x1x25G" , +"4x25G+2x25G" ,"2x2x25G" ,"2x1x25Gx2FVL" ,"1x2x25G" }; +static const char *const kdrv[] = {OPAE_KDRV_UNKNOWN ,OPAE_KDRV_IGB_UIO , +OPAE_KDRV_VFIO_PCI ,OPAE_KDRV_UIO_PCI }; + +RTE_INIT (init_api_env) +{ + eal_inited =0 ; + opae_log_level =OPAE_LOG_ERR ; + opae_log_file =NULL ; + ifpga_rawdev_logtype =0 ; + +opae_log_info ("API environment is initialized\n" ); +} + +RTE_FINI (clean_api_env) +{ +if (opae_log_file) { +fclose (opae_log_file); + opae_log_file =NULL ; + } +opae_log_info ("API environment is cleaned\n" ); +} + +void opae_get_api_version (opae_api_version *version) +{ +if (version) +memcpy (version, &api_ver,sizeof (opae_api_version)); +opae_log_info ("API version is %u.%u.%u\n" , + api_ver.major, api_ver.minor, api_ver.micro); +} + +int opae_set_log_level (int level) +{ +if ((level >=OPAE_LOG_API ) && (level <=OPAE_LOG_DEBUG )) + opae_log_level = level; +opae_log_api ("Current log level is %s\n" , + log_level_name[opae_log_level]); +return opae_log_level; +} + +int opae_set_log_file (char *path,int clean) +{ +FILE *f =NULL ; +time_t start; +struct tm *lt =NULL ; + +if (path) { +if (clean) + f =fopen (path,"w+" ); +else + f =fopen (path,"a+" ); + +if (f) { +if (opae_log_file) { +fclose (opae_log_file); + opae_log_file =NULL ; + } +time (&start); + lt =localtime (&start); +if (lt) +fprintf (f,"================%d-%02d-%02d " +"%02d:%02d:%02d================\n" , +1900 + lt->tm_year,1 + lt->tm_mon, + lt->tm_mday, + lt->tm_hour, lt->tm_min, lt->tm_sec); +fflush (f); + opae_log_file = f; + }else { +opae_log_err ("failed to open log file \'%s\'\n" , path); +return -1 ; + } + }else { +if (opae_log_file) { +fclose (opae_log_file); + opae_log_file =NULL ; + } + } + +return 0 ; +} + +int opae_get_image_info (const char *image,opae_img_info *info) +{ +int fd =-1 ; +off_t file_size =0 ; +opae_img_hdr hdr; +ssize_t read_size =0 ; +int ret =0 ; + +if (!image || !info) { +opae_log_err ("Input parameter of %s is invalid\n" , __func__); +return -EINVAL ; + } + + fd =open (image,O_RDONLY ); +if (fd <0 ) { +opae_log_err ("Failed to open \'%s\' for RD [e:%s]\n" , + image,strerror (errno)); +return -EIO ; + } + + file_size =lseek (fd,0 ,SEEK_END ); +if (file_size < (OPAE_IMG_HDR_SIZE +OPAE_IMG_PL_MIN_SIZE )) { +opae_log_err ("Size of \'%s\' is less than expected [e:%u]\n" , + image,OPAE_IMG_HDR_SIZE +OPAE_IMG_PL_MIN_SIZE ); + ret = -EINVAL ; + goto close_fd; + } + +/* read image header */ +lseek (fd,0 ,SEEK_SET ); + read_size =read (fd, (void *)&hdr,sizeof (opae_img_hdr)); +if (read_size <0 ) { +opae_log_err ("Failed to read from \'%s\' [e:%s]\n" , + image,strerror (errno)); + ret = -EIO ; + goto close_fd; + } +if ((size_t )read_size !=sizeof (opae_img_hdr)) { +opae_log_err ("Read length %zd is not expected [e:%zu]\n" , + read_size,sizeof (opae_img_hdr)); + ret = -EIO ; + goto close_fd; + } + + info->total_len = file_size; +/* check signed image header */ +if (hdr.magic ==OPAE_IMG_BLK0_MAGIC ) { + info->type =OPAE_IMG_TYPE (hdr.payload_type); + info->subtype =OPAE_IMG_SUBTYPE (hdr.payload_type); + info->payload_offset =OPAE_IMG_HDR_SIZE ; + info->payload_len = hdr.payload_len; + }else { +opae_log_err ("Image \'%s\' can not be recognized\n" , image); + ret = -EINVAL ; + } +close_fd: +close (fd); +return ret; +} + +static int write_file (char *path,char *buf,int size) +{ +int fd =-1 ; +ssize_t n =0 ; + +if (!path || !buf || (size <=0 )) +return -EINVAL ; + + fd =open (path,O_WRONLY ); +if (fd <0 ) { +opae_log_err ("Failed to open \'%s\' for WR [e:%s]\n" , + path,strerror (errno)); +return -EIO ; + } +opae_log_dbg ("Write \"%s\" to \'%s\'\n" , buf, path); + n =write (fd, buf, size); +if (n < size) { +opae_log_err ("Failed to write to \'%s\' [e:%s]\n" , + path,strerror (errno)); +close (fd); +return -EIO ; + } +close (fd); + +return 0 ; +} + +static int read_file (char *path,char *buf,int size) +{ +int fd =-1 ; +ssize_t n =0 ; + +if (!path || !buf || (size <=0 )) +return -EINVAL ; + + fd =open (path,O_RDONLY ); +if (fd <0 ) { +opae_log_err ("Failed to open \'%s\' for RD [e:%s]\n" , + path,strerror (errno)); +return -EIO ; + } + n =read (fd, buf, size); +if (n <0 ) { +opae_log_err ("Failed to read from \'%s\' [e:%s]\n" , + path,strerror (errno)); +close (fd); +return -EIO ; + } +close (fd); + +if (n >0 ) + buf[n-1 ] =0 ; + +opae_log_dbg ("Read \"%s\" from \'%s\'\n" , buf, path); +return 0 ; +} + +int opae_get_proc_type (void ) +{ +int type =-1 ; + +if (eal_inited) { +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) + type =0 ; +else + type =1 ; + } +opae_log_info ("Current process type is %s\n" , proc_type_name[type+1 ]); + +return type; +} + +static bool check_eal (int inited) +{ +if (!eal_inited) { +if (inited) { +opae_log_warn ("EAL is not initialized\n" ); +return 0 ; + } + }else { +if (!inited) { +opae_log_warn ("EAL is already initialized\n" ); +return 0 ; + } + } + +return 1 ; +} + +int opae_init (int eal_init_result) +{ +int ret =0 ; + +if (!check_eal (0 )) +return 0 ; + +if (eal_init_result <0 ) { +if (rte_errno ==EALREADY ) { + eal_inited =1 ; +opae_log_info ("EAL already initialized\n" ); + }else { +opae_log_err ("Cannot initialize EAL\n" ); + ret =-1 ; + } + }else { + eal_inited =1 ; +opae_log_info ("Initialize EAL done\n" ); + } + +return ret; +} + +int opae_cleanup (void ) +{ +if (!check_eal (1 )) +return -EPERM ; + +ifpga_rawdev_cleanup (); + +return 0 ; +} + +static int compare_pci_id (opae_pci_id *id,opae_pci_id *expected_id) +{ +if ((expected_id->class_id !=BIT_SET_32 ) && + (expected_id->class_id != id->class_id)) +return -1 ; +if ((expected_id->vendor_id !=BIT_SET_16 ) && + (expected_id->vendor_id != id->vendor_id)) +return -1 ; +if ((expected_id->device_id !=BIT_SET_16 ) && + (expected_id->device_id != id->device_id)) +return -1 ; +if ((expected_id->subsystem_vendor_id !=BIT_SET_16 ) && + (expected_id->subsystem_vendor_id != id->subsystem_vendor_id)) +return -1 ; +if ((expected_id->subsystem_device_id !=BIT_SET_16 ) && + (expected_id->subsystem_device_id != id->subsystem_device_id)) +return -1 ; + +return 0 ; +} + +static int parse_sysfs_value (char *node,uint32_t *val) +{ +char buf[16 ]; +char *end =NULL ; +int ret =0 ; + + ret =read_file (node, buf,sizeof (buf)); +if (ret <0 ) +return ret; + + *val = (uint32_t )strtoul (buf, &end,0 ); +return 0 ; +} + +static int get_pci_id (const char *dev_path,opae_pci_id *id) +{ +char path[PATH_MAX ] = {0 }; +uint32_t tmp; + +if (!dev_path || !id) +return -EINVAL ; + +snprintf (path,sizeof (path),"%s/vendor" , dev_path); +if (parse_sysfs_value (path, &tmp) <0 ) +return -ENODEV ; + id->vendor_id = (uint16_t )tmp; + +snprintf (path,sizeof (path),"%s/device" , dev_path); +if (parse_sysfs_value (path, &tmp) <0 ) +return -ENODEV ; + id->device_id = (uint16_t )tmp; + +snprintf (path,sizeof (path),"%s/subsystem_vendor" , dev_path); +if (parse_sysfs_value (path, &tmp) <0 ) +return -ENODEV ; + id->subsystem_vendor_id = (uint16_t )tmp; + +snprintf (path,sizeof (path),"%s/subsystem_device" , dev_path); +if (parse_sysfs_value (path, &tmp) <0 ) +return -ENODEV ; + id->subsystem_device_id = (uint16_t )tmp; + +snprintf (path,sizeof (path),"%s/class" , dev_path); +if (parse_sysfs_value (path, &tmp) <0 ) +return -ENODEV ; + id->class_id = (uint32_t )tmp &RTE_CLASS_ANY_ID ; + +return 0 ; +} + +static int extract_path (char *in,int ridx,char *out,uint32_t size) +{ +char src[PATH_MAX ] = {0 }; +char *p =NULL ; +int ret =0 ; + +if (!in || (strlen (in) >PATH_MAX ) || (ridx <0 ) || !out) +return -EINVAL ; + +strncpy (src, in,sizeof (src)); + *out =0 ; + +while (1 ) { + p =strrchr (src,'/' ); +if (p) { + *p++ =0 ; +if (*p) { +if (ridx-- <=0 ) { +if (size >strlen (p)) { +strncpy (out, p, size); + ret =strlen (p); + } +break ; + } + } + }else { +break ; + } + } + +return ret; +} + +int opae_enumerate (opae_pci_id *filter,pcidev_id list,int size) +{ +DIR *dir =NULL ; +struct dirent *dirent =NULL ; +char path[PATH_MAX ] = {0 }; +opae_pci_id id; +int n =0 ; + +if (!filter || (size <0 ) || (!list && (size >0 ))) { +opae_log_err ("Input parameter of %s is invalid\n" , __func__); +return -EINVAL ; + } + + dir =opendir (rte_pci_get_sysfs_path ()); +if (!dir) { +opae_log_err ("Failed to open \'%s\'\n" , +rte_pci_get_sysfs_path ()); +return -EINVAL ; + } +while ((dirent =readdir (dir))) { +if (!strcmp (dirent->d_name,"." )) +continue ; +if (!strcmp (dirent->d_name,".." )) +continue ; + +snprintf (path,PATH_MAX ,"%s/%s" ,rte_pci_get_sysfs_path (), + dirent->d_name); +if (get_pci_id (path, &id) <0 ) +continue ; +if (compare_pci_id (&id, filter) <0 ) +continue ; + +if (n++ < size) { +snprintf (list->bdf,sizeof (list->bdf),"%s" , + dirent->d_name); + list++; + } + } +closedir (dir); + +return n; +} + +static int get_driver (pcidev_id id,char *drv_name,uint32_t size) +{ +char path[PATH_MAX ] = {0 }; +char link[PATH_MAX ] = {0 }; +int ret =0 ; + +if (!id || !drv_name) { + ret = -EINVAL ; + goto end; + } + size--;/* reserve one byte for the end of string */ + +snprintf (path,PATH_MAX ,"%s/%s/driver" , +rte_pci_get_sysfs_path (), id->bdf); + ret =readlink (path, link,PATH_MAX ); +if (ret >=PATH_MAX ) { +opae_log_err ("Link path too long [%d]\n" , ret); + ret = -ENAMETOOLONG ; + goto end; + } +if (ret >0 ) { + ret =extract_path (link,0 , drv_name, size); + }else { + *drv_name =0 ; +opae_log_info ("No link path for \'%s\'\n" , path); + ret =0 ; + } + +end: +if (ret <0 ) +opae_log_err ("Failed to get driver of %s\n" , id->bdf); + +return ret; +} + +static int get_pci_addr (const char *bdf,opae_pci_addr *addr) +{ +unsigned int domain =0 ; +unsigned int bus =0 ; +unsigned int devid =0 ; +unsigned int function =0 ; +int ret =0 ; + +if (!bdf || !addr) +return -EINVAL ; + + ret =sscanf (bdf,"%04x:%02x:%02x.%d" , + &domain, &bus, &devid, &function); +if (ret ==4 ) { + addr->domain = (uint32_t )domain; + addr->bus = (uint8_t )bus; + addr->devid = (uint8_t )devid; + addr->function = (uint8_t )function; +return 0 ; + } + +return -EINVAL ; +} + +static struct rte_rawdev *get_rte_rawdev (pcidev_id id,int log) +{ +opae_pci_addr addr; +struct rte_rawdev *rdev =NULL ; +char rdev_name[OPAE_NAME_SIZE ] = {0 }; + +if (!id) +return NULL ; + +if (get_pci_addr (id->bdf, &addr) <0 ) +return NULL ; + +snprintf (rdev_name,OPAE_NAME_SIZE ,"IFPGA:%02x:%02x.%x" , + addr.bus, addr.devid, addr.function); + rdev =rte_rawdev_pmd_get_named_dev (rdev_name); +if (log && !rdev) +opae_log_warn ("%s is not probed\n" , id->bdf); + +return rdev; +} + +static struct rte_pci_device *get_rte_pcidev (pcidev_id id,int log) +{ +struct rte_rawdev *rdev =NULL ; +struct rte_pci_bus *pci_bus =NULL ; +struct rte_pci_device *pci_dev =NULL ; + +if (!id) +return NULL ; + + pci_bus =ifpga_get_pci_bus (); +if (pci_bus) { + TAILQ_FOREACH(pci_dev, &pci_bus->device_list, next) { +if (!strcmp (id->bdf, pci_dev->name)) +return pci_dev; + } + }else { + rdev =get_rte_rawdev (id,0 ); +if (rdev && rdev->device) { + pci_dev =RTE_DEV_TO_PCI (rdev->device); +return pci_dev; + } + } + +if (log) +opae_log_err ("No rte_pci_device for %s\n" , id->bdf); + +return NULL ; +} + +static int lock (pcidev_id id) +{ +struct rte_rawdev *rdev =NULL ; +int ret =0 ; + + rdev =get_rte_rawdev (id,0 ); +if (rdev) + ret =ifpga_rawdev_lock (rdev); + +return ret; +} + +static int unlock (pcidev_id id) +{ +struct rte_rawdev *rdev =NULL ; +int ret =0 ; + + rdev =get_rte_rawdev (id,0 ); +if (rdev) + ret =ifpga_rawdev_unlock (rdev); + +return ret; +} + +int opae_load_rsu_status (pcidev_id id,uint32_t *status,uint32_t *progress) +{ +struct rte_rawdev *rdev =NULL ; +uint32_t value =0 ; + +if (!check_eal (1 )) +return -EPERM ; + + rdev =get_rte_rawdev (id,1 ); +if (rdev) + value =ifpga_rawdev_get_rsu_stat (rdev); +else +return -ENODEV ; + +if (status) + *status = (value >>16 ) &0xffff ; +if (progress) + *progress = value &0xffff ; + +return 0 ; +} + +int opae_store_rsu_status (pcidev_id id,uint32_t status,uint32_t progress) +{ +struct rte_rawdev *rdev =NULL ; +uint32_t value =0 ; + +if (!check_eal (1 )) +return -EPERM ; + + rdev =get_rte_rawdev (id,1 ); +if (rdev) { + value = ((status <<16 ) &0xffff0000 ) | (progress &0xffff ); +ifpga_rawdev_set_rsu_stat (rdev, value); + }else { +return -ENODEV ; + } + +return 0 ; +} + +static int get_pci_property (pcidev_id id,opae_pci_property *prop) +{ +char path[PATH_MAX ] = {0 }; +int ret =0 ; + +if (!id || !prop) +return -EINVAL ; + +snprintf (path,PATH_MAX ,"%s/%s" ,rte_pci_get_sysfs_path (), id->bdf); + + ret =get_pci_id (path, &prop->id); +if (ret <0 ) +return ret; + + ret =get_pci_addr (id->bdf, &prop->addr); +if (ret <0 ) +return ret; + +snprintf (prop->pci_addr,OPAE_NAME_SIZE ,"%s" , id->bdf); +get_driver (id, prop->drv_name,sizeof (prop->drv_name)); + +return 0 ; +} + +static int get_fme_property (pcidev_id id,opae_fme_property *prop) +{ +struct rte_rawdev *rdev =NULL ; +ifpga_fme_property fme_prop; +opae_bitstream_id bbs_id; +int ret =0 ; + +if (!prop) +return -EINVAL ; + + rdev =get_rte_rawdev (id,1 ); +if (!rdev) +return -ENODEV ; + + ret =ifpga_rawdev_get_fme_property (rdev, &fme_prop); +if (!ret) { + prop->boot_page = fme_prop.boot_page; + prop->num_ports = fme_prop.num_ports; + prop->bitstream_id = fme_prop.bitstream_id; + prop->bitstream_metadata = fme_prop.bitstream_metadata; +memcpy (prop->pr_id.b, fme_prop.pr_id.b,sizeof (opae_uuid)); + + bbs_id.id = prop->bitstream_id; +if (bbs_id.major <sizeof (platform_name) /sizeof (char *)) { +snprintf (prop->platform_name, +sizeof (prop->platform_name),"%s" , + platform_name[bbs_id.major]); + }else { +snprintf (prop->platform_name, +sizeof (prop->platform_name),"unknown" ); + } + +snprintf (prop->dcp_version,sizeof (prop->dcp_version), +"DCP 1.%u" , bbs_id.minor); + +if (bbs_id.patch <sizeof (release_name)/sizeof (char *)) { +snprintf (prop->release_name,sizeof (prop->release_name), +"%s" , release_name[bbs_id.patch]); + }else { +snprintf (prop->release_name,sizeof (prop->release_name), +"unknown" ); + } + +if (bbs_id.major ==0 ) {/* Vista Creek */ +if (bbs_id.interface < +sizeof (interface_type) /sizeof (char *)) { +snprintf (prop->interface_type, +sizeof (prop->interface_type),"%s" , + interface_type[bbs_id.interface]); + }else { +snprintf (prop->interface_type, +sizeof (prop->interface_type),"unknown" ); + } + }else { +snprintf (prop->interface_type, +sizeof (prop->interface_type),"unknown" ); + } + +snprintf (prop->build_version,sizeof (prop->build_version), +"%u.%u.%u" , bbs_id.build_major, bbs_id.build_minor, + bbs_id.build_patch); + } + +return ret; +} + +static int get_port_property (pcidev_id id,uint32_t port, +opae_port_property *prop) +{ +struct rte_rawdev *rdev =NULL ; +ifpga_port_property port_prop; +int ret =0 ; + +if (!prop || (port >=OPAE_MAX_PORT_NUM )) +return -EINVAL ; + + rdev =get_rte_rawdev (id,1 ); +if (!rdev) +return -ENODEV ; + + ret =ifpga_rawdev_get_port_property (rdev, port, &port_prop); +if (!ret) { +memcpy (prop->afu_id.b, port_prop.afu_id.b,sizeof (opae_uuid)); + prop->type = port_prop.type; + prop->index = port; + } + +return 0 ; +} + +static int get_bmc_property (pcidev_id id,opae_bmc_property *prop) +{ +struct rte_rawdev *rdev =NULL ; +ifpga_bmc_property bmc_prop; +opae_bmc_version ver; +int ret =0 ; + +if (!prop) +return -EINVAL ; + + rdev =get_rte_rawdev (id,1 ); +if (!rdev) +return -ENODEV ; + + ret =ifpga_rawdev_get_bmc_property (rdev, &bmc_prop); +if (!ret) { + ver.version = bmc_prop.bmc_version; +snprintf (prop->bmc_version,sizeof (prop->bmc_version),"%c.%u.%u.%u" , + ver.board, ver.major, ver.minor, ver.micro); + + ver.version = bmc_prop.fw_version; +snprintf (prop->fw_version,sizeof (prop->fw_version),"%c.%u.%u.%u" , + ver.board, ver.major, ver.minor, ver.micro); + } + +return 0 ; +} + +int opae_get_property (pcidev_id id,opae_fpga_property *prop,int type) +{ +uint32_t status =0 ; +uint32_t i =0 ; +int ret =0 ; + +if (!prop) { +opae_log_err ("Input parameter of %s is invalid\n" , __func__); +return -EINVAL ; + } + +if (type ==0 ) + type =OPAE_PROP_ALL ; + +memset (prop,0 ,sizeof (opae_fpga_property)); + +/* PCI properties */ +if (type &OPAE_PROP_PCI ) { + ret =get_pci_property (id, &prop->pci); +if (ret <0 ) { +opae_log_err ("Failed to get PCI property\n" ); +return ret; + } + } + +if (type ==OPAE_PROP_PCI ) +return 0 ; + +if (!check_eal (1 )) +return -EPERM ; + +if (!get_rte_rawdev (id,1 )) +return -ENODEV ; + +lock (id); +opae_load_rsu_status (id, &status,NULL ); +if (status ==IFPGA_RSU_REBOOT ) { +opae_log_warn ("Reboot is in progress\n" ); + ret = -EAGAIN ; + goto unlock_dev; + } + +/* FME properties */ +if (type & (OPAE_PROP_FME |OPAE_PROP_PORT )) { + ret =get_fme_property (id, &prop->fme); +if (ret) { +opae_log_err ("Failed to get FME property\n" ); + goto unlock_dev; + } + } + +/* PORT properties */ +if (type &OPAE_PROP_PORT ) { +for (i =0 ; i < prop->fme.num_ports; i++) { + ret =get_port_property (id, i, &prop->port[i]); +if (ret) { +opae_log_err ("Failed to get port property\n" ); + goto unlock_dev; + } + } + } + +/* BMC properties */ +if (type &OPAE_PROP_BMC ) { + ret =get_bmc_property (id, &prop->bmc); +if (ret) { +opae_log_err ("Failed to get BMC property\n" ); + goto unlock_dev; + } + } + +unlock_dev: +unlock (id); +return ret; +} + +int opae_get_phy_info (pcidev_id id,opae_phy_info *info) +{ +struct rte_rawdev *rdev =NULL ; +ifpga_phy_info phy_info; +int ret =0 ; + +if (!info) +return -EINVAL ; + + rdev =get_rte_rawdev (id,1 ); +if (!rdev) +return -ENODEV ; + + ret =ifpga_rawdev_get_phy_info (rdev, &phy_info); +if (!ret) { + info->num_retimers = phy_info.num_retimers; + info->link_speed = phy_info.link_speed; + info->link_status = phy_info.link_status; + } + +return ret; +} + +static int update_driver (pcidev_id id,char *drv_name) +{ +struct rte_pci_device *pci_dev =NULL ; +char name[OPAE_NAME_SIZE ] = {0 }; +int ret =0 ; + +if (!id) +return -EINVAL ; + +if (drv_name) { +if (strlen (drv_name) >=OPAE_NAME_SIZE ) { +opae_log_err ("Driver name \'%s\' too long\n" , + drv_name); +return -EINVAL ; + } +strncpy (name, drv_name,sizeof (name)); + }else { + ret =get_driver (id, name,sizeof (name)); +if (ret <0 ) +return ret; + } + + pci_dev =get_rte_pcidev (id,0 ); +if (pci_dev) { +if (strlen (name) ==0 ) { + pci_dev->kdrv =RTE_PCI_KDRV_NONE ; + }else { +if (!strcmp (name,OPAE_KDRV_VFIO_PCI )) + pci_dev->kdrv =RTE_PCI_KDRV_VFIO ; +else if (!strcmp (name,OPAE_KDRV_IGB_UIO )) + pci_dev->kdrv =RTE_PCI_KDRV_IGB_UIO ; +else if (!strcmp (name,OPAE_KDRV_UIO_PCI )) + pci_dev->kdrv =RTE_PCI_KDRV_UIO_GENERIC ; +else + pci_dev->kdrv =RTE_PCI_KDRV_UNKNOWN ; + } + } + +return 0 ; +} + +int opae_unbind_driver (pcidev_id id) +{ +char path[PATH_MAX ] = {0 }; +char drv_name[OPAE_NAME_SIZE ] = {0 }; +char null[] = {0 }; +int ret =0 ; + +if (get_rte_rawdev (id,0 )) { +opae_log_err ("%s is probed, remove it first\n" , id->bdf); +return -EBUSY ; + } + + ret =get_driver (id, drv_name,sizeof (drv_name)); +if (ret <0 ) +return ret; + +if (strlen (drv_name) >0 ) { +snprintf (path,PATH_MAX ,"/sys/bus/pci/drivers/%s/unbind" , + drv_name); + ret =write_file (path, id->bdf,strlen (id->bdf) +1 ); +if (ret ==0 ) + ret =update_driver (id, null); + } + +return ret; +} + +static int check_driver (const char *drv_name) +{ +char path[PATH_MAX ] = {0 }; +struct stat buf; + +if (!drv_name) +return -EINVAL ; + +if (strlen (drv_name) >0 ) { +snprintf (path,PATH_MAX ,"/sys/bus/pci/drivers/%s" , drv_name); +if ((stat (path, &buf) <0 ) || ((buf.st_mode &S_IFDIR ) ==0 )) { +opae_log_warn ("Driver %s is not installed\n" , + drv_name); +return -EINVAL ; + } + } + +return 0 ; +} + +int opae_bind_driver (pcidev_id id,char *drv_name) +{ +char path[PATH_MAX ] = {0 }; +char name[OPAE_NAME_SIZE ] = {0 }; +char null[] = {0 }; +int ret =0 ; + + ret =check_driver (drv_name); +if (ret <0 ) +return ret; + + ret =get_driver (id, name,sizeof (name)); +if (ret <0 ) +return ret; + +if (!strcmp (drv_name, name))/* driver not change */ +return 0 ; + + ret =opae_unbind_driver (id); +if (ret <0 ) +return ret; + +if (strlen (drv_name) >0 ) { +/* bind driver */ +snprintf (path,PATH_MAX ,"%s/%s/driver_override" , +rte_pci_get_sysfs_path (), id->bdf); + ret =write_file (path, drv_name,strlen (drv_name) +1 ); +if (ret <0 ) + goto update_drv; + +snprintf (path,PATH_MAX ,"/sys/bus/pci/drivers/%s/bind" , + drv_name); + ret =write_file (path, id->bdf,strlen (id->bdf) +1 ); +if (ret <0 ) + goto update_drv; + +snprintf (path,PATH_MAX ,"%s/%s/driver_override" , +rte_pci_get_sysfs_path (), id->bdf); + ret =write_file (path, null,1 ); +if (ret <0 ) + goto update_drv; + } + +update_drv: + ret =update_driver (id,NULL ); +if (ret <0 ) +opae_log_err ("Failed to update driver information of %s\n" , + id->bdf); + +return 0 ; +} + +int opae_probe_device (pcidev_id id) +{ +struct rte_pci_bus *pci_bus =NULL ; +struct rte_pci_device *pci_dev =NULL ; + +if (!id) { +opae_log_err ("Input parameter of %s is invalid\n" , __func__); +return -EINVAL ; + } + +if (!check_eal (1 )) +return -EPERM ; + +/* make sure device is added in rte_pci_bus devices list */ + pci_bus =ifpga_get_pci_bus (); +if (pci_bus && pci_bus->bus.scan) + pci_bus->bus.scan (); + + pci_dev =get_rte_pcidev (id,1 ); +if (!pci_dev) +return -ENODEV ; + +if (pci_dev->kdrv !=RTE_PCI_KDRV_VFIO ) { +opae_log_err ("vfio-pci driver is not bound to %s\n" , id->bdf); +return -EINVAL ; + } + +if (!pci_bus || !pci_bus->bus.plug) +return -ENODEV ; + +return pci_bus->bus.plug (&pci_dev->device); +} + +int opae_remove_device (pcidev_id id) +{ +struct rte_pci_device *pci_dev =NULL ; +struct rte_pci_driver *pci_drv =NULL ; +int ret =0 ; + +if (!check_eal (1 )) +return -EPERM ; + + pci_dev =get_rte_pcidev (id,0 ); +if (pci_dev && pci_dev->driver) { + pci_drv = pci_dev->driver; + ret = pci_drv->remove (pci_dev); +if (ret <0 ) { +opae_log_err ("Failed to remove %s [e:%d]\n" , + id->bdf, ret); +return ret; + } + pci_dev->driver =NULL ; + pci_dev->device.driver =NULL ; +if (pci_drv->drv_flags &RTE_PCI_DRV_NEED_MAPPING ) +rte_pci_unmap_device (pci_dev); + } + +return ret; +} + +static int is_pac (pcidev_id id) +{ +char path[PATH_MAX ] = {0 }; +opae_pci_id pci_id; + +if (!id) +return 0 ; + +snprintf (path,PATH_MAX ,"%s/%s" ,rte_pci_get_sysfs_path (), id->bdf); +if (get_pci_id (path, &pci_id) <0 ) +return 0 ; + +if ((pci_id.vendor_id ==0x8086 ) && (pci_id.device_id ==0x0b30 )) +return 1 ; + +return 0 ; +} + +int opae_get_parent (pcidev_id id,pcidev_id parent) +{ +char path[PATH_MAX ] = {0 }; +char link[PATH_MAX ] = {0 }; +int ret =0 ; + +if (!id || !parent) { +opae_log_err ("Input parameter of %s is invalid\n" , __func__); + ret = -EINVAL ; + goto end; + } + +snprintf (path,PATH_MAX ,"%s/%s" ,rte_pci_get_sysfs_path (), id->bdf); + ret =readlink (path, link,PATH_MAX ); +if (ret >=PATH_MAX ) { +opae_log_err ("Length of link path exceeds %u\n" ,PATH_MAX ); + ret = -ENAMETOOLONG ; + goto end; + } + +if (ret >0 ) { + ret =extract_path (link,1 , parent->bdf,sizeof (parent->bdf)); +if (!strncmp (parent->bdf,"pci" ,3 )) { + parent->bdf[0 ] =0 ; + ret = -ENODEV ; + } + }else { + parent->bdf[0 ] =0 ; +if (ret ==0 ) +opae_log_err ("Length of link path is 0\n" ); +else +opae_log_err ("No link path for \'%s\'\n" , path); + } +end: +if (ret <=0 ) +opae_log_err ("%s has no parent\n" , id->bdf); + +return ret; +} + +int opae_get_child (pcidev_id id,pcidev_id child,int size) +{ +glob_t pglob = {.gl_pathc =0 , .gl_pathv =NULL }; +char path[PATH_MAX ] = {0 }; +int i, count =0 ; +int len =0 ; +int ret =0 ; + +if (!id || (size <0 )) { +opae_log_err ("Input parameter of %s is invalid\n" , __func__); +return -EINVAL ; + } + +snprintf (path,PATH_MAX ,"%s/%s/*:*:*.?" ,rte_pci_get_sysfs_path (), + id->bdf); + ret =glob (path,0 ,NULL , &pglob); +if (ret ==0 ) { +if (child && (size >0 )) { +for (i =0 ; i < (int )pglob.gl_pathc; i++) { + len =extract_path (pglob.gl_pathv[i],0 , + child->bdf,sizeof (child->bdf)); +if (len <=0 ) { + child->bdf[0 ] =0 ; +continue ; + } +if (++count >= size) +break ; + child++; + } + }else { + count = (int )pglob.gl_pathc; + } +globfree (&pglob); + }else { +if (pglob.gl_pathv) +globfree (&pglob); + } + +return count; +} + +int opae_get_pf1 (pcidev_id id,pcidev_id peer,int size) +{ +opae_pci_device parent; +opae_pci_device child[4 ]; +int n =0 ; +int ret =0 ; + +if (!id) { +opae_log_err ("Input parameter of %s is invalid\n" , __func__); +return -EINVAL ; + } + +if (!is_pac (id)) { +opae_log_info ("%s has no peer function\n" , id->bdf); +return -EINVAL ; + } + + ret =opae_get_parent (id, &parent); +if (ret <0 ) +return -ENODEV ; + ret =opae_get_parent (&parent, &parent); +if (ret <0 ) +return -ENODEV ; + + n =opae_get_child (&parent, child, +sizeof (child) /sizeof (opae_pci_device)); +/* there should have four downstream ports of PCI switch on board */ +if (n ==4 ) { + n =opae_get_child (&child[3 ], peer, size); + }else { + peer->bdf[0 ] =0 ; +opae_log_dbg ("%s has %d child(s)\n" , parent.bdf, n); + n =0 ; + } + +return n; +} + +void opae_check_pcidev_list (void ) +{ +int i =0 ; +unsigned int k =0 ; +struct rte_pci_bus *pci_bus =NULL ; +struct rte_pci_device *pci_dev =NULL ; + +if (!check_eal (1 )) +return ; + + pci_bus =ifpga_get_pci_bus (); +if (!pci_bus) +return ; + +printf (" ID NAME SEG BUS DEV FUNC VID DID KDRV\n" ); + TAILQ_FOREACH(pci_dev, &pci_bus->device_list, next) { + k = pci_dev->kdrv; +printf ("%3d %s %04x %02x %02x %2d %04x %04x %s\n" , + i, pci_dev->name, pci_dev->addr.domain, + pci_dev->addr.bus, pci_dev->addr.devid, + pci_dev->addr.function, pci_dev->id.vendor_id, + pci_dev->id.device_id, + k >RTE_PCI_KDRV_UIO_GENERIC ?"" : kdrv[k]); + i++; + } +} + +int opae_update_flash (pcidev_id id,const char *image,uint64_t *status) +{ +struct rte_rawdev *rdev =NULL ; +opae_img_info info; +int ret =0 ; + + ret =opae_get_image_info (image, &info); +if (ret <0 ) { +opae_log_err ("Failed to get image information [e:%d]\n" , ret); +return -EINVAL ; + } + +if ((info.type !=OPAE_IMG_TYPE_BBS ) && + (info.type !=OPAE_IMG_TYPE_BMC )) { +opae_log_err ("Image is not supported [t:%u]\n" , info.type); +return -EOPNOTSUPP ; + } + +if (!check_eal (1 )) +return -EPERM ; + + rdev =get_rte_rawdev (id,1 ); +if (!rdev) +return -ENODEV ; + +return ifpga_rawdev_update_flash (rdev, image, status); +} + +int opae_cancel_flash_update (pcidev_id id,int force) +{ +struct rte_rawdev *rdev =NULL ; + +if (!check_eal (1 )) +return -EPERM ; + + rdev =get_rte_rawdev (id,1 ); +if (!rdev) +return -ENODEV ; + +return ifpga_rawdev_stop_flash_update (rdev, force); +} + +#define PCI_EXT_CAP_ID_ERR 0x01/* Advanced Error Reporting */ +#define PCI_CFG_SPACE_SIZE 256 +#define PCI_CFG_SPACE_EXP_SIZE 4096 +#define PCI_EXT_CAP_ID (hdr) ((int)((hdr) & 0x0000ffff)) +#define PCI_EXT_CAP_NEXT (hdr) (((hdr) >> 20) & 0xffc) + +static int find_pci_ecap (int fd,int cap) +{ +uint32_t header =0 ; +int ttl = (PCI_CFG_SPACE_EXP_SIZE -PCI_CFG_SPACE_SIZE ) /8 ; +int pos =PCI_CFG_SPACE_SIZE ;/* start of extension capability area */ +int ret =0 ; + + ret =pread (fd, &header,sizeof (header), pos); +if (ret <0 ) { +opae_log_err ("Failed to read from PCI configuration space [e:%s]\n" , +strerror (errno)); +return ret; + } +opae_log_dbg ("Read 0x%08x from PCI configuration space 0x%x\n" , + header, pos); + +if (header ==0 ) { +opae_log_err ("Capability is empty\n" ); +return 0 ; + } + +while (ttl-- >0 ) { +if ((PCI_EXT_CAP_ID (header) == cap) && (pos !=0 )) +return pos; + + pos =PCI_EXT_CAP_NEXT (header); +if (pos <PCI_CFG_SPACE_SIZE ) { +opae_log_err ("Position of capability is invalid" +"[e:%d]\n" , pos); +break ; + } + ret =pread (fd, &header,sizeof (header), pos); +if (ret <0 ) { +opae_log_err ("Failed to read from PCI config space [e:%s]\n" , +strerror (errno)); +return ret; + } +opae_log_dbg ("Read 0x%08x from PCI configuration space 0x%x\n" , + header, pos); + } + +return 0 ; +} + +static int set_aer (pcidev_id id,uint32_t v1,uint32_t v2,int record) +{ +char path[PATH_MAX ] = {0 }; +uint32_t val =0 ; +int fd =-1 ; +int pos =0 ; +int ret =0 ; + +if (!id) +return -EINVAL ; + +snprintf (path,PATH_MAX ,"%s/%s/config" , +rte_pci_get_sysfs_path (), id->bdf); + fd =open (path,O_RDWR ); +if (fd <0 ) { +opae_log_err ("Failed to open \'%s\' for RDWR [e:%s]\n" , + path,strerror (errno)); +return -EIO ; + } + + pos =find_pci_ecap (fd,PCI_EXT_CAP_ID_ERR ); +if (pos <=0 ) { +opae_log_warn ("AER capability is not present\n" ); + ret = -ENXIO ; + goto close_fd; + } + +if (record) { + ret =pread (fd, &val,sizeof (val), pos +0x08 ); +if (ret <0 ) { +opae_log_err ("Failed to read from PCI config space [e:%s]\n" , +strerror (errno)); + goto close_fd; + } +opae_log_dbg ("Read 0x%08x from PCI configuration space 0x%x\n" , + val, pos +0x08 ); + dev_aer[0 ] = val; + + ret =pread (fd, &val,sizeof (val), pos +0x14 ); +if (ret <0 ) { +opae_log_err ("Failed to read from PCI config space [e:%s]\n" , +strerror (errno)); + goto close_fd; + } +opae_log_dbg ("Read 0x%08x from PCI configuration space 0x%x\n" , + val, pos +0x14 ); + dev_aer[1 ] = val; + } + +opae_log_dbg ("Write 0x%08x to PCI configuration space 0x%x\n" , + v1, pos +0x08 ); + ret =pwrite (fd, &v1,sizeof (v1), pos +0x08 ); +if (ret <0 ) { +opae_log_err ("Failed to write to PCI config space 0x%x [e:%s]\n" , + pos +0x08 ,strerror (errno)); + goto close_fd; + } + +opae_log_dbg ("Write 0x%08x to PCI configuration space 0x%x\n" , + v2, pos +0x14 ); + ret =pwrite (fd, &v2,sizeof (v2), pos +0x14 ); +if (ret <0 ) { +opae_log_err ("Failed to write to PCI config space 0x%x [e:%s]\n" , + pos +0x14 ,strerror (errno)); + } + +close_fd: +close (fd); +return ret <0 ? ret :0 ; +} + +static int enable_aer (pcidev_id id) +{ +if (!id) { +opae_log_err ("Input parameter of %s is invalid\n" , __func__); +return -EINVAL ; + } + +opae_log_info ("Enable AER of %s\n" , id->bdf); + +return set_aer (id, dev_aer[0 ], dev_aer[1 ],0 ); +} + +static int disable_aer (pcidev_id id) +{ +if (!id) { +opae_log_err ("Input parameter of %s is invalid\n" , __func__); +return -EINVAL ; + } + +opae_log_info ("Disable AER of %s\n" , id->bdf); + +return set_aer (id,0xffffffff ,0xffffffff ,1 ); +} + +static int reload (pcidev_id id,int type,int page) +{ +struct rte_rawdev *rdev =NULL ; +int ret =0 ; + + rdev =get_rte_rawdev (id,1 ); +if (rdev) + ret =ifpga_rawdev_reload (rdev, type, page); +else + ret = -ENODEV ; + +return ret; +} + +static int remove_tree (pcidev_id id) +{ +int i, n =0 ; +pcidev_id child; +int ret =0 ; + +if (!id) { +opae_log_err ("Input parameter of %s is invalid\n" , __func__); +return -EINVAL ; + } + + n =opae_get_child (id,NULL ,0 ); +if (n >0 ) { + child = (pcidev_id )rte_zmalloc (NULL , +sizeof (opae_pci_device) * n,0 ); +if (!child) { +opae_log_err ("Failed to malloc for children of %s\n" , + id->bdf); + ret = -ENOMEM ; + goto end; + } + +opae_get_child (id, child, n); +for (i =0 ; i < n; i++) +remove_tree (&child[i]); +opae_free (child); + } + +end: +opae_remove_device (id); +return ret; +} + +static int remove_device (pcidev_id id) +{ +char path[PATH_MAX ] = {0 }; +char one[] = {'1' ,0 }; +int ret =0 ; + +if (!id) { +opae_log_err ("Input parameter of %s is invalid\n" , __func__); +return -EINVAL ; + } +opae_log_info ("Remove %s from system\n" , id->bdf); + +snprintf (path,PATH_MAX ,"%s/%s/remove" , +rte_pci_get_sysfs_path (), id->bdf); + ret =write_file (path, one,strlen (one)); +if (ret <0 ) { +opae_log_err ("Failed to remove %s from system\n" , id->bdf); +return ret; + } + +remove_tree (id); + +return 0 ; +} + +static int scan_device (pcidev_id parent,pcidev_id id) +{ +char path[PATH_MAX ] = {0 }; +char bus[8 ] = {0 }; +char one[] = {'1' ,0 }; +char pwr[16 ] = {0 }; +char pwr_on[] = {'o' ,'n' ,0 }; +int pwr_on_failed =0 ; +int ret =0 ; + +if (!parent) { +opae_log_err ("Input parameter of %s is invalid\n" , __func__); +return -EINVAL ; + } +opae_log_info ("Rescan devices under %s\n" , parent->bdf); + +if (id) {/* scan specified bus under parent device */ +snprintf (path,PATH_MAX ,"%s/%s/power/control" , +rte_pci_get_sysfs_path (), parent->bdf); + ret =read_file (path, pwr,sizeof (pwr)); +if (ret <0 ) +return ret; + +if (strcmp (pwr,"on" )) { + ret =write_file (path, pwr_on,strlen (pwr_on)); +if (ret <0 ) + pwr_on_failed =1 ; +else +sleep (1 ); + } + +snprintf (bus,sizeof (bus),"%s" , id->bdf); +snprintf (path,PATH_MAX ,"%s/%s/pci_bus/%s/rescan" , +rte_pci_get_sysfs_path (), parent->bdf, bus); + ret =write_file (path, one,strlen (one)); +if (ret <0 ) +return ret; + +if (pwr_on_failed) {/* workaround for power on failed */ + ret =write_file (path, one,strlen (one)); +if (ret <0 ) +return ret; + } + +if (strcmp (pwr,"on" )) { +snprintf (path,PATH_MAX ,"%s/%s/power/control" , +rte_pci_get_sysfs_path (), parent->bdf); + ret =write_file (path, pwr,strlen (pwr)); + } + }else {/* scan all buses under parent device */ +snprintf (path,PATH_MAX ,"%s/%s/rescan" , +rte_pci_get_sysfs_path (), parent->bdf); + ret =write_file (path, one,strlen (one)); + } + +return ret; +} + +int opae_reboot_device (pcidev_id id,int type,int page) +{ +opae_pci_device fpga;/* FPGA after reboot */ +opae_pci_device parent; +opae_pci_device peer[2 ];/* physical function 1 of FPGA */ +opae_pci_device peer_parent; +opae_pci_device ups;/* upstream port device */ +opae_pci_device root;/* port connected to PAC */ +pcidev_id peer_primary =NULL ; +uint32_t rsu_stat =0 ; +char drv_name[OPAE_NAME_SIZE ] = {0 }; +int n =0 ; +int i =0 ; +int ret =0 ; + +if (!id) { +opae_log_err ("Input parameter of %s is invalid\n" , __func__); +return -EINVAL ; + } + +if (!is_pac (id)) { +opae_log_err ("%s can not be rebooted\n" , id->bdf); +return -EINVAL ; + } + + ret =opae_get_parent (id, &parent); +if (ret <0 ) +return -ENODEV ; + ret =opae_get_parent (&parent, &ups); +if (ret <0 ) +return -ENODEV ; + ret =opae_get_parent (&ups, &root); +if (ret <0 ) +return -ENODEV ; + + n =opae_get_pf1 (id, peer,sizeof (peer) /sizeof (opae_pci_device)); +if (n <=0 ) { +opae_log_err ("PF1 of %s is not found\n" , id->bdf); + }else { + peer_primary = &peer[0 ]; + ret =opae_get_parent (peer_primary, &peer_parent); +if (ret <0 ) +return -ENODEV ; + } + +get_driver (id, drv_name,sizeof (drv_name));/* save original driver */ + +if (!check_eal (1 )) +return -EPERM ; + +if (!get_rte_rawdev (id,1 )) +return -ENODEV ; + +lock (id); +opae_load_rsu_status (id, &rsu_stat,NULL ); +if (rsu_stat !=IFPGA_RSU_IDLE ) { +unlock (id); +if (rsu_stat ==IFPGA_RSU_REBOOT ) +opae_log_warn ("Reboot is in progress\n" ); +else +opae_log_warn ("Flash is in progress\n" ); +return -EAGAIN ; + } +opae_store_rsu_status (id,IFPGA_RSU_REBOOT ,0 ); +unlock (id); + +if (type ==IFPGA_BOOT_TYPE_FPGA ) { +/* disable AER */ + ret =disable_aer (&parent); +if (ret <0 ) { +opae_log_err ("Failed to disable AER of %s\n" , + parent.bdf); + goto reboot_end; + } + ret =disable_aer (&peer_parent); +if (ret <0 ) { +opae_log_err ("Failed to disable AER of %s\n" , + peer_parent.bdf); + goto reboot_end; + } +opae_store_rsu_status (id,IFPGA_RSU_REBOOT ,1 ); + +/* trigger reconfiguration */ + ret =reload (id, type, page); +opae_store_rsu_status (id,IFPGA_RSU_REBOOT ,2 ); +if (ret ==0 ) { + ret =remove_device (id); +for (i =0 ; i < n; i++) + ret +=remove_device (&peer[i]); +if (ret ==0 ) { +opae_log_info ("Wait 10 seconds for FPGA reloading\n" ); +sleep (10 ); + ret =scan_device (&parent, id); +if (ret <0 ) +opae_log_err ("Failed to rescan %s\n" , + id->bdf); +if (peer_primary) { + ret =scan_device (&peer_parent, + peer_primary); +if (ret <0 ) { +opae_log_err ("Failed to rescan %s\n" , + peer_primary->bdf); + } + } + } + } + +/* restore AER */ +if (enable_aer (&parent) <0 ) { +opae_log_err ("Failed to enable AER of %s\n" , + parent.bdf); + } +if (enable_aer (&peer_parent) <0 ) { +opae_log_err ("Failed to enable AER of %s\n" , + peer_parent.bdf); + } + }else if (type ==IFPGA_BOOT_TYPE_BMC ) { +/* disable AER */ + ret =disable_aer (&root); +if (ret <0 ) { +opae_log_err ("Failed to disable AER of %s\n" , root.bdf); + goto reboot_end; + } +opae_store_rsu_status (id,IFPGA_RSU_REBOOT ,1 ); + +/* trigger reconfiguration */ + ret =reload (id, type, page); +opae_store_rsu_status (id,IFPGA_RSU_REBOOT ,2 ); +if (ret ==0 ) { + ret +=remove_device (&ups); +if (ret ==0 ) { +opae_log_info ("Wait 10 seconds for BMC reloading\n" ); +sleep (10 ); + ret =scan_device (&root, &ups); +if (ret <0 ) +opae_log_err ("Failed to rescan %s\n" , + ups.bdf); + } + } + +/* restore AER */ +if (enable_aer (&root) <0 ) +opae_log_err ("Failed to enable AER of %s\n" , root.bdf); + }else { +opae_log_err ("Type of reboot is not supported [t:%d]\n" , type); + ret = -EINVAL ; + goto reboot_end; + } + +/* update id if bdf changed after reboot */ +if (opae_get_child (&parent, &fpga,1 ) ==1 ) { +if (strcmp (id->bdf, fpga.bdf)) + id = &fpga; + } + + ret =opae_bind_driver (id, drv_name); +if (ret <0 ) +opae_log_err ("Failed to bind original driver of %s\n" , id->bdf); + + ret =opae_probe_device (id); +if (ret <0 ) +opae_log_err ("Failed to probe %s [e:%d]\n" , id->bdf, ret); + +reboot_end: +opae_store_rsu_status (id,IFPGA_RSU_IDLE ,0 ); +return ret; +} + +int opae_partial_reconfigure (pcidev_id id,int port,const char *gbs) +{ +struct rte_rawdev *rdev =NULL ; + +if (!id || !gbs) { +opae_log_err ("Input parameter of %s is invalid\n" , __func__); +return -EINVAL ; + } + +if (!check_eal (1 )) +return -EPERM ; + + rdev =get_rte_rawdev (id,1 ); +if (!rdev) +return -ENODEV ; + +return ifpga_rawdev_partial_reconfigure (rdev, port, gbs); +} + +int opae_read_pci_cfg (pcidev_id id,uint32_t address,uint32_t *value) +{ +char path[PATH_MAX ] = {0 }; +int fd =-1 ; +int ret =0 ; + +if (!id || !value) { +opae_log_err ("Input parameter of %s is invalid\n" , __func__); +return -EINVAL ; + } + +snprintf (path,PATH_MAX ,"%s/%s/config" ,rte_pci_get_sysfs_path (), + id->bdf); + fd =open (path,O_RDONLY ); +if (fd <0 ) { +opae_log_dbg ("Failed to open \'%s\' for RDONLY [e:%s]\n" , + path,strerror (errno)); +return -EIO ; + } + + ret =pread (fd, value,4 , address); +if (ret <0 ) { +opae_log_err ("Failed to read from PCI device %s [e:%s]\n" , + id->bdf,strerror (errno)); +close (fd); +return ret; + } + +opae_log_dbg ("CONFIG+0x%08x -> 0x%08x\n" , address, *value); +close (fd); +return 0 ; +} + +int opae_write_pci_cfg (pcidev_id id,uint32_t address,uint32_t value) +{ +char path[PATH_MAX ] = {0 }; +int fd =-1 ; +int ret =0 ; + +if (!id) { +opae_log_err ("Input parameter of %s is invalid\n" , __func__); +return -EINVAL ; + } + +snprintf (path,PATH_MAX ,"%s/%s/config" ,rte_pci_get_sysfs_path (), + id->bdf); + fd =open (path,O_WRONLY ); +if (fd <0 ) { +opae_log_dbg ("Failed to open \'%s\' for WRONLY [e:%s]\n" , + path,strerror (errno)); +return -EIO ; + } + + ret =pwrite (fd, &value,4 , address); +if (ret <0 ) { +opae_log_err ("Failed to write to PCI device %s [e:%s]\n" , + id->bdf,strerror (errno)); +close (fd); +return ret; + } + +opae_log_dbg ("CONFIG+0x%08x <- 0x%08x\n" , address, value); +close (fd); +return 0 ; +}
drivers/raw/ifpga/ifpga_opae_api.h new file +248 -0
248 lines not on this page · raw message
drivers/raw/ifpga/ifpga_rawdev.c +146 -6
185 lines not on this page · raw message
drivers/raw/ifpga/ifpga_rawdev.h +15 -0
@@ -89,6 +89,12 @@ typedef struct {uint32_t fw_version; } ifpga_bmc_property; +typedef struct { +uint32_t num_retimers; +uint32_t link_speed; +uint32_t link_status; +}ifpga_phy_info ; + int ifpga_register_msix_irq(struct rte_rawdev *dev, int port_id, enum ifpga_irq_type type, int vec_start, int count,
@@ -98,15 +104,24 @@ int ifpga_unregister_msix_irq(enum ifpga_irq_type type, int vec_start, rte_intr_callback_fn handler, void *arg); +struct rte_pci_bus *ifpga_get_pci_bus (void ); +int ifpga_rawdev_lock (struct rte_rawdev *dev); +int ifpga_rawdev_unlock (struct rte_rawdev *dev); +uint32_t ifpga_rawdev_get_rsu_stat (struct rte_rawdev *dev); +void ifpga_rawdev_set_rsu_stat (struct rte_rawdev *dev,uint32_t value);int ifpga_rawdev_get_fme_property (struct rte_rawdev *dev,ifpga_fme_property *prop);int ifpga_rawdev_get_port_property (struct rte_rawdev *dev,uint32_t port,ifpga_port_property *prop);int ifpga_rawdev_get_bmc_property (struct rte_rawdev *dev,ifpga_bmc_property *prop); +int ifpga_rawdev_get_phy_info (struct rte_rawdev *dev,ifpga_phy_info *info);int ifpga_rawdev_update_flash (struct rte_rawdev *dev,const char *image,uint64_t *status);int ifpga_rawdev_stop_flash_update (struct rte_rawdev *dev,int force);int ifpga_rawdev_reload (struct rte_rawdev *dev,int type,int page); +int ifpga_rawdev_partial_reconfigure (struct rte_rawdev *dev,int port, +const char *file); +void ifpga_rawdev_cleanup (void );#endif /* _IFPGA_RAWDEV_H_ */
drivers/raw/ifpga/meson.build +3 -1
@@ -13,8 +13,10 @@ objs = [base_objs] deps += ['ethdev', 'rawdev', 'pci', 'bus_pci', 'kvargs', 'bus_vdev', 'bus_ifpga', 'net', 'net_i40e', 'net_ipn3ke'] -sources = files('ifpga_rawdev.c') +sources = files('ifpga_rawdev.c', 'ifpga_opae_api.c') includes += include_directories('base') includes += include_directories('../../net/ipn3ke') includes += include_directories('../../net/i40e') + +install_headers('ifpga_opae_api.h')
Signature · 1 line
2.29.2
Re: [dpdk-dev] [PATCH v11 3/4] raw/ifpga: add OPAE API for OpenStack Cyborg
Quoted 23 lines · Wei Huang
Cyborg is an OpenStack project that aims to provide a general purpose management framework for acceleration resources (i.e. various types of accelerators such as GPU, FPGA, NP, ODP, DPDK/SPDK and so on). It needs some OPAE type APIs to manage PACs (Programmable Acceleration Card) with Intel FPGA. Below major functions are added to meets Cyborg requirements.
1.opae_init()set up OPAE environment.
2.opae_cleanup()clean up OPAE environment.
3.opae_enumerate()searches PAC with specific FPGA.
4.opae_get_property()gets properties of FPGA.
5.opae_partial_reconfigure()perform partial configuration on FPGA.
6.opae_get_image_info()gets information of image file.
7.opae_update_flash()updates FPGA flash with specific image file.
8.opae_cancel_flash_update()cancel process of FPGA flash update.
9.opae_probe_device()manually probe specific FPGA with ifpga driver.
10.opae_remove_device()manually remove specific FPGA from ifpga driver.
11.opae_bind_driver()binds specific FPGA with specified kernel driver.
12.opae_unbind_driver()unbinds specific FPGA from kernel driver.
13.opae_reboot_device()reboots specific FPGA (do reconfiguration).Signed-off-by Wei Huang <[email protected]>Acked-by Tianfei Zhang <[email protected]>Acked-by Rosen Xu <[email protected]>
<...>
Quoted 10 lines
drivers/raw/ifpga/ifpga_opae_api.c new file +10
-0+ +RTE_INIT (init_api_env) +{ + eal_inited =0 ; + opae_log_level =OPAE_LOG_ERR ; + opae_log_file =NULL ; + ifpga_rawdev_logtype =0 ; + +opae_log_info ("API environment is initialized\n" ); +}
drivers/raw/ifpga/ifpga_opae_api.c line 58
Quoted 10 lines
drivers/raw/ifpga/ifpga_opae_api.c new file +10
-0+ +RTE_FINI (clean_api_env) +{ +if (opae_log_file) { +fclose (opae_log_file); + opae_log_file =NULL ; + } +opae_log_info ("API environment is cleaned\n" ); +} +
There are called in constructor and destructor, which means all DPDK applications have this PMD compiled will run above and print some log, is this really what we want here? Is there a way to limit the initialize to the PMD context, like 'probe()' etc?
Re: [dpdk-dev] [PATCH v11 3/4] raw/ifpga: add OPAE API for OpenStack Cyborg
Hi Ferruh,
These log information will not be output for the default log level is OPAE_LOG_ERR, they are output only when log level is set to OPAE_LOG_INFO. I can remove them if you think it's necessary.
All these opae_xxx functions will check the input PCI address is valid or not, and the rawdev PMD will also filter input PCI device ID. I think it's enough to prevent these function to be misused.
Thanks,
Wei
-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Friday, January 22, 2021 00:30
To: Huang, Wei <[email protected]>; [email protected]; Xu, Rosen <[email protected]>; Zhang, Qi Z <[email protected]>
Cc: [email protected]; Zhang, Tianfei <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v11 3/4] raw/ifpga: add OPAE API for OpenStack Cyborg
Quoted 23 lines · Wei Huang
Cyborg is an OpenStack project that aims to provide a general purpose management framework for acceleration resources (i.e. various types of accelerators such as GPU, FPGA, NP, ODP, DPDK/SPDK and so on). It needs some OPAE type APIs to manage PACs (Programmable Acceleration Card) with Intel FPGA. Below major functions are added to meets Cyborg requirements.
1.opae_init()set up OPAE environment.
2.opae_cleanup()clean up OPAE environment.
3.opae_enumerate()searches PAC with specific FPGA.
4.opae_get_property()gets properties of FPGA.
5.opae_partial_reconfigure()perform partial configuration on FPGA.
6.opae_get_image_info()gets information of image file.
7.opae_update_flash()updates FPGA flash with specific image file.
8.opae_cancel_flash_update()cancel process of FPGA flash update.
9.opae_probe_device()manually probe specific FPGA with ifpga driver.
10.opae_remove_device()manually remove specific FPGA from ifpga driver.
11.opae_bind_driver()binds specific FPGA with specified kernel driver.
12.opae_unbind_driver()unbinds specific FPGA from kernel driver.
13.opae_reboot_device()reboots specific FPGA (do reconfiguration).Signed-off-by Wei Huang <[email protected]>Acked-by Tianfei Zhang <[email protected]>Acked-by Rosen Xu <[email protected]>
<...>
Quoted 8 lines
drivers/raw/ifpga/ifpga_opae_api.c new file +8
-0+ +RTE_INIT (init_api_env) +{ + eal_inited =0 ; + opae_log_level =OPAE_LOG_ERR ; + opae_log_file =NULL ; + ifpga_rawdev_logtype =0 ; +
drivers/raw/ifpga/ifpga_opae_api.c line 58
Quoted 10 lines
drivers/raw/ifpga/ifpga_opae_api.c new file +10
-0+opae_log_info ("API environment is initialized\n" ); } + +RTE_FINI (clean_api_env) +{ +if (opae_log_file) { +fclose (opae_log_file); + opae_log_file =NULL ; + } +opae_log_info ("API environment is cleaned\n" ); } +
There are called in constructor and destructor, which means all DPDK applications have this PMD compiled will run above and print some log, is this really what we want here? Is there a way to limit the initialize to the PMD context, like 'probe()' etc?
Re: [dpdk-dev] [PATCH v11 3/4] raw/ifpga: add OPAE API for OpenStack Cyborg
On 1/22/2021 2:43 AM, Huang, Wei wrote:
<Please do not top post, your comment moved below>
Quoted 57 lines
Quoted 47 lines
-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Friday, January 22, 2021 00:30
To: Huang, Wei <[email protected]>; [email protected]; Xu, Rosen <[email protected]>; Zhang, Qi Z <[email protected]>
Cc: [email protected]; Zhang, Tianfei <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v11 3/4] raw/ifpga: add OPAE API for OpenStack CyborgQuoted 23 lines · Wei Huang
Cyborg is an OpenStack project that aims to provide a general purpose management framework for acceleration resources (i.e. various types of accelerators such as GPU, FPGA, NP, ODP, DPDK/SPDK and so on). It needs some OPAE type APIs to manage PACs (Programmable Acceleration Card) with Intel FPGA. Below major functions are added to meets Cyborg requirements.
1.opae_init()set up OPAE environment.
2.opae_cleanup()clean up OPAE environment.
3.opae_enumerate()searches PAC with specific FPGA.
4.opae_get_property()gets properties of FPGA.
5.opae_partial_reconfigure()perform partial configuration on FPGA.
6.opae_get_image_info()gets information of image file.
7.opae_update_flash()updates FPGA flash with specific image file.
8.opae_cancel_flash_update()cancel process of FPGA flash update.
9.opae_probe_device()manually probe specific FPGA with ifpga driver.
10.opae_remove_device()manually remove specific FPGA from ifpga driver.
11.opae_bind_driver()binds specific FPGA with specified kernel driver.
12.opae_unbind_driver()unbinds specific FPGA from kernel driver.
13.opae_reboot_device()reboots specific FPGA (do reconfiguration).Signed-off-by Wei Huang <[email protected]>Acked-by Tianfei Zhang <[email protected]>Acked-by Rosen Xu <[email protected]><...>
Quoted 13 lines
drivers/raw/ifpga/ifpga_opae_api.c new file +13
-0+ +RTE_INIT (init_api_env) +{ +eal_inited =0 ; +opae_log_level =OPAE_LOG_ERR ; +opae_log_file =NULL ; +ifpga_rawdev_logtype =0 ; + +opae_log_info ("API environment is initialized\n" ); } + +RTE_FINI (clean_api_env) +{ +if (opae_log_file) {Quoted 9 lines
drivers/raw/ifpga/ifpga_opae_api.c line 58
Quoted 5 lines
drivers/raw/ifpga/ifpga_opae_api.c new file +5
-0+fclose (opae_log_file); +opae_log_file =NULL ; +} +opae_log_info ("API environment is cleaned\n" ); } +There are called in constructor and destructor, which means all DPDK applications have this PMD compiled will run above and print some log, is this really what we want here? Is there a way to limit the initialize to the PMD context, like '
probe()' etc?
Quoted 4 lines
Hi Ferruh,
These log information will not be output for the default log level is
OPAE_LOG_ERR, they are output only when log level is set toOPAE_LOG_INFO. I can remove them if you think it's necessary.
Although they don't output because of default log level, I think it is wrong that they run for _all_ dpdk applications, I still think it should run for the scope of the driver, not for all dpdk applications.
Quoted 4 lines
All these
opae_xxxfunctions will check the input PCI address is valid or not, and the rawdev PMD will also filter input PCI device ID. I think it's enough to prevent these function to be misused.Thanks,
Wei
Re: [dpdk-dev] [PATCH v11 3/4] raw/ifpga: add OPAE API for OpenStack Cyborg
In reply to Wei HuangQuoted 11 lines · Wei Huang
Cyborg is an OpenStack project that aims to provide a general purpose management framework for acceleration resources (i.e. various types of accelerators such as GPU, FPGA, NP, ODP, DPDK/SPDK and so on). It needs some OPAE type APIs to manage PACs (Programmable Acceleration Card) with Intel FPGA. Below major functions are added to meets Cyborg requirements.
1.opae_init()set up OPAE environment.
2.opae_cleanup()clean up OPAE environment.
3.opae_enumerate()searches PAC with specific FPGA.
4.opae_get_property()gets properties of FPGA.
5.opae_partial_reconfigure()perform partial configuration on FPGA.
Quoted 9 lines
6.
opae_get_image_info()gets information of image file.
7.opae_update_flash()updates FPGA flash with specific image file.
8.opae_cancel_flash_update()cancel process of FPGA flash update.
9.opae_probe_device()manually probe specific FPGA with ifpga driver.
10.opae_remove_device()manually remove specific FPGA from ifpga driver.
11.opae_bind_driver()binds specific FPGA with specified kernel driver.
12.opae_unbind_driver()unbinds specific FPGA from kernel driver.
13.opae_reboot_device()reboots specific FPGA (do reconfiguration).
Hi Wei,
As far as I understand you are adding above public functions which are on top of raw/ifpga driver functions, so they are like PMD specific APIs, I think there are a few problems with it:
1) Do we really need/want this much PMD specific API? Can't we have them through the rawdev abstraction layer?
2) DPDK public APIs are part of API/ABI policy, so there are a few rules they have to follow, like:
- They should start with 'rte_' prefix, and the PMD specific APIs should start with 'rte_pmd_' prefix
- They should be in the .map file
- They should be experimental at least one release
- They should be fully documented in a doxygen format
- Header file should be added to index file for API documentation
Please don't update above before 1) is clearified and we are sure new APIs are required.
<...>
drivers/raw/ifpga/meson.build objs = [base_objs] · line 22
Quoted 13 lines
drivers/raw/ifpga/meson.build +3
-1@@ -13,8 +13,10 @@ objs = [base_objs] deps += ['ethdev', 'rawdev', 'pci', 'bus_pci', 'kvargs', 'bus_vdev', 'bus_ifpga', 'net', 'net_i40e', 'net_ipn3ke'] -sources = files('ifpga_rawdev.c') +sources = files('ifpga_rawdev.c', 'ifpga_opae_api.c') includes += include_directories('base') includes += include_directories('../../net/ipn3ke') includes += include_directories('../../net/i40e') + +install_headers('ifpga_opae_api.h')
There is a 'headers' helper that you can use for meson. Also the header file name should start with 'rte_pmd_'.
Even before this patch, isn't application has to include the rawdev PMD header? Why that header was not installed?
Re: [dpdk-dev] [PATCH v11 3/4] raw/ifpga: add OPAE API for OpenStack Cyborg
Hi Ferruh,
That's a good question.
The answer is YES, we do need all these functions. We have complete the integrated test with Cyborg, there is no redundant function. Let me show you a use case in Cyborg. Cyborg will update FPGA flash and reboot it to make it effective, so they will call functions like below.
1. opae_enumerate() to find the target FPGA
2. opae_get_property() to check FPGA version
3. opae_get_image_info() to check the update image
4. opae_update_flash() to update FPGA flash
5. opae_reboot_device() to reboot FPGA
6. After reboot, FPGA kernel driver will not be vfio-pci by default, opae_bind_driver() is used to bind vfio-pci kernel driver.
7. opae_probe_device() to attach ifpga PMD to FPGA, then this FPGA can be managed by Cyborg again.
These functions will be wrapped in Python package. Cyborg require this Python package can be downloaded from PyPI and compiled without DPDK installed.
So there is an independent project which will create a static library file from target DPDK. This library is part of Python package and will be used when compiling Python module. That's why I didn't export these function in map file. BTW, the header file ifpga_opae_api.h is also integrated into Python package from target DPDK.
Thanks,
Wei
-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Friday, January 22, 2021 00:30
To: Huang, Wei <[email protected]>; [email protected]; Xu, Rosen <[email protected]>; Zhang, Qi Z <[email protected]>
Cc: [email protected]; Zhang, Tianfei <[email protected]>; Ray Kinsella <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v11 3/4] raw/ifpga: add OPAE API for OpenStack Cyborg
Quoted 11 lines · Wei Huang
Cyborg is an OpenStack project that aims to provide a general purpose management framework for acceleration resources (i.e. various types of accelerators such as GPU, FPGA, NP, ODP, DPDK/SPDK and so on). It needs some OPAE type APIs to manage PACs (Programmable Acceleration Card) with Intel FPGA. Below major functions are added to meets Cyborg requirements.
1.opae_init()set up OPAE environment.
2.opae_cleanup()clean up OPAE environment.
3.opae_enumerate()searches PAC with specific FPGA.
4.opae_get_property()gets properties of FPGA.
5.opae_partial_reconfigure()perform partial configuration on FPGA.
Quoted 9 lines
6.
opae_get_image_info()gets information of image file.
7.opae_update_flash()updates FPGA flash with specific image file.
8.opae_cancel_flash_update()cancel process of FPGA flash update.
9.opae_probe_device()manually probe specific FPGA with ifpga driver.
10.opae_remove_device()manually remove specific FPGA from ifpga driver.
11.opae_bind_driver()binds specific FPGA with specified kernel driver.
12.opae_unbind_driver()unbinds specific FPGA from kernel driver.
13.opae_reboot_device()reboots specific FPGA (do reconfiguration).
Hi Wei,
As far as I understand you are adding above public functions which are on top of raw/ifpga driver functions, so they are like PMD specific APIs, I think there are a few problems with it:
1) Do we really need/want this much PMD specific API? Can't we have them through the rawdev abstraction layer?
2) DPDK public APIs are part of API/ABI policy, so there are a few rules they have to follow, like:
- They should start with 'rte_' prefix, and the PMD specific APIs should start with 'rte_pmd_' prefix
- They should be in the .map file
- They should be experimental at least one release
- They should be fully documented in a doxygen format
- Header file should be added to index file for API documentation
Please don't update above before 1) is clearified and we are sure new APIs are required.
<...>
drivers/raw/ifpga/meson.build objs = [base_objs] · line 22
Quoted 13 lines
drivers/raw/ifpga/meson.build +3
-1@@ -13,8 +13,10 @@ objs = [base_objs] deps += ['ethdev', 'rawdev', 'pci', 'bus_pci', 'kvargs', 'bus_vdev', 'bus_ifpga', 'net', 'net_i40e', 'net_ipn3ke'] -sources = files('ifpga_rawdev.c') +sources = files('ifpga_rawdev.c', 'ifpga_opae_api.c') includes += include_directories('base') includes += include_directories('../../net/ipn3ke') includes += include_directories('../../net/i40e') + +install_headers('ifpga_opae_api.h')
There is a 'headers' helper that you can use for meson. Also the header file name should start with 'rte_pmd_'.
Even before this patch, isn't application has to include the rawdev PMD header? Why that header was not installed?
Re: [dpdk-dev] [PATCH v11 3/4] raw/ifpga: add OPAE API for OpenStack Cyborg
In reply to Wei HuangQuoted 12 lines · Wei Huang
drivers/raw/ifpga/ifpga_opae_api.c new file +12
-0+int opae_init (int eal_init_result) +{ +int ret =0 ; + +if (!check_eal (0 )) +return 0 ; + +if (eal_init_result <0 ) { +if (rte_errno ==EALREADY ) { + eal_inited =1 ; +opae_log_info ("EAL already initialized\n" ); + }else {
drivers/raw/ifpga/ifpga_opae_api.c line 287
Quoted 10 lines
drivers/raw/ifpga/ifpga_opae_api.c new file +10
-0+opae_log_err ("Cannot initialize EAL\n" ); + ret =-1 ; + } + }else { + eal_inited =1 ; +opae_log_info ("Initialize EAL done\n" ); + } + +return ret; +}
Why an PMD API needs to know eal initialization status? This may be pointing a bad design, is it possible that most of the 'opae_' API code should go to the application?
Re: [dpdk-dev] [PATCH v11 3/4] raw/ifpga: add OPAE API for OpenStack Cyborg
Hi Ferruh,
Most of other APIs depends on EAL environment, they will make sure EAL is ready be executing their task.
As replied in previous mail, these functions will be called in Python environment. Python application should initialize EAL explicitly before calling other APIs like opae_update_flash. If application forget to do such initialization, an error information will be output when calling other APIs.
Thanks,
Wei
-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Friday, January 22, 2021 00:34
To: Huang, Wei <[email protected]>; [email protected]; Xu, Rosen <[email protected]>; Zhang, Qi Z <[email protected]>
Cc: [email protected]; Zhang, Tianfei <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v11 3/4] raw/ifpga: add OPAE API for OpenStack Cyborg
Quoted 12 lines · Wei Huang
drivers/raw/ifpga/ifpga_opae_api.c new file +12
-0+int opae_init (int eal_init_result) +{ +int ret =0 ; + +if (!check_eal (0 )) +return 0 ; + +if (eal_init_result <0 ) { +if (rte_errno ==EALREADY ) { + eal_inited =1 ; +opae_log_info ("EAL already initialized\n" ); + }else {
drivers/raw/ifpga/ifpga_opae_api.c line 287
Quoted 10 lines
drivers/raw/ifpga/ifpga_opae_api.c new file +10
-0+opae_log_err ("Cannot initialize EAL\n" ); + ret =-1 ; + } + }else { + eal_inited =1 ; +opae_log_info ("Initialize EAL done\n" ); + } + +return ret; +}
Why an PMD API needs to know eal initialization status? This may be pointing a bad design, is it possible that most of the 'opae_' API code should go to the application?
PATCH v11 4/4 examples/ifpga: add example for opae ifpga API
In reply to Wei HuangAn example application shows how to use opae ifpga APIs. You can test each API by running corresponding command. A guide is also added to show how to run the example.
---
v2: fix coding style issue in commands.c
---
v3: add guide for running example
---
v4: fix compilation issue of ifpga.rst
---
v5: add ifpga.rst into sample_app_ug/index.rst
--- MAINTAINERS | 1 + doc/guides/sample_app_ug/ifpga.rst | 435 +++++++++ doc/guides/sample_app_ug/index.rst | 1 + examples/ifpga/Makefile | 45 + examples/ifpga/commands.c | 1321 ++++++++++++++++++++++++++++ examples/ifpga/commands.h | 16 + examples/ifpga/main.c | 41 + examples/ifpga/meson.build | 19 + 8 files changed, 1879 insertions(+) create mode 100644 doc/guides/sample_app_ug/ifpga.rst create mode 100644 examples/ifpga/Makefile create mode 100644 examples/ifpga/commands.c create mode 100644 examples/ifpga/commands.h create mode 100644 examples/ifpga/main.c create mode 100644 examples/ifpga/meson.build
MAINTAINERS +1 -0
@@ -1237,6 +1237,7 @@ M: Tianfei zhang <[email protected]> T: git://dpdk.org/next/dpdk-next-net-intel F: drivers/raw/ifpga/ F: doc/guides/rawdevs/ifpga.rst +F: doc/guides/sample_app_ug/ifpga.rst IOAT Rawdev M: Bruce Richardson <[email protected]>
doc/guides/sample_app_ug/ifpga.rst new file +435 -0
435 lines not on this page · raw message
doc/guides/sample_app_ug/index.rst +1 -0
@@ -23,6 +23,7 @@ Sample Applications User Guides kernel_nic_interface keep_alive ioat + ifpga l2_forward_crypto l2_forward_job_stats l2_forward_real_virtual
examples/ifpga/Makefile new file +45 -0
45 lines not on this page · raw message
examples/ifpga/commands.c new file +1321 -0
1,321 lines not on this page · raw message
examples/ifpga/commands.h new file +16 -0
@@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020-2021 Intel Corporation +*/ + +#ifndef _COMMANDS_H_ +#define _COMMANDS_H_ + +#include "ifpga_opae_api.h" + +extern cmdline_parse_ctx_t main_ctx[]; + +typedef struct { +char s[38 ]; +}uuid_str ; + +#endif /* _COMMANDS_H_ */
examples/ifpga/main.c new file +41 -0
41 lines not on this page · raw message
examples/ifpga/meson.build new file +19 -0
@@ -0,0 +1,19 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2020-2021 Intel Corporation + +# meson file, for building this example as part of a main DPDK build. +# +# To build this example as a standalone application with an already-installed +# DPDK instance, use 'make' + +# require the raw_ifpga library +build = dpdk_conf.has('RTE_RAW_IFPGA') +if not build + subdir_done() +endif + +deps += 'raw_ifpga' +allow_experimental_apis = true +sources = files( + 'main.c', 'commands.c' +)
Signature · 1 line
2.29.2
Re: [dpdk-dev] [dpdk-stable] [PATCH v11 4/4] examples/ifpga: add example for opae ifpga API
Quoted 23 lines · Wei Huang
doc/guides/sample_app_ug/ifpga.rst new file +23
-0+get_property command +~~~~~~~~~~~~~~~~~~~~ + +Display property information of specified FPGA. Property type is defined as below. +0 - All properties +1 - PCI property +2 - FME property +4 - port property +8 - BMC property +PCI property is always available, other properties can only be displayed after +ifpga driver is probed to the FPGA. + +.. code-block:: console + + opae> get_property 24:00.0 0 + PCI: + PCIe s:b:d.f : 0000:24:00.0 + kernel driver : vfio-pci + FME: + platform : Vista Creek + DCP version : DCP 1.2 + phase : Beta + interface : 2x2x25G
doc/guides/sample_app_ug/ifpga.rst line 224
Quoted 10 lines
doc/guides/sample_app_ug/ifpga.rst new file +10
-0+ build version : 0.0.2 + ports num : 1 + boot page : user + pr interface id : a5d72a3c-c8b0-4939-912c-f715e5dc10ca + PORT0: + access type : PF + accelerator id : 8892c23e-2eed-4b44-8bb6-5c88606e07df + BMC: + MAX10 version : D.2.0.5 + NIOS FW version : D.2.0.12
Are all commands interacting device need to provide the PCI BDF as parameter to select the device? Like '24:00.0' in above sample.
Why not get the unique rawdev identifier, like port_id equivalent of the ethdev, and use it in the command line, won't it be easier?
Re: [dpdk-dev] [dpdk-stable] [PATCH v11 4/4] examples/ifpga: add example for opae ifpga API
Hi Ferruh,
Yes, all the command need PCI BDF as identifier of FPGA, because Cyborg use PCI BDF to call these APIs, Cyborg does not see any DPDK data structures. That's why I add a group of middle layer functions in ifpga_opae_api.c to isolate Cyborg from DPDK.
Thanks,
Wei
-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Friday, January 22, 2021 00:49
To: Huang, Wei <[email protected]>; [email protected]; Xu, Rosen <[email protected]>; Zhang, Qi Z <[email protected]>
Cc: [email protected]; Zhang, Tianfei <[email protected]>
Subject: Re: [dpdk-stable] [PATCH v11 4/4] examples/ifpga: add example for opae ifpga API
Quoted 23 lines · Wei Huang
doc/guides/sample_app_ug/ifpga.rst new file +23
-0+get_property command +~~~~~~~~~~~~~~~~~~~~ + +Display property information of specified FPGA. Property type is defined as below. +0 - All properties +1 - PCI property +2 - FME property +4 - port property +8 - BMC property +PCI property is always available, other properties can only be +displayed after ifpga driver is probed to the FPGA. + +.. code-block:: console + + opae> get_property 24:00.0 0 + PCI: + PCIe s:b:d.f : 0000:24:00.0 + kernel driver : vfio-pci + FME: + platform : Vista Creek + DCP version : DCP 1.2 + phase : Beta + interface : 2x2x25G
doc/guides/sample_app_ug/ifpga.rst line 224
Quoted 10 lines
doc/guides/sample_app_ug/ifpga.rst new file +10
-0+ build version : 0.0.2 + ports num : 1 + boot page : user + pr interface id : a5d72a3c-c8b0-4939-912c-f715e5dc10ca + PORT0: + access type : PF + accelerator id : 8892c23e-2eed-4b44-8bb6-5c88606e07df + BMC: + MAX10 version : D.2.0.5 + NIOS FW version : D.2.0.12
Are all commands interacting device need to provide the PCI BDF as parameter to select the device? Like '24:00.0' in above sample.
Why not get the unique rawdev identifier, like port_id equivalent of the ethdev, and use it in the command line, won't it be easier?
Zhang, Qi Z
AppliedQuoted 62 lines
-----Original Message-----
From: Huang, Wei <[email protected]>
Sent: Thursday, January 21, 2021 2:04 PM
To: [email protected]; Xu, Rosen <[email protected]>; Zhang, Qi Z <[email protected]>
Cc: [email protected]; Zhang, Tianfei <[email protected]>; Huang, Wei <[email protected]>
Subject: [PATCH v11 0/4] raw/ifpga: add extra OPAE APIsCyborg is part of OpenStack, it needs some OPAE APIs to manage devices with Intel FPGA. The first three patches implement extra APIs to meet Cyborg requirement. The last patch add an example to show how to use these APIs.
Main changes from v10:
- remove call ofrte_eal_init()andrte_eal_cleanup()
- renameopae_init_eal()toopae_init()
- renameopae_cleanup_eal()toopae_cleanup()Wei Huang (4):
raw/ifpga: add fpga rsu function
raw/ifpga: add fpga property get function
raw/ifpga: add OPAE API for OpenStack Cyborg
examples/ifpga: add example for opae ifpga APIMAINTAINERS | 1 + doc/guides/sample_app_ug/ifpga.rst | 435 +++++ doc/guides/sample_app_ug/index.rst | 1 + drivers/raw/ifpga/base/ifpga_api.c | 34 + drivers/raw/ifpga/base/ifpga_defines.h | 1 + drivers/raw/ifpga/base/ifpga_feature_dev.c | 21 + drivers/raw/ifpga/base/ifpga_feature_dev.h | 1 + drivers/raw/ifpga/base/ifpga_fme.c | 36 +- drivers/raw/ifpga/base/ifpga_fme_rsu.c | 435 +++++ drivers/raw/ifpga/base/ifpga_hw.h | 1 + drivers/raw/ifpga/base/ifpga_sec_mgr.c | 639 +++++++ drivers/raw/ifpga/base/ifpga_sec_mgr.h | 89 + drivers/raw/ifpga/base/meson.build | 2 + drivers/raw/ifpga/base/opae_hw_api.c | 77 + drivers/raw/ifpga/base/opae_hw_api.h | 13 + drivers/raw/ifpga/base/opae_ifpga_hw_api.h | 1 + drivers/raw/ifpga/base/opae_intel_max10.c | 48 + drivers/raw/ifpga/base/opae_intel_max10.h | 44 + drivers/raw/ifpga/ifpga_opae_api.c | 1793 ++++++++++++++++++++ drivers/raw/ifpga/ifpga_opae_api.h | 248 +++ drivers/raw/ifpga/ifpga_rawdev.c | 352 ++++ drivers/raw/ifpga/ifpga_rawdev.h | 50 +- drivers/raw/ifpga/meson.build | 4 +- examples/ifpga/Makefile | 45 + examples/ifpga/commands.c | 1321 ++++++++++++++ examples/ifpga/commands.h | 16 + examples/ifpga/main.c | 41 + examples/ifpga/meson.build | 19 + 28 files changed, 5764 insertions(+), 4 deletions(-) create mode 100644 doc/guides/sample_app_ug/ifpga.rst create mode 100644 drivers/raw/ifpga/base/ifpga_fme_rsu.c create mode 100644 drivers/raw/ifpga/base/ifpga_sec_mgr.c create mode 100644 drivers/raw/ifpga/base/ifpga_sec_mgr.h create mode 100644 drivers/raw/ifpga/ifpga_opae_api.c create mode 100644 drivers/raw/ifpga/ifpga_opae_api.h create mode 100644 examples/ifpga/Makefile create mode 100644
Quoted 6 lines
examples/ifpga/commands.ccreate mode 100644examples/ifpga/commands.hcreate mode 100644examples/ifpga/main.ccreate mode 100644 examples/ifpga/meson.build--
2.29.2
Applied to dpdk-next-net-intel.
Thanks
Qi