Skip to contentSkip to conversation

Modularization of DFL private feature drivers

PATCH Applied Xu Yilun 4 participants 19 messages Aug 10, 2020 – Aug 28, 2020

Subsystems (3)

v4 of 8 Applied Latest is v8 →

lore
Share & tools
lore

Xu Yilun

author maintainer
#0 lore raw

v4 cover Modularization of DFL private feature drivers

This patchset makes it possible to develop independent driver modules for DFL private features. It also helps to leverage existing kernel drivers to enable some IP blocks in DFL.

Patch #1: An improvement of feature id definition. The feature id will be
used as the key field for dfl device/driver matching.
Patch #2: Release the dfl mmio regions after enumeration, so that private
feature drivers could request mmio region in their own drivers.
Patch #3: Introduce the dfl bus, then dfl devices could be supported by
independent dfl drivers.
Patch #4: An example of the dfl driver for N3000 nios private feature.

Main changes from v1:
- Add the new Patch #1, to improve the feature id definition.
- Change the dfl bus uevent format.
- Change the dfl device's sysfs name format.
- refactor dfl_dev_add()
- Add the Patch #4 as an example of the dfl driver.
- A lot of minor fixes for comments from Hao and Tom.

Main changes from v2:
- Add the doc for dfl-n3000-nios driver.
- Minor fixes for comments from Tom.

Main changes from v3:
- improve the dfl devices' uevent format, 4 bits for type & 12 bits for id
- change dfl_device->type to u8
- A dedicate field in struct dfl_feature for dfl device instance.
- error out if dfl_device already exist on dfl_devs_init().
- Move the err log in regmap implementation, and delete n3000_nios_writel/readl(), they have nothing to wrapper now.
- Minor fixes and comments improvement.

Xu Yilun (4):
fpga: dfl: change data type of feature id to u16
fpga: dfl: map feature mmio resources in their own feature drivers
fpga: dfl: create a dfl bus type to support DFL devices
fpga: dfl: add support for N3000 Nios private feature

 Documentation/ABI/testing/sysfs-bus-dfl            |  15 +
 .../ABI/testing/sysfs-bus-dfl-devices-n3000-nios   |  18 +
 Documentation/fpga/dfl-n3000-nios.rst              |  41 ++
 Documentation/fpga/index.rst                       |   1 +
 drivers/fpga/Kconfig                               |  12 +
 drivers/fpga/Makefile                              |   2 +
 drivers/fpga/dfl-fme-perf.c                        |   2 +-
 drivers/fpga/dfl-n3000-nios.c                      | 475 +++++++++++++++++++++
 drivers/fpga/dfl-pci.c                             |  24 +-
 drivers/fpga/dfl.c                                 | 473 ++++++++++++++++----
 drivers/fpga/dfl.h                                 | 102 ++++-
 11 files changed, 1062 insertions(+), 103 deletions(-)
 create mode 100644 Documentation/ABI/testing/sysfs-bus-dfl
 create mode 100644 Documentation/ABI/testing/sysfs-bus-dfl-devices-n3000-nios
 create mode 100644 Documentation/fpga/dfl-n3000-nios.rst
 create mode 100644 drivers/fpga/dfl-n3000-nios.c
Signature · 1 line
2.7.4
author maintainer
#1 lore raw

PATCH v4 1/4 fpga: dfl: change data type of feature id to u16

The feature id is stored in a 12 bit field in DFH. So a u16 variable is enough for feature id.

This patch changes all feature id related places to fit u16.

---
v3: no change.
v4: no change.

---
 drivers/fpga/dfl-fme-perf.c |  2 +-
 drivers/fpga/dfl.c          | 29 +++++++++++++++--------------
 drivers/fpga/dfl.h          | 10 +++++-----
 3 files changed, 21 insertions(+), 20 deletions(-)
drivers/fpga/dfl-fme-perf.c +1 -1
index 6ce1ed2..5312662 100644
@@ -148,7 +148,7 @@ struct fme_perf_priv {
 	struct device *dev;
 	void __iomem *ioaddr;
 	struct pmu pmu;
-	u64 id;
+	u16 id;
 
 	u32 fab_users;
 	u32 fab_port_id;
drivers/fpga/dfl.c +15 -14
index 649958a..18575d9 100644
@@ -58,7 +58,7 @@ static const char *dfl_pdata_key_strings[DFL_ID_MAX] = {
  */
 struct dfl_dev_info {
 	const char *name;
-	u32 dfh_id;
+	u16 dfh_id;
 	struct idr id;
 	enum dfl_fpga_devt_type devt_type;
 };
@@ -134,7 +134,7 @@ static enum dfl_id_type feature_dev_id_type(struct platform_device *pdev)
 	return DFL_ID_MAX;
 }
 
-static enum dfl_id_type dfh_id_to_type(u32 id)
+static enum dfl_id_type dfh_id_to_type(u16 id)
 {
 	int i;
 
@@ -454,7 +454,7 @@ struct build_feature_devs_info {
  * @nr_irqs: number of irqs of this sub feature.
  */
 struct dfl_feature_info {
-	u64 fid;
+	u16 fid;
 	struct resource mmio_res;
 	void __iomem *ioaddr;
 	struct list_head node;
@@ -649,7 +649,7 @@ static inline u32 feature_size(void __iomem *start)
 	return ofst ? ofst : 4096;
 }
 
-static u64 feature_id(void __iomem *start)
+static u16 feature_id(void __iomem *start)
 {
 	u64 v = readq(start + DFH);
 	u16 id = FIELD_GET(DFH_ID, v);
@@ -667,7 +667,7 @@ static u64 feature_id(void __iomem *start)
 }
 
 static int parse_feature_irqs(struct build_feature_devs_info *binfo,
-			      resource_size_t ofst, u64 fid,
+			      resource_size_t ofst, u16 fid,
 			      unsigned int *irq_base, unsigned int *nr_irqs)
 {
 	void __iomem *base = binfo->ioaddr + ofst;
@@ -713,12 +713,12 @@ static int parse_feature_irqs(struct build_feature_devs_info *binfo,
 		return 0;
 	}
 
-	dev_dbg(binfo->dev, "feature: 0x%llx, irq_base: %u, nr_irqs: %u\n",
+	dev_dbg(binfo->dev, "feature: 0x%x, irq_base: %u, nr_irqs: %u\n",
 		fid, ibase, inr);
 
 	if (ibase + inr > binfo->nr_irqs) {
 		dev_err(binfo->dev,
-			"Invalid interrupt number in feature 0x%llx\n", fid);
+			"Invalid interrupt number in feature 0x%x\n", fid);
 		return -EINVAL;
 	}
 
@@ -726,7 +726,7 @@ static int parse_feature_irqs(struct build_feature_devs_info *binfo,
 		virq = binfo->irq_table[ibase + i];
 		if (virq < 0 || virq > NR_IRQS) {
 			dev_err(binfo->dev,
-				"Invalid irq table entry for feature 0x%llx\n",
+				"Invalid irq table entry for feature 0x%x\n",
 				fid);
 			return -EINVAL;
 		}
@@ -748,7 +748,7 @@ static int parse_feature_irqs(struct build_feature_devs_info *binfo,
 static int
 create_feature_instance(struct build_feature_devs_info *binfo,
 			struct dfl_fpga_enum_dfl *dfl, resource_size_t ofst,
-			resource_size_t size, u64 fid)
+			resource_size_t size, u16 fid)
 {
 	unsigned int irq_base, nr_irqs;
 	struct dfl_feature_info *finfo;
@@ -819,9 +819,10 @@ static int parse_feature_fiu(struct build_feature_devs_info *binfo,
 			     struct dfl_fpga_enum_dfl *dfl,
 			     resource_size_t ofst)
 {
-	u32 id, offset;
-	u64 v;
 	int ret = 0;
+	u32 offset;
+	u16 id;
+	u64 v;
 
 	v = readq(dfl->ioaddr + ofst + DFH);
 	id = FIELD_GET(DFH_ID, v);
@@ -855,8 +856,8 @@ static int parse_feature_private(struct build_feature_devs_info *binfo,
 				 resource_size_t ofst)
 {
 	if (!binfo->feature_dev) {
-		dev_err(binfo->dev, "the private feature %llx does not belong to any AFU.\n",
-			(unsigned long long)feature_id(dfl->ioaddr + ofst));
+		dev_err(binfo->dev, "the private feature 0x%x does not belong to any AFU.\n",
+			feature_id(dfl->ioaddr + ofst));
 		return -EINVAL;
 	}
 
@@ -1424,7 +1425,7 @@ static int do_set_irq_trigger(struct dfl_feature *feature, unsigned int idx,
 		return 0;
 
 	feature->irq_ctx[idx].name =
-		kasprintf(GFP_KERNEL, "fpga-irq[%u](%s-%llx)", idx,
+		kasprintf(GFP_KERNEL, "fpga-irq[%u](%s-%x)", idx,
 			  dev_name(&pdev->dev), feature->id);
 	if (!feature->irq_ctx[idx].name)
 		return -ENOMEM;
drivers/fpga/dfl.h +5 -5
index a32dfba..bc61942 100644
@@ -197,7 +197,7 @@ int dfl_fpga_check_port_id(struct platform_device *pdev, void *pport_id);
  * @id: unique dfl private feature id.
  */
 struct dfl_feature_id {
-	u64 id;
+	u16 id;
 };
 
 /**
@@ -240,7 +240,7 @@ struct dfl_feature_irq_ctx {
  */
 struct dfl_feature {
 	struct platform_device *dev;
-	u64 id;
+	u16 id;
 	int resource_index;
 	void __iomem *ioaddr;
 	struct dfl_feature_irq_ctx *irq_ctx;
@@ -365,7 +365,7 @@ struct platform_device *dfl_fpga_inode_to_feature_dev(struct inode *inode)
 	   (feature) < (pdata)->features + (pdata)->num; (feature)++)
 
 static inline
-struct dfl_feature *dfl_get_feature_by_id(struct device *dev, u64 id)
+struct dfl_feature *dfl_get_feature_by_id(struct device *dev, u16 id)
 {
 	struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
 	struct dfl_feature *feature;
@@ -378,7 +378,7 @@ struct dfl_feature *dfl_get_feature_by_id(struct device *dev, u64 id)
 }
 
 static inline
-void __iomem *dfl_get_feature_ioaddr_by_id(struct device *dev, u64 id)
+void __iomem *dfl_get_feature_ioaddr_by_id(struct device *dev, u16 id)
 {
 	struct dfl_feature *feature = dfl_get_feature_by_id(dev, id);
 
@@ -389,7 +389,7 @@ void __iomem *dfl_get_feature_ioaddr_by_id(struct device *dev, u64 id)
 	return NULL;
 }
 
-static inline bool is_dfl_feature_present(struct device *dev, u64 id)
+static inline bool is_dfl_feature_present(struct device *dev, u16 id)
 {
 	return !!dfl_get_feature_ioaddr_by_id(dev, id);
 }
Signature · 1 line
2.7.4
maintainer Applied
#2 lore raw

Re: [PATCH v4 1/4] fpga: dfl: change data type of feature id to u16

Quoted 197 lines · Xu Yilun

The feature id is stored in a 12 bit field in DFH. So a u16 variable is enough for feature id.

This patch changes all feature id related places to fit u16.

---
v3: no change.
v4: no change.

---
 drivers/fpga/dfl-fme-perf.c |  2 +-
 drivers/fpga/dfl.c          | 29 +++++++++++++++--------------
 drivers/fpga/dfl.h          | 10 +++++-----
 3 files changed, 21 insertions(+), 20 deletions(-)
drivers/fpga/dfl-fme-perf.c +1 -1
index 6ce1ed2..5312662 100644
@@ -148,7 +148,7 @@ struct fme_perf_priv {
 	struct device *dev;
 	void __iomem *ioaddr;
 	struct pmu pmu;
-	u64 id;
+	u16 id;
 
 	u32 fab_users;
 	u32 fab_port_id;
drivers/fpga/dfl.c +15 -14
index 649958a..18575d9 100644
@@ -58,7 +58,7 @@ static const char *dfl_pdata_key_strings[DFL_ID_MAX] = {
  */
 struct dfl_dev_info {
 	const char *name;
-	u32 dfh_id;
+	u16 dfh_id;
 	struct idr id;
 	enum dfl_fpga_devt_type devt_type;
 };
@@ -134,7 +134,7 @@ static enum dfl_id_type feature_dev_id_type(struct platform_device *pdev)
 	return DFL_ID_MAX;
 }
 
-static enum dfl_id_type dfh_id_to_type(u32 id)
+static enum dfl_id_type dfh_id_to_type(u16 id)
 {
 	int i;
 
@@ -454,7 +454,7 @@ struct build_feature_devs_info {
  * @nr_irqs: number of irqs of this sub feature.
  */
 struct dfl_feature_info {
-	u64 fid;
+	u16 fid;
 	struct resource mmio_res;
 	void __iomem *ioaddr;
 	struct list_head node;
@@ -649,7 +649,7 @@ static inline u32 feature_size(void __iomem *start)
 	return ofst ? ofst : 4096;
 }
 
-static u64 feature_id(void __iomem *start)
+static u16 feature_id(void __iomem *start)
 {
 	u64 v = readq(start + DFH);
 	u16 id = FIELD_GET(DFH_ID, v);
@@ -667,7 +667,7 @@ static u64 feature_id(void __iomem *start)
 }
 
 static int parse_feature_irqs(struct build_feature_devs_info *binfo,
-			      resource_size_t ofst, u64 fid,
+			      resource_size_t ofst, u16 fid,
 			      unsigned int *irq_base, unsigned int *nr_irqs)
 {
 	void __iomem *base = binfo->ioaddr + ofst;
@@ -713,12 +713,12 @@ static int parse_feature_irqs(struct build_feature_devs_info *binfo,
 		return 0;
 	}
 
-	dev_dbg(binfo->dev, "feature: 0x%llx, irq_base: %u, nr_irqs: %u\n",
+	dev_dbg(binfo->dev, "feature: 0x%x, irq_base: %u, nr_irqs: %u\n",
 		fid, ibase, inr);
 
 	if (ibase + inr > binfo->nr_irqs) {
 		dev_err(binfo->dev,
-			"Invalid interrupt number in feature 0x%llx\n", fid);
+			"Invalid interrupt number in feature 0x%x\n", fid);
 		return -EINVAL;
 	}
 
@@ -726,7 +726,7 @@ static int parse_feature_irqs(struct build_feature_devs_info *binfo,
 		virq = binfo->irq_table[ibase + i];
 		if (virq < 0 || virq > NR_IRQS) {
 			dev_err(binfo->dev,
-				"Invalid irq table entry for feature 0x%llx\n",
+				"Invalid irq table entry for feature 0x%x\n",
 				fid);
 			return -EINVAL;
 		}
@@ -748,7 +748,7 @@ static int parse_feature_irqs(struct build_feature_devs_info *binfo,
 static int
 create_feature_instance(struct build_feature_devs_info *binfo,
 			struct dfl_fpga_enum_dfl *dfl, resource_size_t ofst,
-			resource_size_t size, u64 fid)
+			resource_size_t size, u16 fid)
 {
 	unsigned int irq_base, nr_irqs;
 	struct dfl_feature_info *finfo;
@@ -819,9 +819,10 @@ static int parse_feature_fiu(struct build_feature_devs_info *binfo,
 			     struct dfl_fpga_enum_dfl *dfl,
 			     resource_size_t ofst)
 {
-	u32 id, offset;
-	u64 v;
 	int ret = 0;
+	u32 offset;
+	u16 id;
+	u64 v;
 
 	v = readq(dfl->ioaddr + ofst + DFH);
 	id = FIELD_GET(DFH_ID, v);
@@ -855,8 +856,8 @@ static int parse_feature_private(struct build_feature_devs_info *binfo,
 				 resource_size_t ofst)
 {
 	if (!binfo->feature_dev) {
-		dev_err(binfo->dev, "the private feature %llx does not belong to any AFU.\n",
-			(unsigned long long)feature_id(dfl->ioaddr + ofst));
+		dev_err(binfo->dev, "the private feature 0x%x does not belong to any AFU.\n",
+			feature_id(dfl->ioaddr + ofst));
 		return -EINVAL;
 	}
 
@@ -1424,7 +1425,7 @@ static int do_set_irq_trigger(struct dfl_feature *feature, unsigned int idx,
 		return 0;
 
 	feature->irq_ctx[idx].name =
-		kasprintf(GFP_KERNEL, "fpga-irq[%u](%s-%llx)", idx,
+		kasprintf(GFP_KERNEL, "fpga-irq[%u](%s-%x)", idx,
 			  dev_name(&pdev->dev), feature->id);
 	if (!feature->irq_ctx[idx].name)
 		return -ENOMEM;
drivers/fpga/dfl.h +5 -5
index a32dfba..bc61942 100644
@@ -197,7 +197,7 @@ int dfl_fpga_check_port_id(struct platform_device *pdev, void *pport_id);
  * @id: unique dfl private feature id.
  */
 struct dfl_feature_id {
-	u64 id;
+	u16 id;
 };
 
 /**
@@ -240,7 +240,7 @@ struct dfl_feature_irq_ctx {
  */
 struct dfl_feature {
 	struct platform_device *dev;
-	u64 id;
+	u16 id;
 	int resource_index;
 	void __iomem *ioaddr;
 	struct dfl_feature_irq_ctx *irq_ctx;
@@ -365,7 +365,7 @@ struct platform_device *dfl_fpga_inode_to_feature_dev(struct inode *inode)
 	   (feature) < (pdata)->features + (pdata)->num; (feature)++)
 
 static inline
-struct dfl_feature *dfl_get_feature_by_id(struct device *dev, u64 id)
+struct dfl_feature *dfl_get_feature_by_id(struct device *dev, u16 id)
 {
 	struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
 	struct dfl_feature *feature;
@@ -378,7 +378,7 @@ struct dfl_feature *dfl_get_feature_by_id(struct device *dev, u64 id)
 }
 
 static inline
-void __iomem *dfl_get_feature_ioaddr_by_id(struct device *dev, u64 id)
+void __iomem *dfl_get_feature_ioaddr_by_id(struct device *dev, u16 id)
 {
 	struct dfl_feature *feature = dfl_get_feature_by_id(dev, id);
 
@@ -389,7 +389,7 @@ void __iomem *dfl_get_feature_ioaddr_by_id(struct device *dev, u64 id)
 	return NULL;
 }
 
-static inline bool is_dfl_feature_present(struct device *dev, u64 id)
+static inline bool is_dfl_feature_present(struct device *dev, u16 id)
 {
 	return !!dfl_get_feature_ioaddr_by_id(dev, id);
 }
Signature · 1 line
2.7.4

Applied to for-next,

Thanks

#3 lore raw

RE: [PATCH v4 1/4] fpga: dfl: change data type of feature id to u16

From: Moritz Fischer

Quoted 7 lines

Sent: 12 August 2020 04:56

Quoted 4 lines · Xu Yilun

The feature id is stored in a 12 bit field in DFH. So a u16 variable is enough for feature id.

This patch changes all feature id related places to fit u16.

How much bigger does it make the kernel?

David

-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)

author maintainer
#4 lore raw

Re: [PATCH v4 1/4] fpga: dfl: change data type of feature id to u16

Quoted 10 lines · David Laight

From: Moritz Fischer

Quoted 7 lines

Sent: 12 August 2020 04:56

Quoted 4 lines · Xu Yilun

The feature id is stored in a 12 bit field in DFH. So a u16 variable is enough for feature id.

This patch changes all feature id related places to fit u16.

How much bigger does it make the kernel?

The patch changes the definition of feature id from u64 to u16, and will make the kernel slightly smaller.

Actually it is not the main purpose of this change. In the following patches we will put the feature_id definiton in mod_devicetable.h in order to support MODULE_DEVICE_TABLE(). The modpost will also use the definitions in mod_devicetable.h but seems it doesn't support u64 now. So we reduced the unnecessary bits definition in feature_id to fit modpost.

Thanks,
Yilun

Quoted 6 lines

David

-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)

#5 lore raw

RE: [PATCH v4 1/4] fpga: dfl: change data type of feature id to u16

From: Xu Yilun

Quoted 9 lines

Sent: 13 August 2020 08:59

Quoted 7 lines · David Laight

From: Moritz Fischer

Quoted 6 lines

Sent: 12 August 2020 04:56

Quoted 3 lines · Xu Yilun

The feature id is stored in a 12 bit field in DFH. So a u16 variable is enough for feature id.

Quoted 6 lines
Quoted 3 lines
Quoted 1 line
Quoted 1 line

This patch changes all feature id related places to fit u16.

How much bigger does it make the kernel?

The patch changes the definition of feature id from u64 to u16, and will make the kernel slightly smaller.

Unlikely.
Most of the structures will gain a 'pad' field. Using u16 for function parameters and results almost certainly requires instructions to mask the value.
Any arithmetic on u16 will require masking instructions on (probably) all architectures except x86.

Using 'unsigned int' is probably best.

u16 is never a good idea unless you are defining enough of them in a structure (eg as an array) to reduce the structure size below some threshold.
(Or are matching some hardware layout.)

David

-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)

author maintainer
#6 lore raw

Re: [PATCH v4 1/4] fpga: dfl: change data type of feature id to u16

Quoted 24 lines · David Laight

From: Xu Yilun

Quoted 9 lines

Sent: 13 August 2020 08:59

Quoted 7 lines · David Laight

From: Moritz Fischer

Quoted 6 lines

Sent: 12 August 2020 04:56

Quoted 3 lines · Xu Yilun

The feature id is stored in a 12 bit field in DFH. So a u16 variable is enough for feature id.

Quoted 6 lines
Quoted 3 lines
Quoted 1 line
Quoted 1 line

This patch changes all feature id related places to fit u16.

How much bigger does it make the kernel?

The patch changes the definition of feature id from u64 to u16, and will make the kernel slightly smaller.

Unlikely.
Most of the structures will gain a 'pad' field. Using u16 for function parameters and results almost certainly requires instructions to mask the value.
Any arithmetic on u16 will require masking instructions on (probably) all architectures except x86.

Quoted 6 lines

Using 'unsigned int' is probably best.

u16 is never a good idea unless you are defining enough of them in a structure (eg as an array) to reduce the structure size below some threshold.
(Or are matching some hardware layout.)

I got it. Thanks for your detailed explanation. I think we may change them to u32. Is it the same case for u8? Think we may also change the dfl_device_id.type.

Hi Moritz:

The patch is applied to for-next, is it possible we recall it, or we make another fix after it?

Thanks,
Yilun.

Quoted 6 lines

David

-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)

#7 lore raw

RE: [PATCH v4 1/4] fpga: dfl: change data type of feature id to u16

From: Xu Yilun <[email protected]>

Quoted 29 lines

Sent: 13 August 2020 10:04

Quoted 26 lines · David Laight

From: Xu Yilun

Quoted 9 lines

Sent: 13 August 2020 08:59

Quoted 7 lines · David Laight

From: Moritz Fischer

Quoted 6 lines

Sent: 12 August 2020 04:56

Quoted 3 lines · Xu Yilun

The feature id is stored in a 12 bit field in DFH. So a u16 variable is enough for feature id.

Quoted 6 lines
Quoted 3 lines
Quoted 1 line
Quoted 1 line

This patch changes all feature id related places to fit u16.

How much bigger does it make the kernel?

The patch changes the definition of feature id from u64 to u16, and will make the kernel slightly smaller.

Unlikely.
Most of the structures will gain a 'pad' field. Using u16 for function parameters and results almost certainly requires instructions to mask the value.
Any arithmetic on u16 will require masking instructions on (probably) all architectures except x86.

Using 'unsigned int' is probably best.

Quoted 7 lines
Quoted 4 lines

u16 is never a good idea unless you are defining enough of them in a structure (eg as an array) to reduce the structure size below some threshold.
(Or are matching some hardware layout.)

I got it. Thanks for your detailed explanation. I think we may change them to u32. Is it the same case for u8? Think we may also change the dfl_device_id.type.

Loosely 'yes' but it isn't worth the churn of 'random' changes. And they aren't often passed to/from functions - which I'm 98% sure requires masking.

I commented because the compiler was going to add pad fields after your u16 values - so you'd get do space saving and probably more code.

David

-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)

maintainer
#8 lore raw

Re: [PATCH v4 1/4] fpga: dfl: change data type of feature id to u16

In reply to Xu Yilun
Quoted 36 lines · Xu Yilun
Quoted 24 lines · David Laight

From: Xu Yilun

Quoted 9 lines

Sent: 13 August 2020 08:59

Quoted 7 lines · David Laight

From: Moritz Fischer

Quoted 6 lines

Sent: 12 August 2020 04:56

Quoted 3 lines · Xu Yilun

The feature id is stored in a 12 bit field in DFH. So a u16 variable is enough for feature id.

Quoted 6 lines
Quoted 3 lines
Quoted 1 line
Quoted 1 line

This patch changes all feature id related places to fit u16.

How much bigger does it make the kernel?

The patch changes the definition of feature id from u64 to u16, and will make the kernel slightly smaller.

Unlikely.
Most of the structures will gain a 'pad' field. Using u16 for function parameters and results almost certainly requires instructions to mask the value.
Any arithmetic on u16 will require masking instructions on (probably) all architectures except x86.

Quoted 6 lines

Using 'unsigned int' is probably best.

u16 is never a good idea unless you are defining enough of them in a structure (eg as an array) to reduce the structure size below some threshold.
(Or are matching some hardware layout.)

I got it. Thanks for your detailed explanation. I think we may change them to u32. Is it the same case for u8? Think we may also change the dfl_device_id.type.

Quoted 7 lines

Hi Moritz:

The patch is applied to for-next, is it possible we recall it, or we make another fix after it?

Thanks,
Yilun.

Sorry for the delay, can you send a follow-up please?

Cheers,
Moritz

author maintainer
#9 lore raw

Re: [PATCH v4 1/4] fpga: dfl: change data type of feature id to u16

Quoted 39 lines · Moritz Fischer
Quoted 38 lines · Xu Yilun
Quoted 24 lines · David Laight

From: Xu Yilun

Quoted 9 lines

Sent: 13 August 2020 08:59

Quoted 7 lines · David Laight

From: Moritz Fischer

Quoted 6 lines

Sent: 12 August 2020 04:56

Quoted 3 lines · Xu Yilun

The feature id is stored in a 12 bit field in DFH. So a u16 variable is enough for feature id.

Quoted 6 lines
Quoted 3 lines
Quoted 1 line
Quoted 1 line

This patch changes all feature id related places to fit u16.

How much bigger does it make the kernel?

The patch changes the definition of feature id from u64 to u16, and will make the kernel slightly smaller.

Unlikely.
Most of the structures will gain a 'pad' field. Using u16 for function parameters and results almost certainly requires instructions to mask the value.
Any arithmetic on u16 will require masking instructions on (probably) all architectures except x86.

Quoted 6 lines

Using 'unsigned int' is probably best.

u16 is never a good idea unless you are defining enough of them in a structure (eg as an array) to reduce the structure size below some threshold.
(Or are matching some hardware layout.)

I got it. Thanks for your detailed explanation. I think we may change them to u32. Is it the same case for u8? Think we may also change the dfl_device_id.type.

Hi Moritz:

Quoted 7 lines
Quoted 5 lines

The patch is applied to for-next, is it possible we recall it, or we make another fix after it?

Thanks,
Yilun.

Sorry for the delay, can you send a follow-up please?

Hi moritz:

I think I don't have to change it now. As discussed with David, these fields aren't often accessed. So it should be OK.

Thanks,
Yilun.

Quoted 3 lines

Cheers,
Moritz

author maintainer
#10 lore raw

PATCH v4 2/4 fpga: dfl: map feature mmio resources in their own feature drivers

In reply to Xu Yilun

This patch makes preparation for modularization of DFL sub feature drivers.

Currently, if we need to support a new DFL sub feature, an entry should be added to fme/port_feature_drvs[] in dfl-fme/port-main.c. And we need to re-compile the whole DFL modules. That make the DFL drivers hard to be extended.

Another consideration is that DFL may contain some IP blocks which are already supported by kernel, most of them are supported by platform device drivers. We could create platform devices for these IP blocks and get them supported by these drivers.

An important issue is that platform device drivers usually requests mmio resources on probe. But now dfl mmio is mapped in dfl bus driver (e.g. dfl-pci) as a whole region. Then platform device drivers for sub features can't request their own mmio resources again. This is what the patch trying to resolve.

This patch changes the DFL enumeration. DFL bus driver will unmap mmio resources after first step enumeration and pass enumeration info to DFL framework. Then DFL framework will map the mmio resources again, do 2nd step enumeration, and also unmap the mmio resources. In this way, sub feature drivers could then request their own mmio resources as needed.

An exception is that mmio resource of FIU headers are still mapped in dfl bus driver. The FIU headers have some fundamental functions (sriov set, port enable/disable) needed for dfl bus devices and other sub features. They should not be unmapped as long as dfl bus device is alive.

---
v2: delete dfl_binfo_shift() cause no shift is possible for FIU parsing.
Only map bar0 for first phase enumeration in dfl-pci, we can find
all dfl mmio resource info in bar0.
Some minor fixes for comments from Hao & Tom.
v3: no change
v4: minor fixes for Hao's comments.

---
 drivers/fpga/dfl-pci.c |  24 +++----
 drivers/fpga/dfl.c     | 185 +++++++++++++++++++++++++++++++++----------------
 drivers/fpga/dfl.h     |   7 +-
 3 files changed, 139 insertions(+), 77 deletions(-)
drivers/fpga/dfl-pci.c +11 -13
index e220bec..a2203d0 100644
@@ -31,12 +31,12 @@ struct cci_drvdata {
 	struct dfl_fpga_cdev *cdev;	/* container device */
 };
 
-static void __iomem *cci_pci_ioremap_bar(struct pci_dev *pcidev, int bar)
+static void __iomem *cci_pci_ioremap_bar0(struct pci_dev *pcidev)
 {
-	if (pcim_iomap_regions(pcidev, BIT(bar), DRV_NAME))
+	if (pcim_iomap_regions(pcidev, BIT(0), DRV_NAME))
 		return NULL;
 
-	return pcim_iomap_table(pcidev)[bar];
+	return pcim_iomap_table(pcidev)[0];
 }
 
 static int cci_pci_alloc_irq(struct pci_dev *pcidev)
@@ -156,8 +156,8 @@ static int cci_enumerate_feature_devs(struct pci_dev *pcidev)
 			goto irq_free_exit;
 	}
 
-	/* start to find Device Feature List from Bar 0 */
-	base = cci_pci_ioremap_bar(pcidev, 0);
+	/* start to find Device Feature List in Bar 0 */
+	base = cci_pci_ioremap_bar0(pcidev);
 	if (!base) {
 		ret = -ENOMEM;
 		goto irq_free_exit;
@@ -172,7 +172,7 @@ static int cci_enumerate_feature_devs(struct pci_dev *pcidev)
 		start = pci_resource_start(pcidev, 0);
 		len = pci_resource_len(pcidev, 0);
 
-		dfl_fpga_enum_info_add_dfl(info, start, len, base);
+		dfl_fpga_enum_info_add_dfl(info, start, len);
 
 		/*
 		 * find more Device Feature Lists (e.g. Ports) per information
@@ -196,26 +196,24 @@ static int cci_enumerate_feature_devs(struct pci_dev *pcidev)
 			 */
 			bar = FIELD_GET(FME_PORT_OFST_BAR_ID, v);
 			offset = FIELD_GET(FME_PORT_OFST_DFH_OFST, v);
-			base = cci_pci_ioremap_bar(pcidev, bar);
-			if (!base)
-				continue;
-
 			start = pci_resource_start(pcidev, bar) + offset;
 			len = pci_resource_len(pcidev, bar) - offset;
 
-			dfl_fpga_enum_info_add_dfl(info, start, len,
-						   base + offset);
+			dfl_fpga_enum_info_add_dfl(info, start, len);
 		}
 	} else if (dfl_feature_is_port(base)) {
 		start = pci_resource_start(pcidev, 0);
 		len = pci_resource_len(pcidev, 0);
 
-		dfl_fpga_enum_info_add_dfl(info, start, len, base);
+		dfl_fpga_enum_info_add_dfl(info, start, len);
 	} else {
 		ret = -ENODEV;
 		goto irq_free_exit;
 	}
 
+	/* release I/O mappings for next step enumeration */
+	pcim_iounmap_regions(pcidev, BIT(0));
+
 	/* start enumeration with prepared enumeration information */
 	cdev = dfl_fpga_feature_devs_enumerate(info);
 	if (IS_ERR(cdev)) {
drivers/fpga/dfl.c +127 -58

401 lines not on this page · raw message

drivers/fpga/dfl.h +1 -6
index bc61942..5973769 100644
@@ -441,22 +441,17 @@ struct dfl_fpga_enum_info {
  *
  * @start: base address of this device feature list.
  * @len: size of this device feature list.
- * @ioaddr: mapped base address of this device feature list.
  * @node: node in list of device feature lists.
  */
 struct dfl_fpga_enum_dfl {
 	resource_size_t start;
 	resource_size_t len;
-
-	void __iomem *ioaddr;
-
 	struct list_head node;
 };
 
 struct dfl_fpga_enum_info *dfl_fpga_enum_info_alloc(struct device *dev);
 int dfl_fpga_enum_info_add_dfl(struct dfl_fpga_enum_info *info,
-			       resource_size_t start, resource_size_t len,
-			       void __iomem *ioaddr);
+			       resource_size_t start, resource_size_t len);
 int dfl_fpga_enum_info_add_irq(struct dfl_fpga_enum_info *info,
 			       unsigned int nr_irqs, int *irq_table);
 void dfl_fpga_enum_info_free(struct dfl_fpga_enum_info *info);
Signature · 1 line
2.7.4
#11 lore raw

RE: [PATCH v4 2/4] fpga: dfl: map feature mmio resources in their own feature drivers

Quoted 13 lines

-----Original Message-----
From: [email protected] <[email protected]> On Behalf Of Xu Yilun
Sent: Monday, August 10, 2020 10:41 AM
To: [email protected]; [email protected]; linux- [email protected]
Cc: [email protected]; [email protected]; Xu, Yilun <[email protected]>; Wu, Hao <[email protected]>; Matthew Gerlach <[email protected]>; Weight, Russell H <[email protected]>
Subject: [PATCH v4 2/4] fpga: dfl: map feature mmio resources in their own feature drivers

Quoted 5 lines

This patch makes preparation for modularization of DFL sub feature drivers.

Currently, if we need to support a new DFL sub feature, an entry should be added to fme/port_feature_drvs[] in dfl-fme/port-main.c. And we need

I think it's afu-main.c

Quoted 2 lines

to re-compile the whole DFL modules. That make the DFL drivers hard to be extended.

And maybe we can drop this description or move it to cover letter as it's not the problem this patch is going to resolve.

Quoted 203 lines

Another consideration is that DFL may contain some IP blocks which are already supported by kernel, most of them are supported by platform device drivers. We could create platform devices for these IP blocks and get them supported by these drivers.

An important issue is that platform device drivers usually requests mmio resources on probe. But now dfl mmio is mapped in dfl bus driver (e.g. dfl-pci) as a whole region. Then platform device drivers for sub features can't request their own mmio resources again. This is what the patch trying to resolve.

This patch changes the DFL enumeration. DFL bus driver will unmap mmio resources after first step enumeration and pass enumeration info to DFL framework. Then DFL framework will map the mmio resources again, do 2nd step enumeration, and also unmap the mmio resources. In this way, sub feature drivers could then request their own mmio resources as needed.

An exception is that mmio resource of FIU headers are still mapped in dfl bus driver. The FIU headers have some fundamental functions (sriov set, port enable/disable) needed for dfl bus devices and other sub features. They should not be unmapped as long as dfl bus device is alive.

---
v2: delete dfl_binfo_shift() cause no shift is possible for FIU parsing.
Only map bar0 for first phase enumeration in dfl-pci, we can find
all dfl mmio resource info in bar0.
Some minor fixes for comments from Hao & Tom.
v3: no change
v4: minor fixes for Hao's comments.

---
 drivers/fpga/dfl-pci.c |  24 +++----
 drivers/fpga/dfl.c     | 185 +++++++++++++++++++++++++++++++++--------------
--
 drivers/fpga/dfl.h     |   7 +-
 3 files changed, 139 insertions(+), 77 deletions(-)
drivers/fpga/dfl-pci.c +5 -5
index e220bec..a2203d0 100644
@@ -31,12 +31,12 @@ struct cci_drvdata {
 	struct dfl_fpga_cdev *cdev;	/* container device */
 };
 
-static void __iomem *cci_pci_ioremap_bar(struct pci_dev *pcidev, int bar)
+static void __iomem *cci_pci_ioremap_bar0(struct pci_dev *pcidev)
 {
-	if (pcim_iomap_regions(pcidev, BIT(bar), DRV_NAME))
+	if (pcim_iomap_regions(pcidev, BIT(0), DRV_NAME))
 		return NULL;
 
-	return pcim_iomap_table(pcidev)[bar];
+	return pcim_iomap_table(pcidev)[0];
 }
 
 static int cci_pci_alloc_irq(struct pci_dev *pcidev)
@@ -156,8 +156,8 @@ static int cci_enumerate_feature_devs(struct pci_dev
 *pcidev)
 			goto irq_free_exit;
 	}
 
-	/* start to find Device Feature List from Bar 0 */
-	base = cci_pci_ioremap_bar(pcidev, 0);
+	/* start to find Device Feature List in Bar 0 */
+	base = cci_pci_ioremap_bar0(pcidev);
 	if (!base) {
 		ret = -ENOMEM;
drivers/fpga/dfl-pci.c +1 -1
 		goto irq_free_exit;
@@ -172,7 +172,7 @@ static int cci_enumerate_feature_devs(struct pci_dev
 *pcidev)
 		start = pci_resource_start(pcidev, 0);
 		len = pci_resource_len(pcidev, 0);
 
-		dfl_fpga_enum_info_add_dfl(info, start, len, base);
+		dfl_fpga_enum_info_add_dfl(info, start, len);
 
 		/*
drivers/fpga/dfl-pci.c +5 -7
 		 * find more Device Feature Lists (e.g. Ports) per information
@@ -196,26 +196,24 @@ static int cci_enumerate_feature_devs(struct
 pci_dev *pcidev)
 			 */
 			bar = FIELD_GET(FME_PORT_OFST_BAR_ID, v);
 			offset = FIELD_GET(FME_PORT_OFST_DFH_OFST, v);
-			base = cci_pci_ioremap_bar(pcidev, bar);
-			if (!base)
-				continue;
-
 			start = pci_resource_start(pcidev, bar) + offset;
 			len = pci_resource_len(pcidev, bar) - offset;
 
-			dfl_fpga_enum_info_add_dfl(info, start, len,
-						   base + offset);
+			dfl_fpga_enum_info_add_dfl(info, start, len);
 		}
 	} else if (dfl_feature_is_port(base)) {
 		start = pci_resource_start(pcidev, 0);
 		len = pci_resource_len(pcidev, 0);
 
-		dfl_fpga_enum_info_add_dfl(info, start, len, base);
+		dfl_fpga_enum_info_add_dfl(info, start, len);
 	} else {
 		ret = -ENODEV;
 		goto irq_free_exit;
 	}
 
+	/* release I/O mappings for next step enumeration */
+	pcim_iounmap_regions(pcidev, BIT(0));
+
 	/* start enumeration with prepared enumeration information */
 	cdev = dfl_fpga_feature_devs_enumerate(info);
drivers/fpga/dfl-pci.c +0 -0
 	if (IS_ERR(cdev)) {
drivers/fpga/dfl.c +2 -0
index 18575d9..b5257d7 100644
@@ -250,6 +250,8 @@ int dfl_fpga_check_port_id(struct platform_device
 *pdev, void *pport_id)
 }
 EXPORT_SYMBOL_GPL(dfl_fpga_check_port_id);
 
+#define is_header_feature(feature) ((feature)->id ==
 FEATURE_ID_FIU_HEADER)
+
 /**

* dfl_fpga_dev_feature_uinit - uinit for sub features of dfl feature device
* @pdev: feature device.

drivers/fpga/dfl.c +14 -0
@@ -273,8 +275,22 @@ static int dfl_feature_instance_init(struct
 platform_device *pdev,
 				     struct dfl_feature *feature,
 				     struct dfl_feature_driver *drv)
 {
+	void __iomem *base;
 	int ret = 0;
 
+	if (!is_header_feature(feature)) {
+		base = devm_platform_ioremap_resource(pdev,
+						      feature->resource_index);
+		if (IS_ERR(base)) {
+			dev_err(&pdev->dev,
+				"ioremap failed for feature 0x%x!\n",
+				feature->id);
+			return PTR_ERR(base);
+		}
+
+		feature->ioaddr = base;
+	}
+
 	if (drv->ops->init) {
 		ret = drv->ops->init(pdev, feature);
drivers/fpga/dfl.c +3 -1
 		if (ret)
@@ -427,7 +443,9 @@ EXPORT_SYMBOL_GPL(dfl_fpga_dev_ops_unregister);
  * @irq_table: Linux IRQ numbers for all irqs, indexed by local irq index of
  *	       this device.
  * @feature_dev: current feature device.
- * @ioaddr: header register region address of feature device in enumeration.
+ * @ioaddr: header register region address of current FIU in enumeration.
+ * @start: register resource start of current FIU.
+ * @len: max register resource length of current FIU.
  * @sub_features: a sub features linked list for feature device in

enumeration.
* @feature_num: number of sub features for feature device in
enumeration.
*/

drivers/fpga/dfl.c +3 -4
@@ -439,6 +457,8 @@ struct build_feature_devs_info {
 
 	struct platform_device *feature_dev;
 	void __iomem *ioaddr;
+	resource_size_t start;
+	resource_size_t len;
 	struct list_head sub_features;
 	int feature_num;
 };
@@ -484,10 +504,7 @@ static int build_info_commit_dev(struct
 build_feature_devs_info *binfo)
 	struct dfl_feature_platform_data *pdata;
 	struct dfl_feature_info *finfo, *p;
 	enum dfl_id_type type;
-	int ret, index = 0;
-
-	if (!fdev)
-		return 0;
+	int ret, index = 0, res_idx = 0;
 
 	type = feature_dev_id_type(fdev);
drivers/fpga/dfl.c +1 -1
 	if (WARN_ON_ONCE(type >= DFL_ID_MAX))
@@ -530,16 +547,30 @@ static int build_info_commit_dev(struct
 build_feature_devs_info *binfo)
 
 	/* fill features and resource information for feature dev */
 	list_for_each_entry_safe(finfo, p, &binfo->sub_features, node) {
-		struct dfl_feature *feature = &pdata->features[index];
+		struct dfl_feature *feature = &pdata->features[index++];
 		struct dfl_feature_irq_ctx *ctx;
 		unsigned int i;
 
 		/* save resource information for each feature */

drivers/fpga/dfl.c static int build_info_commit_dev(struct · line 562

Quoted 10 lines
drivers/fpga/dfl.c +5 -3
 		feature->dev = fdev;
 		feature->id = finfo->fid;
-		feature->resource_index = index;
-		feature->ioaddr = finfo->ioaddr;
-		fdev->resource[index++] = finfo->mmio_res;
+
+		/*
+		 * map header resource for dfl bus device. Don't add header
+		 * resource to feature devices, or the resource tree will be
+		 * disordered and cause warning on resource release

I didn't get this description well, could you please explain it a little more here?

Quoted 169 lines
drivers/fpga/dfl.c +12 -0
+		 */
+		if (is_header_feature(feature)) {
+			feature->resource_index = -1;
+			feature->ioaddr =
+				devm_ioremap_resource(binfo->dev,
+						      &finfo->mmio_res);
+			if (IS_ERR(feature->ioaddr))
+				return PTR_ERR(feature->ioaddr);
+		} else {
+			feature->resource_index = res_idx;
+			fdev->resource[res_idx++] = finfo->mmio_res;
+		}
 
 		if (finfo->nr_irqs) {
drivers/fpga/dfl.c +1 -7
 			ctx = devm_kcalloc(binfo->dev, finfo->nr_irqs,
@@ -582,19 +613,13 @@ static int build_info_commit_dev(struct
 build_feature_devs_info *binfo)
 
 static int
 build_info_create_dev(struct build_feature_devs_info *binfo,
-		      enum dfl_id_type type, void __iomem *ioaddr)
+		      enum dfl_id_type type)
 {
 	struct platform_device *fdev;
-	int ret;
 
 	if (type >= DFL_ID_MAX)
 		return -EINVAL;
 
-	/* we will create a new device, commit current device first */
-	ret = build_info_commit_dev(binfo);
-	if (ret)
-		return ret;
-
 	/*
 	 * we use -ENODEV as the initialization indicator which indicates
drivers/fpga/dfl.c +1 -1
 	 * whether the id need to be reclaimed
@@ -605,7 +630,7 @@ build_info_create_dev(struct
 build_feature_devs_info *binfo,
 
 	binfo->feature_dev = fdev;
 	binfo->feature_num = 0;
-	binfo->ioaddr = ioaddr;
+
 	INIT_LIST_HEAD(&binfo->sub_features);
 
drivers/fpga/dfl.c +4 -5
 	fdev->id = dfl_id_alloc(type, &fdev->dev);
@@ -747,18 +772,17 @@ static int parse_feature_irqs(struct
 build_feature_devs_info *binfo,
  */
 static int
 create_feature_instance(struct build_feature_devs_info *binfo,
-			struct dfl_fpga_enum_dfl *dfl, resource_size_t ofst,
-			resource_size_t size, u16 fid)
+			resource_size_t ofst, resource_size_t size, u16 fid)
 {
 	unsigned int irq_base, nr_irqs;
 	struct dfl_feature_info *finfo;
 	int ret;
 
 	/* read feature size and id if inputs are invalid */
-	size = size ? size : feature_size(dfl->ioaddr + ofst);
-	fid = fid ? fid : feature_id(dfl->ioaddr + ofst);
+	size = size ? size : feature_size(binfo->ioaddr + ofst);
+	fid = fid ? fid : feature_id(binfo->ioaddr + ofst);
 
-	if (dfl->len - ofst < size)
+	if (binfo->len - ofst < size)
 		return -EINVAL;
 
drivers/fpga/dfl.c +1 -2
 	ret = parse_feature_irqs(binfo, ofst, fid, &irq_base, &nr_irqs);
@@ -770,12 +794,11 @@ create_feature_instance(struct
 build_feature_devs_info *binfo,
 		return -ENOMEM;
 
 	finfo->fid = fid;
-	finfo->mmio_res.start = dfl->start + ofst;
+	finfo->mmio_res.start = binfo->start + ofst;
 	finfo->mmio_res.end = finfo->mmio_res.start + size - 1;
 	finfo->mmio_res.flags = IORESOURCE_MEM;
 	finfo->irq_base = irq_base;
 	finfo->nr_irqs = nr_irqs;
-	finfo->ioaddr = dfl->ioaddr + ofst;
 
 	list_add_tail(&finfo->node, &binfo->sub_features);
drivers/fpga/dfl.c +0 -1
 	binfo->feature_num++;
@@ -784,7 +807,6 @@ create_feature_instance(struct
 build_feature_devs_info *binfo,
 }
 
 static int parse_feature_port_afu(struct build_feature_devs_info *binfo,
-				  struct dfl_fpga_enum_dfl *dfl,
 				  resource_size_t ofst)
 {
drivers/fpga/dfl.c +5 -4
 	u64 v = readq(binfo->ioaddr + PORT_HDR_CAP);
@@ -792,21 +814,22 @@ static int parse_feature_port_afu(struct
 build_feature_devs_info *binfo,
 
 	WARN_ON(!size);
 
-	return create_feature_instance(binfo, dfl, ofst, size,
 FEATURE_ID_AFU);
+	return create_feature_instance(binfo, ofst, size, FEATURE_ID_AFU);
 }
 
+#define is_feature_dev_detected(binfo) (!!(binfo)->feature_dev)
+
 static int parse_feature_afu(struct build_feature_devs_info *binfo,
-			     struct dfl_fpga_enum_dfl *dfl,
 			     resource_size_t ofst)
 {
-	if (!binfo->feature_dev) {
+	if (!is_feature_dev_detected(binfo)) {
 		dev_err(binfo->dev, "this AFU does not belong to any
 FIU.\n");
 		return -EINVAL;
 	}
 
 	switch (feature_dev_id_type(binfo->feature_dev)) {
 	case PORT_ID:
-		return parse_feature_port_afu(binfo, dfl, ofst);
+		return parse_feature_port_afu(binfo, ofst);
drivers/fpga/dfl.c +0 -0
 	default:
 		dev_info(binfo->dev, "AFU belonging to FIU %s is not

supported yet.\n",

drivers/fpga/dfl.c +32 -1
 			 binfo->feature_dev->name);
@@ -815,8 +838,39 @@ static int parse_feature_afu(struct
 build_feature_devs_info *binfo,
 	return 0;
 }
 
+static int dfl_binfo_prepare(struct build_feature_devs_info *binfo,
+			     resource_size_t start, resource_size_t len)
+{
+	struct device *dev = binfo->dev;
+	void __iomem *ioaddr;
+
+	if (!devm_request_mem_region(dev, start, len, dev_name(dev))) {
+		dev_err(dev, "request region fail, start:%pa, len:%pa\n",
+			&start, &len);
+		return -EBUSY;
+	}
+
+	ioaddr = devm_ioremap(dev, start, len);
+	if (!ioaddr) {
+		dev_err(dev, "ioremap region fail, start:%pa, len:%pa\n",
+			&start, &len);
+		return -ENOMEM;
+	}
+
+	binfo->start = start;
+	binfo->len = len;
+	binfo->ioaddr = ioaddr;
+
+	return 0;
+}
+
+static void dfl_binfo_complete(struct build_feature_devs_info *binfo)
+{
+	devm_iounmap(binfo->dev, binfo->ioaddr);
+	devm_release_mem_region(binfo->dev, binfo->start, binfo->len);
+}
+
 static int parse_feature_fiu(struct build_feature_devs_info *binfo,
-			     struct dfl_fpga_enum_dfl *dfl,
 			     resource_size_t ofst)
 {
drivers/fpga/dfl.c +0 -0
 	int ret = 0;

drivers/fpga/dfl.c static int parse_feature_fiu(struct · line 883

Quoted 7 lines
drivers/fpga/dfl.c +2 -1
@@ -824,27 +878,39 @@ static int parse_feature_fiu(struct
 build_feature_devs_info *binfo,
 	u16 id;
 	u64 v;
 
-	v = readq(dfl->ioaddr + ofst + DFH);
+	if (is_feature_dev_detected(binfo)) {
+		dfl_binfo_complete(binfo);

Some functions are dfl_bind_xxxx but other are build_info_xxxx. If possible, could we unify them using same format? E.g. build_info_prepare/complete_xxx

Thanks
Hao

author maintainer
#12 lore raw

Re: [PATCH v4 2/4] fpga: dfl: map feature mmio resources in their own feature drivers

Quoted 13 lines · Hao
Quoted 13 lines

-----Original Message-----
From: [email protected] <[email protected]> On Behalf Of Xu Yilun
Sent: Monday, August 10, 2020 10:41 AM
To: [email protected]; [email protected]; linux- [email protected]
Cc: [email protected]; [email protected]; Xu, Yilun <[email protected]>; Wu, Hao <[email protected]>; Matthew Gerlach <[email protected]>; Weight, Russell H <[email protected]>
Subject: [PATCH v4 2/4] fpga: dfl: map feature mmio resources in their own feature drivers

Quoted 7 lines
Quoted 5 lines

This patch makes preparation for modularization of DFL sub feature drivers.

Currently, if we need to support a new DFL sub feature, an entry should be added to fme/port_feature_drvs[] in dfl-fme/port-main.c. And we need

I think it's afu-main.c

Yes, I'll change it.

Quoted 6 lines
Quoted 3 lines

to re-compile the whole DFL modules. That make the DFL drivers hard to be extended.

And maybe we can drop this description or move it to cover letter as it's not the problem this patch is going to resolve.

OK.

Quoted 203 lines
Quoted 203 lines

Another consideration is that DFL may contain some IP blocks which are already supported by kernel, most of them are supported by platform device drivers. We could create platform devices for these IP blocks and get them supported by these drivers.

An important issue is that platform device drivers usually requests mmio resources on probe. But now dfl mmio is mapped in dfl bus driver (e.g. dfl-pci) as a whole region. Then platform device drivers for sub features can't request their own mmio resources again. This is what the patch trying to resolve.

This patch changes the DFL enumeration. DFL bus driver will unmap mmio resources after first step enumeration and pass enumeration info to DFL framework. Then DFL framework will map the mmio resources again, do 2nd step enumeration, and also unmap the mmio resources. In this way, sub feature drivers could then request their own mmio resources as needed.

An exception is that mmio resource of FIU headers are still mapped in dfl bus driver. The FIU headers have some fundamental functions (sriov set, port enable/disable) needed for dfl bus devices and other sub features. They should not be unmapped as long as dfl bus device is alive.

---
v2: delete dfl_binfo_shift() cause no shift is possible for FIU parsing.
Only map bar0 for first phase enumeration in dfl-pci, we can find
all dfl mmio resource info in bar0.
Some minor fixes for comments from Hao & Tom.
v3: no change
v4: minor fixes for Hao's comments.

---
 drivers/fpga/dfl-pci.c |  24 +++----
 drivers/fpga/dfl.c     | 185 +++++++++++++++++++++++++++++++++--------------
--
 drivers/fpga/dfl.h     |   7 +-
 3 files changed, 139 insertions(+), 77 deletions(-)
drivers/fpga/dfl-pci.c +5 -5
index e220bec..a2203d0 100644
@@ -31,12 +31,12 @@ struct cci_drvdata {
 struct dfl_fpga_cdev *cdev;/* container device */
 };
 
-static void __iomem *cci_pci_ioremap_bar(struct pci_dev *pcidev, int bar)
+static void __iomem *cci_pci_ioremap_bar0(struct pci_dev *pcidev)
 {
-if (pcim_iomap_regions(pcidev, BIT(bar), DRV_NAME))
+if (pcim_iomap_regions(pcidev, BIT(0), DRV_NAME))
 return NULL;
 
-return pcim_iomap_table(pcidev)[bar];
+return pcim_iomap_table(pcidev)[0];
 }
 
 static int cci_pci_alloc_irq(struct pci_dev *pcidev)
@@ -156,8 +156,8 @@ static int cci_enumerate_feature_devs(struct pci_dev
 *pcidev)
 goto irq_free_exit;
 }
 
-/* start to find Device Feature List from Bar 0 */
-base = cci_pci_ioremap_bar(pcidev, 0);
+/* start to find Device Feature List in Bar 0 */
+base = cci_pci_ioremap_bar0(pcidev);
 if (!base) {
 ret = -ENOMEM;

goto irq_free_exit;

drivers/fpga/dfl-pci.c +1 -1
@@ -172,7 +172,7 @@ static int cci_enumerate_feature_devs(struct pci_dev
 *pcidev)
 start = pci_resource_start(pcidev, 0);
 len = pci_resource_len(pcidev, 0);
 
-dfl_fpga_enum_info_add_dfl(info, start, len, base);
+dfl_fpga_enum_info_add_dfl(info, start, len);
 
 /*

* find more Device Feature Lists (e.g. Ports) per information

drivers/fpga/dfl-pci.c +5 -7
@@ -196,26 +196,24 @@ static int cci_enumerate_feature_devs(struct
 pci_dev *pcidev)
  */
 bar = FIELD_GET(FME_PORT_OFST_BAR_ID, v);
 offset = FIELD_GET(FME_PORT_OFST_DFH_OFST, v);
-base = cci_pci_ioremap_bar(pcidev, bar);
-if (!base)
-continue;
-
 start = pci_resource_start(pcidev, bar) + offset;
 len = pci_resource_len(pcidev, bar) - offset;
 
-dfl_fpga_enum_info_add_dfl(info, start, len,
-   base + offset);
+dfl_fpga_enum_info_add_dfl(info, start, len);
 }
 } else if (dfl_feature_is_port(base)) {
 start = pci_resource_start(pcidev, 0);
 len = pci_resource_len(pcidev, 0);
 
-dfl_fpga_enum_info_add_dfl(info, start, len, base);
+dfl_fpga_enum_info_add_dfl(info, start, len);
 } else {
 ret = -ENODEV;
 goto irq_free_exit;
 }
 
+/* release I/O mappings for next step enumeration */
+pcim_iounmap_regions(pcidev, BIT(0));
+
 /* start enumeration with prepared enumeration information */
 cdev = dfl_fpga_feature_devs_enumerate(info);

if (IS_ERR(cdev)) {

drivers/fpga/dfl.c +2 -0
index 18575d9..b5257d7 100644
@@ -250,6 +250,8 @@ int dfl_fpga_check_port_id(struct platform_device
 *pdev, void *pport_id)
 }
 EXPORT_SYMBOL_GPL(dfl_fpga_check_port_id);
 
+#define is_header_feature(feature) ((feature)->id ==
 FEATURE_ID_FIU_HEADER)
+
 /**

* dfl_fpga_dev_feature_uinit - uinit for sub features of dfl feature device
* @pdev: feature device.

drivers/fpga/dfl.c +14 -0
@@ -273,8 +275,22 @@ static int dfl_feature_instance_init(struct
 platform_device *pdev,
      struct dfl_feature *feature,
      struct dfl_feature_driver *drv)
 {
+void __iomem *base;
 int ret = 0;
 
+if (!is_header_feature(feature)) {
+base = devm_platform_ioremap_resource(pdev,
+      feature->resource_index);
+if (IS_ERR(base)) {
+dev_err(&pdev->dev,
+"ioremap failed for feature 0x%x!\n",
+feature->id);
+return PTR_ERR(base);
+}
+
+feature->ioaddr = base;
+}
+
 if (drv->ops->init) {
 ret = drv->ops->init(pdev, feature);

if (ret)

drivers/fpga/dfl.c +3 -1
@@ -427,7 +443,9 @@ EXPORT_SYMBOL_GPL(dfl_fpga_dev_ops_unregister);
  * @irq_table: Linux IRQ numbers for all irqs, indexed by local irq index of
  *       this device.
  * @feature_dev: current feature device.
- * @ioaddr: header register region address of feature device in enumeration.
+ * @ioaddr: header register region address of current FIU in enumeration.
+ * @start: register resource start of current FIU.
+ * @len: max register resource length of current FIU.
  * @sub_features: a sub features linked list for feature device in

enumeration.
* @feature_num: number of sub features for feature device in
enumeration.
*/

drivers/fpga/dfl.c +3 -4
@@ -439,6 +457,8 @@ struct build_feature_devs_info {
 
 struct platform_device *feature_dev;
 void __iomem *ioaddr;
+resource_size_t start;
+resource_size_t len;
 struct list_head sub_features;
 int feature_num;
 };
@@ -484,10 +504,7 @@ static int build_info_commit_dev(struct
 build_feature_devs_info *binfo)
 struct dfl_feature_platform_data *pdata;
 struct dfl_feature_info *finfo, *p;
 enum dfl_id_type type;
-int ret, index = 0;
-
-if (!fdev)
-return 0;
+int ret, index = 0, res_idx = 0;
 
 type = feature_dev_id_type(fdev);

if (WARN_ON_ONCE(type >= DFL_ID_MAX))

drivers/fpga/dfl.c +1 -1
@@ -530,16 +547,30 @@ static int build_info_commit_dev(struct
 build_feature_devs_info *binfo)
 
 /* fill features and resource information for feature dev */
 list_for_each_entry_safe(finfo, p, &binfo->sub_features, node) {
-struct dfl_feature *feature = &pdata->features[index];
+struct dfl_feature *feature = &pdata->features[index++];
 struct dfl_feature_irq_ctx *ctx;
 unsigned int i;
 
Quoted 14 lines

drivers/fpga/dfl.c static int build_info_commit_dev(struct build_feature_devs_info *binfo) · line 561

Quoted 11 lines
drivers/fpga/dfl.c +5 -3
 /* save resource information for each feature */
 feature->dev = fdev;
 feature->id = finfo->fid;
-feature->resource_index = index;
-feature->ioaddr = finfo->ioaddr;
-fdev->resource[index++] = finfo->mmio_res;
+
+/*
+ * map header resource for dfl bus device. Don't add header
+ * resource to feature devices, or the resource tree will be
+ * disordered and cause warning on resource release

I didn't get this description well, could you please explain it a little more here?

Yes. I could improve the comments.

The mmio resource of FIU headerS should be owned by dfl bus device (e.g. dfl-pci), cause they have some fundamental functions (sriov set, port enable/disable) needed for dfl bus devices and other sub features. They should not be unmapped as long as dfl bus device is alive.

So we should not assign them to feature devices (dfl-fme/afu) again.

Quoted 174 lines
Quoted 174 lines
drivers/fpga/dfl.c +12 -0
+ */
+if (is_header_feature(feature)) {
+feature->resource_index = -1;
+feature->ioaddr =
+devm_ioremap_resource(binfo->dev,
+      &finfo->mmio_res);
+if (IS_ERR(feature->ioaddr))
+return PTR_ERR(feature->ioaddr);
+} else {
+feature->resource_index = res_idx;
+fdev->resource[res_idx++] = finfo->mmio_res;
+}
 
 if (finfo->nr_irqs) {

ctx = devm_kcalloc(binfo->dev, finfo->nr_irqs,

drivers/fpga/dfl.c +1 -7
@@ -582,19 +613,13 @@ static int build_info_commit_dev(struct
 build_feature_devs_info *binfo)
 
 static int
 build_info_create_dev(struct build_feature_devs_info *binfo,
-      enum dfl_id_type type, void __iomem *ioaddr)
+      enum dfl_id_type type)
 {
 struct platform_device *fdev;
-int ret;
 
 if (type >= DFL_ID_MAX)
 return -EINVAL;
 
-/* we will create a new device, commit current device first */
-ret = build_info_commit_dev(binfo);
-if (ret)
-return ret;
-
 /*
  * we use -ENODEV as the initialization indicator which indicates

* whether the id need to be reclaimed

drivers/fpga/dfl.c +1 -1
@@ -605,7 +630,7 @@ build_info_create_dev(struct
 build_feature_devs_info *binfo,
 
 binfo->feature_dev = fdev;
 binfo->feature_num = 0;
-binfo->ioaddr = ioaddr;
+
 INIT_LIST_HEAD(&binfo->sub_features);
 

fdev->id = dfl_id_alloc(type, &fdev->dev);

drivers/fpga/dfl.c +4 -5
@@ -747,18 +772,17 @@ static int parse_feature_irqs(struct
 build_feature_devs_info *binfo,
  */
 static int
 create_feature_instance(struct build_feature_devs_info *binfo,
-struct dfl_fpga_enum_dfl *dfl, resource_size_t ofst,
-resource_size_t size, u16 fid)
+resource_size_t ofst, resource_size_t size, u16 fid)
 {
 unsigned int irq_base, nr_irqs;
 struct dfl_feature_info *finfo;
 int ret;
 
 /* read feature size and id if inputs are invalid */
-size = size ? size : feature_size(dfl->ioaddr + ofst);
-fid = fid ? fid : feature_id(dfl->ioaddr + ofst);
+size = size ? size : feature_size(binfo->ioaddr + ofst);
+fid = fid ? fid : feature_id(binfo->ioaddr + ofst);
 
-if (dfl->len - ofst < size)
+if (binfo->len - ofst < size)
 return -EINVAL;
 

ret = parse_feature_irqs(binfo, ofst, fid, &irq_base, &nr_irqs);

drivers/fpga/dfl.c +1 -2
@@ -770,12 +794,11 @@ create_feature_instance(struct
 build_feature_devs_info *binfo,
 return -ENOMEM;
 
 finfo->fid = fid;
-finfo->mmio_res.start = dfl->start + ofst;
+finfo->mmio_res.start = binfo->start + ofst;
 finfo->mmio_res.end = finfo->mmio_res.start + size - 1;
 finfo->mmio_res.flags = IORESOURCE_MEM;
 finfo->irq_base = irq_base;
 finfo->nr_irqs = nr_irqs;
-finfo->ioaddr = dfl->ioaddr + ofst;
 
 list_add_tail(&finfo->node, &binfo->sub_features);

binfo->feature_num++;

drivers/fpga/dfl.c +0 -1
@@ -784,7 +807,6 @@ create_feature_instance(struct
 build_feature_devs_info *binfo,
 }
 
 static int parse_feature_port_afu(struct build_feature_devs_info *binfo,
-  struct dfl_fpga_enum_dfl *dfl,
   resource_size_t ofst)
 {

u64 v = readq(binfo->ioaddr + PORT_HDR_CAP);

drivers/fpga/dfl.c +5 -4
@@ -792,21 +814,22 @@ static int parse_feature_port_afu(struct
 build_feature_devs_info *binfo,
 
 WARN_ON(!size);
 
-return create_feature_instance(binfo, dfl, ofst, size,
 FEATURE_ID_AFU);
+return create_feature_instance(binfo, ofst, size, FEATURE_ID_AFU);
 }
 
+#define is_feature_dev_detected(binfo) (!!(binfo)->feature_dev)
+
 static int parse_feature_afu(struct build_feature_devs_info *binfo,
-     struct dfl_fpga_enum_dfl *dfl,
      resource_size_t ofst)
 {
-if (!binfo->feature_dev) {
+if (!is_feature_dev_detected(binfo)) {
 dev_err(binfo->dev, "this AFU does not belong to any
 FIU.\n");
 return -EINVAL;
 }
 
 switch (feature_dev_id_type(binfo->feature_dev)) {
 case PORT_ID:
-return parse_feature_port_afu(binfo, dfl, ofst);
+return parse_feature_port_afu(binfo, ofst);

default:
dev_info(binfo->dev, "AFU belonging to FIU %s is not
supported yet.\n",
binfo->feature_dev->name);

drivers/fpga/dfl.c +32 -1
@@ -815,8 +838,39 @@ static int parse_feature_afu(struct
 build_feature_devs_info *binfo,
 return 0;
 }
 
+static int dfl_binfo_prepare(struct build_feature_devs_info *binfo,
+     resource_size_t start, resource_size_t len)
+{
+struct device *dev = binfo->dev;
+void __iomem *ioaddr;
+
+if (!devm_request_mem_region(dev, start, len, dev_name(dev))) {
+dev_err(dev, "request region fail, start:%pa, len:%pa\n",
+&start, &len);
+return -EBUSY;
+}
+
+ioaddr = devm_ioremap(dev, start, len);
+if (!ioaddr) {
+dev_err(dev, "ioremap region fail, start:%pa, len:%pa\n",
+&start, &len);
+return -ENOMEM;
+}
+
+binfo->start = start;
+binfo->len = len;
+binfo->ioaddr = ioaddr;
+
+return 0;
+}
+
+static void dfl_binfo_complete(struct build_feature_devs_info *binfo)
+{
+devm_iounmap(binfo->dev, binfo->ioaddr);
+devm_release_mem_region(binfo->dev, binfo->start, binfo->len);
+}
+
 static int parse_feature_fiu(struct build_feature_devs_info *binfo,
-     struct dfl_fpga_enum_dfl *dfl,
      resource_size_t ofst)
 {

int ret = 0;

drivers/fpga/dfl.c +0 -0
@@ -824,27 +878,39 @@ static int parse_feature_fiu(struct
 build_feature_devs_info *binfo,
 u16 id;
 u64 v;
 
Quoted 7 lines

drivers/fpga/dfl.c static int parse_feature_fiu(struct build_feature_devs_info *binfo, · line 882

Quoted 3 lines
drivers/fpga/dfl.c +2 -1
-v = readq(dfl->ioaddr + ofst + DFH);
+if (is_feature_dev_detected(binfo)) {
+dfl_binfo_complete(binfo);

Some functions are dfl_bind_xxxx but other are build_info_xxxx. If possible, could we unify them using same format? E.g. build_info_prepare/complete_xxx

Yes I'll fix them.

Quoted 3 lines

Thanks
Hao

author maintainer
#13 lore raw

PATCH v4 3/4 fpga: dfl: create a dfl bus type to support DFL devices

In reply to Xu Yilun

A new bus type "dfl" is introduced for private features which are not initialized by DFL feature drivers (dfl-fme & dfl-afu drivers). So these private features could be handled by separate driver modules.

DFL feature drivers (dfl-fme, dfl-port) will create DFL devices on enumeration. DFL drivers could be registered on this bus to match these DFL devices. They are matched by dfl type & feature_id.

---
v2: change the bus uevent format.
change the dfl device's sysfs name format.
refactor dfl_dev_add().
minor fixes for comments from Hao and Tom.
v3: no change.
v4: improve the uevent format, 4 bits for type & 12 bits for id.
change dfl_device->type to u8.
A dedicate field in struct dfl_feature for dfl device instance. error out if dfl_device already exist on dfl_devs_init().

---
 Documentation/ABI/testing/sysfs-bus-dfl |  15 ++
 drivers/fpga/dfl.c                      | 263 +++++++++++++++++++++++++++++++-
 drivers/fpga/dfl.h                      |  85 +++++++++++
 3 files changed, 355 insertions(+), 8 deletions(-)
 create mode 100644 Documentation/ABI/testing/sysfs-bus-dfl
Documentation/ABI/testing/sysfs-bus-dfl new file +15 -0
new file mode 100644
index 0000000..4d74cc0
@@ -0,0 +1,15 @@
+What:		/sys/bus/dfl/devices/dfl_dev.X/type
+Date:		July 2020
+KernelVersion:	5.10
+Contact:	Xu Yilun <[email protected]>
+Description:	Read-only. It returns type of DFL FIU of the device. Now DFL
+		supports 2 FIU types, 0 for FME, 1 for PORT.
+		Format: 0x%x
+
+What:		/sys/bus/dfl/devices/dfl_dev.X/feature_id
+Date:		July 2020
+KernelVersion:	5.10
+Contact:	Xu Yilun <[email protected]>
+Description:	Read-only. It returns feature identifier local to its DFL FIU
+		type.
+		Format: 0x%x
drivers/fpga/dfl.c +255 -8

308 lines not on this page · raw message

drivers/fpga/dfl.h +85 -0

95 lines not on this page · raw message

Signature · 1 line
2.7.4
#14 lore raw

RE: [PATCH v4 3/4] fpga: dfl: create a dfl bus type to support DFL devices

Quoted 209 lines

A new bus type "dfl" is introduced for private features which are not initialized by DFL feature drivers (dfl-fme & dfl-afu drivers). So these private features could be handled by separate driver modules.

DFL feature drivers (dfl-fme, dfl-port) will create DFL devices on enumeration. DFL drivers could be registered on this bus to match these DFL devices. They are matched by dfl type & feature_id.

---
v2: change the bus uevent format.
change the dfl device's sysfs name format.
refactor dfl_dev_add().
minor fixes for comments from Hao and Tom.
v3: no change.
v4: improve the uevent format, 4 bits for type & 12 bits for id.
change dfl_device->type to u8.
A dedicate field in struct dfl_feature for dfl device instance. error out if dfl_device already exist on dfl_devs_init().

---
 Documentation/ABI/testing/sysfs-bus-dfl |  15 ++
 drivers/fpga/dfl.c                      | 263 +++++++++++++++++++++++++++++++-
 drivers/fpga/dfl.h                      |  85 +++++++++++
 3 files changed, 355 insertions(+), 8 deletions(-)
 create mode 100644 Documentation/ABI/testing/sysfs-bus-dfl
Documentation/ABI/testing/sysfs-bus-dfl new file +15 -0
new file mode 100644
index 0000000..4d74cc0
@@ -0,0 +1,15 @@
+What:		/sys/bus/dfl/devices/dfl_dev.X/type
+Date:		July 2020
+KernelVersion:	5.10
+Contact:	Xu Yilun <[email protected]>
+Description:	Read-only. It returns type of DFL FIU of the device. Now DFL
+		supports 2 FIU types, 0 for FME, 1 for PORT.
+		Format: 0x%x
+
+What:		/sys/bus/dfl/devices/dfl_dev.X/feature_id
+Date:		July 2020
+KernelVersion:	5.10
+Contact:	Xu Yilun <[email protected]>
+Description:	Read-only. It returns feature identifier local to its DFL FIU
+		type.
+		Format: 0x%x
drivers/fpga/dfl.c +132 -6
index b5257d7..51cef85 100644
@@ -30,12 +30,6 @@ static DEFINE_MUTEX(dfl_id_mutex);
  * index to dfl_chardevs table. If no chardev support just set devt_type
  * as one invalid index (DFL_FPGA_DEVT_MAX).
  */
-enum dfl_id_type {
-	FME_ID,		/* fme id allocation and mapping */
-	PORT_ID,	/* port id allocation and mapping */
-	DFL_ID_MAX,
-};
-
 enum dfl_fpga_devt_type {
 	DFL_FPGA_DEVT_FME,
 	DFL_FPGA_DEVT_PORT,
@@ -250,6 +244,245 @@ int dfl_fpga_check_port_id(struct platform_device
 *pdev, void *pport_id)
 }
 EXPORT_SYMBOL_GPL(dfl_fpga_check_port_id);
 
+static DEFINE_IDA(dfl_device_ida);
+
+static const struct dfl_device_id *
+dfl_match_one_device(const struct dfl_device_id *id, struct dfl_device
 *ddev)
+{
+	if (id->type == ddev->type && id->feature_id == ddev->feature_id)
+		return id;
+
+	return NULL;
+}
+
+static int dfl_bus_match(struct device *dev, struct device_driver *drv)
+{
+	struct dfl_device *ddev = to_dfl_dev(dev);
+	struct dfl_driver *ddrv = to_dfl_drv(drv);
+	const struct dfl_device_id *id_entry = ddrv->id_table;
+
+	if (id_entry) {
+		while (id_entry->feature_id) {
+			if (dfl_match_one_device(id_entry, ddev)) {
+				ddev->id_entry = id_entry;
+				return 1;
+			}
+			id_entry++;
+		}
+	}
+
+	return 0;
+}
+
+static int dfl_bus_probe(struct device *dev)
+{
+	struct dfl_device *ddev = to_dfl_dev(dev);
+	struct dfl_driver *ddrv = to_dfl_drv(dev->driver);
+
+	return ddrv->probe(ddev);
+}
+
+static int dfl_bus_remove(struct device *dev)
+{
+	struct dfl_device *ddev = to_dfl_dev(dev);
+	struct dfl_driver *ddrv = to_dfl_drv(dev->driver);
+
+	if (ddrv->remove)
+		ddrv->remove(ddev);
+
+	return 0;
+}
+
+static int dfl_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
+{
+	struct dfl_device *ddev = to_dfl_dev(dev);
+
+	/* The type has 4 valid bits and feature_id has 12 valid bits */
+	return add_uevent_var(env, "MODALIAS=dfl:t%01Xf%03X",
+			      ddev->type, ddev->feature_id);
+}
+
+/* show dfl info fields */
+#define dfl_info_attr(field, format_string)				\
+static ssize_t								\
+field##_show(struct device *dev, struct device_attribute *attr,
 	\
+	     char *buf)							\
+{									\
+	struct dfl_device *ddev = to_dfl_dev(dev);			\
+									\
+	return sprintf(buf, format_string, ddev->field);		\
+}									\
+static DEVICE_ATTR_RO(field)
+
+dfl_info_attr(type, "0x%x\n");
+dfl_info_attr(feature_id, "0x%x\n");
+
+static struct attribute *dfl_dev_attrs[] = {
+	&dev_attr_type.attr,
+	&dev_attr_feature_id.attr,
+	NULL,
+};
+
+ATTRIBUTE_GROUPS(dfl_dev);
+
+static struct bus_type dfl_bus_type = {
+	.name		= "dfl",
+	.match		= dfl_bus_match,
+	.probe		= dfl_bus_probe,
+	.remove		= dfl_bus_remove,
+	.uevent		= dfl_bus_uevent,
+	.dev_groups	= dfl_dev_groups,
+};
+
+static void release_dfl_dev(struct device *dev)
+{
+	struct dfl_device *ddev = to_dfl_dev(dev);
+
+	if (ddev->mmio_res.parent)
+		release_resource(&ddev->mmio_res);
+
+	ida_simple_remove(&dfl_device_ida, ddev->id);
+	kfree(ddev->irqs);
+	kfree(ddev);
+}
+
+static struct dfl_device *
+dfl_dev_add(struct dfl_feature_platform_data *pdata,
+	    struct dfl_feature *feature)
+{
+	struct platform_device *pdev = pdata->dev;
+	struct resource *parent_res;
+	struct dfl_device *ddev;
+	int id, i, ret;
+
+	ddev = kzalloc(sizeof(*ddev), GFP_KERNEL);
+	if (!ddev)
+		return ERR_PTR(-ENOMEM);
+
+	id = ida_simple_get(&dfl_device_ida, 0, 0, GFP_KERNEL);
+	if (id < 0) {
+		dev_err(&pdev->dev, "unable to get id\n");
+		kfree(ddev);
+		return ERR_PTR(id);
+	}
+
+	/* freeing resources by put_device() after device_initialize() */
+	device_initialize(&ddev->dev);
+	ddev->dev.parent = &pdev->dev;
+	ddev->dev.bus = &dfl_bus_type;
+	ddev->dev.release = release_dfl_dev;
+	ddev->id = id;
+	ret = dev_set_name(&ddev->dev, "dfl_dev.%d", id);
+	if (ret)
+		goto put_dev;
+

drivers/fpga/dfl.c int dfl_fpga_check_port_id(struct platform_device · line 391

Quoted 10 lines
drivers/fpga/dfl.c +10 -0
+	ddev->type = feature_dev_id_type(pdev);
+	ddev->feature_id = feature->id;
+	ddev->cdev = pdata->dfl_cdev;
+
+	/* add mmio resource */
+	parent_res = &pdev->resource[feature->resource_index];
+	ddev->mmio_res.flags = IORESOURCE_MEM;
+	ddev->mmio_res.start = parent_res->start;
+	ddev->mmio_res.end = parent_res->end;
+	ddev->mmio_res.name = dev_name(&ddev->dev);

It doesn't reuse the pdev->resource only because it wants a more proper name here?

Quoted 51 lines
drivers/fpga/dfl.c +50 -0
+	ret = insert_resource(parent_res, &ddev->mmio_res);
+	if (ret) {
+		dev_err(&pdev->dev, "%s failed to claim resource: %pR\n",
+			dev_name(&ddev->dev), &ddev->mmio_res);
+		goto put_dev;
+	}
+
+	/* then add irq resource */
+	if (feature->nr_irqs) {
+		ddev->irqs = kcalloc(feature->nr_irqs,
+				     sizeof(*ddev->irqs), GFP_KERNEL);
+		if (!ddev->irqs) {
+			ret = -ENOMEM;
+			goto put_dev;
+		}
+
+		for (i = 0; i < feature->nr_irqs; i++)
+			ddev->irqs[i] = feature->irq_ctx[i].irq;
+
+		ddev->num_irqs = feature->nr_irqs;
+	}
+
+	ret = device_add(&ddev->dev);
+	if (ret)
+		goto put_dev;
+
+	dev_info(&pdev->dev, "add dfl_dev: %s\n", dev_name(&ddev->dev));
+	return ddev;
+
+put_dev:
+	/* calls release_dfl_dev() which does the clean up  */
+	put_device(&ddev->dev);
+	return ERR_PTR(ret);
+}
+
+static void dfl_devs_uinit(struct dfl_feature_platform_data *pdata)
+{
+	struct dfl_feature *feature;
+
+	dfl_fpga_dev_for_each_feature(pdata, feature) {
+		if (feature->ddev) {
+			device_unregister(&feature->ddev->dev);
+			feature->ddev = NULL;
+		}
+	}
+}
+
+static int dfl_devs_init(struct platform_device *pdev)
+{
+	struct dfl_feature_platform_data *pdata = dev_get_platdata(&pdev-
Quoted 1 line

dev);

drivers/fpga/dfl.c int dfl_fpga_check_port_id(struct platform_device *pdev, void *pport_id) · line 448

Quoted 10 lines
drivers/fpga/dfl.c +10 -0
+	struct dfl_feature *feature;
+	struct dfl_device *ddev;
+	int ret;
+
+	dfl_fpga_dev_for_each_feature(pdata, feature) {
+		if (feature->ioaddr)
+			continue;
+
+		if (feature->ddev) {
+			ret = -EBUSY;

or -EEXIST

Quoted 42 lines
drivers/fpga/dfl.c +37 -0
+			goto err;
+		}
+
+		ddev = dfl_dev_add(pdata, feature);
+		if (IS_ERR(ddev)) {
+			ret = PTR_ERR(ddev);
+			goto err;
+		}
+
+		feature->ddev = ddev;
+	}
+
+	return 0;
+
+err:
+	dfl_devs_uinit(pdata);
+	return ret;
+}
+
+int __dfl_driver_register(struct dfl_driver *dfl_drv, struct module *owner)
+{
+	if (!dfl_drv || !dfl_drv->probe || !dfl_drv->id_table)
+		return -EINVAL;
+
+	dfl_drv->drv.owner = owner;
+	dfl_drv->drv.bus = &dfl_bus_type;
+
+	return driver_register(&dfl_drv->drv);
+}
+EXPORT_SYMBOL(__dfl_driver_register);
+
+void dfl_driver_unregister(struct dfl_driver *dfl_drv)
+{
+	driver_unregister(&dfl_drv->drv);
+}
+EXPORT_SYMBOL(dfl_driver_unregister);
+
 #define is_header_feature(feature) ((feature)->id ==

FEATURE_ID_FIU_HEADER)

/**

drivers/fpga/dfl.c void dfl_fpga_dev_feature_uinit(struct · line 499

Quoted 7 lines
drivers/fpga/dfl.c +1 -1
@@ -261,12 +494,15 @@ void dfl_fpga_dev_feature_uinit(struct
 platform_device *pdev)
 	struct dfl_feature_platform_data *pdata = dev_get_platdata(&pdev-
 >dev);
 	struct dfl_feature *feature;
 
-	dfl_fpga_dev_for_each_feature(pdata, feature)
+	dfl_devs_uinit(pdata);

Or it's better to rename as dfl_devs_remove(pdata) or something?

Quoted 55 lines
drivers/fpga/dfl.c +3 -0
+
+	dfl_fpga_dev_for_each_feature(pdata, feature) {
 		if (feature->ops) {
 			if (feature->ops->uinit)
 				feature->ops->uinit(pdev, feature);
 			feature->ops = NULL;
 		}
+	}
 }

EXPORT_SYMBOL_GPL(dfl_fpga_dev_feature_uinit);

drivers/fpga/dfl.c +4 -0
@@ -347,6 +583,10 @@ int dfl_fpga_dev_feature_init(struct
 platform_device *pdev,
 		drv++;
 	}
 
+	ret = dfl_devs_init(pdev);
+	if (ret)
+		goto exit;
+
 	return 0;
 exit:
drivers/fpga/dfl.c +8 -1
 	dfl_fpga_dev_feature_uinit(pdev);
@@ -1282,11 +1522,17 @@ static int __init dfl_fpga_init(void)
 {
 	int ret;
 
+	ret = bus_register(&dfl_bus_type);
+	if (ret)
+		return ret;
+
 	dfl_ids_init();
 
 	ret = dfl_chardev_init();
-	if (ret)
+	if (ret) {
 		dfl_ids_destroy();
+		bus_unregister(&dfl_bus_type);
+	}
 
 	return ret;
 }
@@ -1624,6 +1870,7 @@ static void __exit dfl_fpga_exit(void)
 {
 	dfl_chardev_uinit();
 	dfl_ids_destroy();
+	bus_unregister(&dfl_bus_type);
 }
 
 module_init(dfl_fpga_init);
drivers/fpga/dfl.h +0 -0
index 5973769..d7fd03a 100644

drivers/fpga/dfl.h struct dfl_feature { · line 249

Quoted 4 lines
drivers/fpga/dfl.h +1 -0
@@ -246,6 +246,7 @@ struct dfl_feature {
 	struct dfl_feature_irq_ctx *irq_ctx;
 	unsigned int nr_irqs;
 	const struct dfl_feature_ops *ops;
+	struct dfl_device *ddev;

Add some description please.

Quoted 10 lines
drivers/fpga/dfl.h +2 -0
 	void *priv;
 };
 
@@ -514,4 +515,88 @@ long dfl_feature_ioctl_set_irq(struct
 platform_device *pdev,
 			       struct dfl_feature *feature,
 			       unsigned long arg);
 
+/**
+ * enum dfl_id_type - define the DFL FIU types

drivers/fpga/dfl.h long dfl_feature_ioctl_set_irq(struct · line 530

Quoted 10 lines
drivers/fpga/dfl.h +10 -0
+ */
+enum dfl_id_type {
+	FME_ID,
+	PORT_ID,
+	DFL_ID_MAX,
+};
+
+/**
+ * struct dfl_device_id -  dfl device identifier
+ * @type: Type of DFL FIU of the device. See enum dfl_id_type.

Only 4 bits used.

Hao

Quoted 75 lines
drivers/fpga/dfl.h +72 -0
+ * @feature_id: contains 12 bits feature identifier local to its DFL FIU type.
+ * @driver_data: Driver specific data
+ */
+struct dfl_device_id {
+	u8 type;
+	u16 feature_id;
+	unsigned long driver_data;
+};
+
+/**
+ * struct dfl_device - represent an dfl device on dfl bus
+ *
+ * @dev: Generic device interface.
+ * @id: id of the dfl device
+ * @type: Type of DFL FIU of the device. See enum dfl_id_type.
+ * @feature_id: 16 bits feature identifier local to its DFL FIU type.
+ * @mmio_res: MMIO resource of this dfl device.
+ * @irqs: List of Linux IRQ numbers of this dfl device.
+ * @num_irqs: number of IRQs supported by this dfl device.
+ * @cdev: pointer to DFL FPGA container device this dfl device belongs to.
+ * @id_entry: matched id entry in dfl driver's id table.
+ */
+struct dfl_device {
+	struct device dev;
+	int id;
+	u8 type;
+	u16 feature_id;
+	struct resource mmio_res;
+	int *irqs;
+	unsigned int num_irqs;
+	struct dfl_fpga_cdev *cdev;
+	const struct dfl_device_id *id_entry;
+};
+
+/**
+ * struct dfl_driver - represent an dfl device driver
+ *
+ * @drv: Driver model structure.
+ * @id_table: Pointer to table of device IDs the driver is interested in.
+ *	      { } member terminated.
+ * @probe: Mandatory callback for device binding.
+ * @remove: Callback for device unbinding.
+ */
+struct dfl_driver {
+	struct device_driver drv;
+	const struct dfl_device_id *id_table;
+
+	int (*probe)(struct dfl_device *dfl_dev);
+	void (*remove)(struct dfl_device *dfl_dev);
+};
+
+#define to_dfl_dev(d) container_of(d, struct dfl_device, dev)
+#define to_dfl_drv(d) container_of(d, struct dfl_driver, drv)
+
+/*
+ * use a macro to avoid include chaining to get THIS_MODULE
+ */
+#define dfl_driver_register(drv) \
+	__dfl_driver_register(drv, THIS_MODULE)
+int __dfl_driver_register(struct dfl_driver *dfl_drv, struct module *owner);
+void dfl_driver_unregister(struct dfl_driver *dfl_drv);
+
+/*
+ * module_dfl_driver() - Helper macro for drivers that don't do
+ * anything special in module init/exit.  This eliminates a lot of
+ * boilerplate.  Each module may only use this macro once, and
+ * calling it replaces module_init() and module_exit()
+ */
+#define module_dfl_driver(__dfl_driver) \
+	module_driver(__dfl_driver, dfl_driver_register, \
+		      dfl_driver_unregister)
+

#endif /* __FPGA_DFL_H */
--
2.7.4

author maintainer
#15 lore raw

Re: [PATCH v4 3/4] fpga: dfl: create a dfl bus type to support DFL devices

Quoted 214 lines · Hao
Quoted 214 lines

A new bus type "dfl" is introduced for private features which are not initialized by DFL feature drivers (dfl-fme & dfl-afu drivers). So these private features could be handled by separate driver modules.

DFL feature drivers (dfl-fme, dfl-port) will create DFL devices on enumeration. DFL drivers could be registered on this bus to match these DFL devices. They are matched by dfl type & feature_id.

---
v2: change the bus uevent format.
change the dfl device's sysfs name format.
refactor dfl_dev_add().
minor fixes for comments from Hao and Tom.
v3: no change.
v4: improve the uevent format, 4 bits for type & 12 bits for id.
change dfl_device->type to u8.
A dedicate field in struct dfl_feature for dfl device instance. error out if dfl_device already exist on dfl_devs_init().

---
 Documentation/ABI/testing/sysfs-bus-dfl |  15 ++
 drivers/fpga/dfl.c                      | 263 +++++++++++++++++++++++++++++++-
 drivers/fpga/dfl.h                      |  85 +++++++++++
 3 files changed, 355 insertions(+), 8 deletions(-)
 create mode 100644 Documentation/ABI/testing/sysfs-bus-dfl
Documentation/ABI/testing/sysfs-bus-dfl new file +15 -0
new file mode 100644
index 0000000..4d74cc0
@@ -0,0 +1,15 @@
+What:/sys/bus/dfl/devices/dfl_dev.X/type
+Date:July 2020
+KernelVersion:5.10
+Contact:Xu Yilun <[email protected]>
+Description:Read-only. It returns type of DFL FIU of the device. Now DFL
+supports 2 FIU types, 0 for FME, 1 for PORT.
+Format: 0x%x
+
+What:/sys/bus/dfl/devices/dfl_dev.X/feature_id
+Date:July 2020
+KernelVersion:5.10
+Contact:Xu Yilun <[email protected]>
+Description:Read-only. It returns feature identifier local to its DFL FIU
+type.
+Format: 0x%x
drivers/fpga/dfl.c +137 -6
index b5257d7..51cef85 100644
@@ -30,12 +30,6 @@ static DEFINE_MUTEX(dfl_id_mutex);
  * index to dfl_chardevs table. If no chardev support just set devt_type
  * as one invalid index (DFL_FPGA_DEVT_MAX).
  */
-enum dfl_id_type {
-FME_ID,/* fme id allocation and mapping */
-PORT_ID,/* port id allocation and mapping */
-DFL_ID_MAX,
-};
-
 enum dfl_fpga_devt_type {
 DFL_FPGA_DEVT_FME,
 DFL_FPGA_DEVT_PORT,
@@ -250,6 +244,245 @@ int dfl_fpga_check_port_id(struct platform_device
 *pdev, void *pport_id)
 }
 EXPORT_SYMBOL_GPL(dfl_fpga_check_port_id);
 
+static DEFINE_IDA(dfl_device_ida);
+
+static const struct dfl_device_id *
+dfl_match_one_device(const struct dfl_device_id *id, struct dfl_device
 *ddev)
+{
+if (id->type == ddev->type && id->feature_id == ddev->feature_id)
+return id;
+
+return NULL;
+}
+
+static int dfl_bus_match(struct device *dev, struct device_driver *drv)
+{
+struct dfl_device *ddev = to_dfl_dev(dev);
+struct dfl_driver *ddrv = to_dfl_drv(drv);
+const struct dfl_device_id *id_entry = ddrv->id_table;
+
+if (id_entry) {
+while (id_entry->feature_id) {
+if (dfl_match_one_device(id_entry, ddev)) {
+ddev->id_entry = id_entry;
+return 1;
+}
+id_entry++;
+}
+}
+
+return 0;
+}
+
+static int dfl_bus_probe(struct device *dev)
+{
+struct dfl_device *ddev = to_dfl_dev(dev);
+struct dfl_driver *ddrv = to_dfl_drv(dev->driver);
+
+return ddrv->probe(ddev);
+}
+
+static int dfl_bus_remove(struct device *dev)
+{
+struct dfl_device *ddev = to_dfl_dev(dev);
+struct dfl_driver *ddrv = to_dfl_drv(dev->driver);
+
+if (ddrv->remove)
+ddrv->remove(ddev);
+
+return 0;
+}
+
+static int dfl_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
+{
+struct dfl_device *ddev = to_dfl_dev(dev);
+
+/* The type has 4 valid bits and feature_id has 12 valid bits */
+return add_uevent_var(env, "MODALIAS=dfl:t%01Xf%03X",
+      ddev->type, ddev->feature_id);
+}
+
+/* show dfl info fields */
+#define dfl_info_attr(field, format_string)\
+static ssize_t\
+field##_show(struct device *dev, struct device_attribute *attr,
 \
+     char *buf)\
+{\
+struct dfl_device *ddev = to_dfl_dev(dev);\
+\
+return sprintf(buf, format_string, ddev->field);\
+}\
+static DEVICE_ATTR_RO(field)
+
+dfl_info_attr(type, "0x%x\n");
+dfl_info_attr(feature_id, "0x%x\n");
+
+static struct attribute *dfl_dev_attrs[] = {
+&dev_attr_type.attr,
+&dev_attr_feature_id.attr,
+NULL,
+};
+
+ATTRIBUTE_GROUPS(dfl_dev);
+
+static struct bus_type dfl_bus_type = {
+.name= "dfl",
+.match= dfl_bus_match,
+.probe= dfl_bus_probe,
+.remove= dfl_bus_remove,
+.uevent= dfl_bus_uevent,
+.dev_groups= dfl_dev_groups,
+};
+
+static void release_dfl_dev(struct device *dev)
+{
+struct dfl_device *ddev = to_dfl_dev(dev);
+
+if (ddev->mmio_res.parent)
+release_resource(&ddev->mmio_res);
+
+ida_simple_remove(&dfl_device_ida, ddev->id);
+kfree(ddev->irqs);
+kfree(ddev);
+}
+
+static struct dfl_device *
+dfl_dev_add(struct dfl_feature_platform_data *pdata,
+    struct dfl_feature *feature)
+{
+struct platform_device *pdev = pdata->dev;
+struct resource *parent_res;
+struct dfl_device *ddev;
+int id, i, ret;
+
+ddev = kzalloc(sizeof(*ddev), GFP_KERNEL);
+if (!ddev)
+return ERR_PTR(-ENOMEM);
+
+id = ida_simple_get(&dfl_device_ida, 0, 0, GFP_KERNEL);
+if (id < 0) {
+dev_err(&pdev->dev, "unable to get id\n");
+kfree(ddev);
+return ERR_PTR(id);
+}
+
+/* freeing resources by put_device() after device_initialize() */
+device_initialize(&ddev->dev);
+ddev->dev.parent = &pdev->dev;
+ddev->dev.bus = &dfl_bus_type;
+ddev->dev.release = release_dfl_dev;
+ddev->id = id;
+ret = dev_set_name(&ddev->dev, "dfl_dev.%d", id);
+if (ret)
+goto put_dev;
+
+ddev->type = feature_dev_id_type(pdev);
+ddev->feature_id = feature->id;
+ddev->cdev = pdata->dfl_cdev;
+
+/* add mmio resource */
Quoted 8 lines

drivers/fpga/dfl.c int dfl_fpga_check_port_id(struct platform_device *pdev, void *pport_id) · line 388

Quoted 5 lines
drivers/fpga/dfl.c +5 -0
+parent_res = &pdev->resource[feature->resource_index];
+ddev->mmio_res.flags = IORESOURCE_MEM;
+ddev->mmio_res.start = parent_res->start;
+ddev->mmio_res.end = parent_res->end;
+ddev->mmio_res.name = dev_name(&ddev->dev);

It doesn't reuse the pdev->resource only because it wants a more proper name here?

I think yes. To be more precise, this mmio resource is another resource region owned by the dfl device, although its range is the same as the parent resource. It is the child of the parent resource. And we could see the relationship from /proc/iomem. Also we could know exactly from /proc/iomem which module owns the mem region.

Quoted 52 lines
Quoted 52 lines
drivers/fpga/dfl.c +50 -0
+ret = insert_resource(parent_res, &ddev->mmio_res);
+if (ret) {
+dev_err(&pdev->dev, "%s failed to claim resource: %pR\n",
+dev_name(&ddev->dev), &ddev->mmio_res);
+goto put_dev;
+}
+
+/* then add irq resource */
+if (feature->nr_irqs) {
+ddev->irqs = kcalloc(feature->nr_irqs,
+     sizeof(*ddev->irqs), GFP_KERNEL);
+if (!ddev->irqs) {
+ret = -ENOMEM;
+goto put_dev;
+}
+
+for (i = 0; i < feature->nr_irqs; i++)
+ddev->irqs[i] = feature->irq_ctx[i].irq;
+
+ddev->num_irqs = feature->nr_irqs;
+}
+
+ret = device_add(&ddev->dev);
+if (ret)
+goto put_dev;
+
+dev_info(&pdev->dev, "add dfl_dev: %s\n", dev_name(&ddev->dev));
+return ddev;
+
+put_dev:
+/* calls release_dfl_dev() which does the clean up  */
+put_device(&ddev->dev);
+return ERR_PTR(ret);
+}
+
+static void dfl_devs_uinit(struct dfl_feature_platform_data *pdata)
+{
+struct dfl_feature *feature;
+
+dfl_fpga_dev_for_each_feature(pdata, feature) {
+if (feature->ddev) {
+device_unregister(&feature->ddev->dev);
+feature->ddev = NULL;
+}
+}
+}
+
+static int dfl_devs_init(struct platform_device *pdev)
+{
+struct dfl_feature_platform_data *pdata = dev_get_platdata(&pdev-
Quoted 1 line

dev);

Quoted 12 lines

drivers/fpga/dfl.c int dfl_fpga_check_port_id(struct platform_device *pdev, void *pport_id) · line 448

Quoted 10 lines
drivers/fpga/dfl.c +10 -0
+struct dfl_feature *feature;
+struct dfl_device *ddev;
+int ret;
+
+dfl_fpga_dev_for_each_feature(pdata, feature) {
+if (feature->ioaddr)
+continue;
+
+if (feature->ddev) {
+ret = -EBUSY;

or -EEXIST

Yes, I could change it.

Quoted 45 lines
Quoted 45 lines
drivers/fpga/dfl.c +37 -0
+goto err;
+}
+
+ddev = dfl_dev_add(pdata, feature);
+if (IS_ERR(ddev)) {
+ret = PTR_ERR(ddev);
+goto err;
+}
+
+feature->ddev = ddev;
+}
+
+return 0;
+
+err:
+dfl_devs_uinit(pdata);
+return ret;
+}
+
+int __dfl_driver_register(struct dfl_driver *dfl_drv, struct module *owner)
+{
+if (!dfl_drv || !dfl_drv->probe || !dfl_drv->id_table)
+return -EINVAL;
+
+dfl_drv->drv.owner = owner;
+dfl_drv->drv.bus = &dfl_bus_type;
+
+return driver_register(&dfl_drv->drv);
+}
+EXPORT_SYMBOL(__dfl_driver_register);
+
+void dfl_driver_unregister(struct dfl_driver *dfl_drv)
+{
+driver_unregister(&dfl_drv->drv);
+}
+EXPORT_SYMBOL(dfl_driver_unregister);
+
 #define is_header_feature(feature) ((feature)->id ==

FEATURE_ID_FIU_HEADER)

/**

drivers/fpga/dfl.c +0 -0
@@ -261,12 +494,15 @@ void dfl_fpga_dev_feature_uinit(struct
 platform_device *pdev)
 struct dfl_feature_platform_data *pdata = dev_get_platdata(&pdev-
Quoted 7 lines

drivers/fpga/dfl.c void dfl_fpga_dev_feature_uinit(struct platform_device *pdev) · line 497

Quoted 5 lines
drivers/fpga/dfl.c +1 -1
 >dev);
 struct dfl_feature *feature;
 
-dfl_fpga_dev_for_each_feature(pdata, feature)
+dfl_devs_uinit(pdata);

Or it's better to rename as dfl_devs_remove(pdata) or something?

I could change it.

Quoted 50 lines
Quoted 50 lines
drivers/fpga/dfl.c +3 -0
+
+dfl_fpga_dev_for_each_feature(pdata, feature) {
 if (feature->ops) {
 if (feature->ops->uinit)
 feature->ops->uinit(pdev, feature);
 feature->ops = NULL;
 }
+}
 }

EXPORT_SYMBOL_GPL(dfl_fpga_dev_feature_uinit);

drivers/fpga/dfl.c +4 -0
@@ -347,6 +583,10 @@ int dfl_fpga_dev_feature_init(struct
 platform_device *pdev,
 drv++;
 }
 
+ret = dfl_devs_init(pdev);
+if (ret)
+goto exit;
+
 return 0;
 exit:

dfl_fpga_dev_feature_uinit(pdev);

drivers/fpga/dfl.c +8 -1
@@ -1282,11 +1522,17 @@ static int __init dfl_fpga_init(void)
 {
 int ret;
 
+ret = bus_register(&dfl_bus_type);
+if (ret)
+return ret;
+
 dfl_ids_init();
 
 ret = dfl_chardev_init();
-if (ret)
+if (ret) {
 dfl_ids_destroy();
+bus_unregister(&dfl_bus_type);
+}
 
 return ret;
 }
@@ -1624,6 +1870,7 @@ static void __exit dfl_fpga_exit(void)
 {
 dfl_chardev_uinit();
 dfl_ids_destroy();
+bus_unregister(&dfl_bus_type);
 }
 
Quoted 12 lines

drivers/fpga/dfl.h struct dfl_feature { · line 249

Quoted 10 lines
drivers/fpga/dfl.c +0 -0
 module_init(dfl_fpga_init);
drivers/fpga/dfl.h +1 -0
index 5973769..d7fd03a 100644
@@ -246,6 +246,7 @@ struct dfl_feature {
 struct dfl_feature_irq_ctx *irq_ctx;
 unsigned int nr_irqs;
 const struct dfl_feature_ops *ops;
+struct dfl_device *ddev;

Add some description please.

Yes.

Quoted 9 lines
Quoted 9 lines
drivers/fpga/dfl.h +0 -0
 void *priv;
 };
 
@@ -514,4 +515,88 @@ long dfl_feature_ioctl_set_irq(struct
 platform_device *pdev,
        struct dfl_feature *feature,
        unsigned long arg);
 
Quoted 14 lines

drivers/fpga/dfl.h long dfl_feature_ioctl_set_irq(struct platform_device *pdev, · line 529

Quoted 12 lines
drivers/fpga/dfl.h +12 -0
+/**
+ * enum dfl_id_type - define the DFL FIU types
+ */
+enum dfl_id_type {
+FME_ID,
+PORT_ID,
+DFL_ID_MAX,
+};
+
+/**
+ * struct dfl_device_id -  dfl device identifier
+ * @type: Type of DFL FIU of the device. See enum dfl_id_type.

Only 4 bits used.

I could make the comments more precise.

Thanks,
Yilun.

Quoted 78 lines

Hao

Quoted 75 lines
drivers/fpga/dfl.h +72 -0
+ * @feature_id: contains 12 bits feature identifier local to its DFL FIU type.
+ * @driver_data: Driver specific data
+ */
+struct dfl_device_id {
+u8 type;
+u16 feature_id;
+unsigned long driver_data;
+};
+
+/**
+ * struct dfl_device - represent an dfl device on dfl bus
+ *
+ * @dev: Generic device interface.
+ * @id: id of the dfl device
+ * @type: Type of DFL FIU of the device. See enum dfl_id_type.
+ * @feature_id: 16 bits feature identifier local to its DFL FIU type.
+ * @mmio_res: MMIO resource of this dfl device.
+ * @irqs: List of Linux IRQ numbers of this dfl device.
+ * @num_irqs: number of IRQs supported by this dfl device.
+ * @cdev: pointer to DFL FPGA container device this dfl device belongs to.
+ * @id_entry: matched id entry in dfl driver's id table.
+ */
+struct dfl_device {
+struct device dev;
+int id;
+u8 type;
+u16 feature_id;
+struct resource mmio_res;
+int *irqs;
+unsigned int num_irqs;
+struct dfl_fpga_cdev *cdev;
+const struct dfl_device_id *id_entry;
+};
+
+/**
+ * struct dfl_driver - represent an dfl device driver
+ *
+ * @drv: Driver model structure.
+ * @id_table: Pointer to table of device IDs the driver is interested in.
+ *      { } member terminated.
+ * @probe: Mandatory callback for device binding.
+ * @remove: Callback for device unbinding.
+ */
+struct dfl_driver {
+struct device_driver drv;
+const struct dfl_device_id *id_table;
+
+int (*probe)(struct dfl_device *dfl_dev);
+void (*remove)(struct dfl_device *dfl_dev);
+};
+
+#define to_dfl_dev(d) container_of(d, struct dfl_device, dev)
+#define to_dfl_drv(d) container_of(d, struct dfl_driver, drv)
+
+/*
+ * use a macro to avoid include chaining to get THIS_MODULE
+ */
+#define dfl_driver_register(drv) \
+__dfl_driver_register(drv, THIS_MODULE)
+int __dfl_driver_register(struct dfl_driver *dfl_drv, struct module *owner);
+void dfl_driver_unregister(struct dfl_driver *dfl_drv);
+
+/*
+ * module_dfl_driver() - Helper macro for drivers that don't do
+ * anything special in module init/exit.  This eliminates a lot of
+ * boilerplate.  Each module may only use this macro once, and
+ * calling it replaces module_init() and module_exit()
+ */
+#define module_dfl_driver(__dfl_driver) \
+module_driver(__dfl_driver, dfl_driver_register, \
+      dfl_driver_unregister)
+

#endif /* __FPGA_DFL_H */
--
2.7.4

author maintainer
#16 lore raw

PATCH v4 4/4 fpga: dfl: add support for N3000 Nios private feature

In reply to Xu Yilun

This patch adds support for the Nios handshake private feature on Intel N3000 FPGA Card.

The terminology "Nios" stands for the NIOS II - 32 bit embedded soft processor designed for FPGAs. This private feature provides a handshake interface to FPGA Nois firmware, which receives retimer configuration command from host and executes via an internal SPI master. When Nios finishes the configuration, host takes over the ownership of the SPI master to control an Intel MAX10 BMC Chip on the SPI bus.

For Nios firmware handshake part, this driver requests the retimer configuration for Nios with parameters from module param, and adds some sysfs nodes for user to query Nios state.

For SPI part, this driver adds a spi-altera platform device as well as the MAX10 BMC spi slave info. A spi-altera driver will be matched to handle following the SPI work.

---
v3: Add the doc for this driver
Minor fixes for comments from Tom
v4: Move the err log in regmap implementation, and delete
n3000_nios_writel/readl(), they have nothing to wrapper now.
Some minor fixes and comments improvement.

---
 .../ABI/testing/sysfs-bus-dfl-devices-n3000-nios   |  18 +
 Documentation/fpga/dfl-n3000-nios.rst              |  41 ++
 Documentation/fpga/index.rst                       |   1 +
 drivers/fpga/Kconfig                               |  12 +
 drivers/fpga/Makefile                              |   2 +
 drivers/fpga/dfl-n3000-nios.c                      | 475 +++++++++++++++++++++
 6 files changed, 549 insertions(+)
 create mode 100644 Documentation/ABI/testing/sysfs-bus-dfl-devices-n3000-nios
 create mode 100644 Documentation/fpga/dfl-n3000-nios.rst
 create mode 100644 drivers/fpga/dfl-n3000-nios.c
Documentation/ABI/testing/sysfs-bus-dfl-devices-n3000-nios new file +18 -0
new file mode 100644
index 0000000..4b2b9c5
@@ -0,0 +1,18 @@
+What:		/sys/bus/dfl/devices/dfl_dev.X/fec_mode
+Date:		July 2020
+KernelVersion:	5.10
+Contact:	Xu Yilun <[email protected]>
+Description:	Read-only. It returns the FEC mode of the ethernet retimer
+		configured by NIOS firmware. "rs" for RS FEC mode, "kr" for
+		KR FEC mode, "no" FOR NO FEC mode. The FEC mode could be set
+		by module parameters, but it could only be set once after the
+		board powers up.
+		Format: string
+
+What:		/sys/bus/dfl/devices/dfl_dev.X/nios_fw_version
+Date:		July 2020
+KernelVersion:	5.10
+Contact:	Xu Yilun <[email protected]>
+Description:	Read-only. It returns the NIOS firmware version in FPGA. Its
+		format is "major.minor.patch".
+		Format: %x.%x.%x
Documentation/fpga/dfl-n3000-nios.rst new file +41 -0

41 lines not on this page · raw message

Documentation/fpga/index.rst +1 -0
index f80f956..5fd3c37 100644
@@ -8,6 +8,7 @@ fpga
     :maxdepth: 1
 
     dfl
+    dfl-n3000-nios
 
 .. only::  subproject and html
 
drivers/fpga/Kconfig +12 -0
index 7cd5a29..f820142 100644
@@ -191,6 +191,18 @@ config FPGA_DFL_AFU
 	  to the FPGA infrastructure via a Port. There may be more than one
 	  Port/AFU per DFL based FPGA device.
 
+config FPGA_DFL_N3000_NIOS
+        tristate "FPGA DFL N3000 NIOS Driver"
+        depends on FPGA_DFL
+        select REGMAP
+        help
+	  This is the driver for the nios handshake private feature on Intel
+	  N3000 FPGA Card. This private feature provides a handshake interface
+	  to FPGA NIOS firmware, which receives retimer configuration command
+	  from host and executes via an internal SPI master. When nios finished
+	  the configuration, host takes over the ownership of the SPI master to
+	  control an Intel MAX10 BMC Chip on the SPI bus.
+
 config FPGA_DFL_PCI
 	tristate "FPGA DFL PCIe Device Driver"
 	depends on PCI && FPGA_DFL
drivers/fpga/Makefile +2 -0
index d8e21df..27f20f2 100644
@@ -44,5 +44,7 @@ dfl-fme-objs += dfl-fme-perf.o
 dfl-afu-objs := dfl-afu-main.o dfl-afu-region.o dfl-afu-dma-region.o
 dfl-afu-objs += dfl-afu-error.o
 
+obj-$(CONFIG_FPGA_DFL_N3000_NIOS)      += dfl-n3000-nios.o
+
 # Drivers for FPGAs which implement DFL
 obj-$(CONFIG_FPGA_DFL_PCI)		+= dfl-pci.o
drivers/fpga/dfl-n3000-nios.c new file +475 -0

475 lines not on this page · raw message

Signature · 1 line
2.7.4
#17 lore raw

RE: [PATCH v4 4/4] fpga: dfl: add support for N3000 Nios private feature

Quoted 213 lines

Subject: [PATCH v4 4/4] fpga: dfl: add support for N3000 Nios private feature

This patch adds support for the Nios handshake private feature on Intel N3000 FPGA Card.

The terminology "Nios" stands for the NIOS II - 32 bit embedded soft processor designed for FPGAs. This private feature provides a handshake interface to FPGA Nois firmware, which receives retimer configuration command from host and executes via an internal SPI master. When Nios finishes the configuration, host takes over the ownership of the SPI master to control an Intel MAX10 BMC Chip on the SPI bus.

For Nios firmware handshake part, this driver requests the retimer configuration for Nios with parameters from module param, and adds some sysfs nodes for user to query Nios state.

For SPI part, this driver adds a spi-altera platform device as well as the MAX10 BMC spi slave info. A spi-altera driver will be matched to handle following the SPI work.

---
v3: Add the doc for this driver
Minor fixes for comments from Tom
v4: Move the err log in regmap implementation, and delete
n3000_nios_writel/readl(), they have nothing to wrapper now.
Some minor fixes and comments improvement.

---
 .../ABI/testing/sysfs-bus-dfl-devices-n3000-nios   |  18 +
 Documentation/fpga/dfl-n3000-nios.rst              |  41 ++
 Documentation/fpga/index.rst                       |   1 +
 drivers/fpga/Kconfig                               |  12 +
 drivers/fpga/Makefile                              |   2 +
 drivers/fpga/dfl-n3000-nios.c                      | 475 +++++++++++++++++++++
 6 files changed, 549 insertions(+)
 create mode 100644 Documentation/ABI/testing/sysfs-bus-dfl-devices-
n3000-nios
 create mode 100644 Documentation/fpga/dfl-n3000-nios.rst
 create mode 100644 drivers/fpga/dfl-n3000-nios.c
Documentation/ABI/testing/sysfs-bus-dfl-devices-n3000-nios new file +7 -0
new file mode 100644
index 0000000..4b2b9c5
@@ -0,0 +1,18 @@
+What:		/sys/bus/dfl/devices/dfl_dev.X/fec_mode
+Date:		July 2020
+KernelVersion:	5.10
+Contact:	Xu Yilun <[email protected]>
+Description:	Read-only. It returns the FEC mode of the ethernet retimer
+		configured by NIOS firmware. "rs" for RS FEC mode, "kr" for
+		KR FEC mode, "no" FOR NO FEC mode. The FEC mode could

be set

Documentation/ABI/testing/sysfs-bus-dfl-devices-n3000-nios new file +11 -0
+		by module parameters, but it could only be set once after the
+		board powers up.
+		Format: string
+
+What:		/sys/bus/dfl/devices/dfl_dev.X/nios_fw_version
+Date:		July 2020
+KernelVersion:	5.10
+Contact:	Xu Yilun <[email protected]>
+Description:	Read-only. It returns the NIOS firmware version in FPGA. Its
+		format is "major.minor.patch".
+		Format: %x.%x.%x
Documentation/fpga/dfl-n3000-nios.rst new file +7 -0
new file mode 100644
index 0000000..ecc14e3
@@ -0,0 +1,41 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=====================================
+DFL N3000 Nios Private Feature Driver
+=====================================
+
+The dfl N3000 Nios driver supports for the Nios handshake private feature

on

Documentation/fpga/dfl-n3000-nios.rst new file +3 -0
+Intel N3000 FPGA Card.
+
+The terminology "Nios" stands for the NIOS II - 32 bit embedded soft

processor
+designed for FPGAs. This private feature provides a handshake interface to FPGA
+Nios firmware, which receives the ethernet retimer configuration command from
+host and does the configuration via an internal SPI master. When Nios finishes
+the configuration, host takes over the ownership of the SPI master to control

Documentation/fpga/dfl-n3000-nios.rst new file +3 -0
+an Intel MAX10 BMC Chip on the SPI bus.
+
+So the driver does 2 major tasks on probe, uses the Nios firmware to

configure
+the ethernet retimer, and then creates a spi master platform device with the

Documentation/fpga/dfl-n3000-nios.rst new file +9 -0
+MAX10 device info in spi_board_info.
+
+Module Parameters
+=================
+
+The dfl N3000 Nios driver supports the following module parameters:
+
+* fec_mode: string
+  Require the Nios firmware to set the FEC (Forward Error Correction) mode

of
+ the ethernet retimer on the PAC N3000 FPGA card. The possible values could

Documentation/fpga/dfl-n3000-nios.rst new file +13 -0
+  be:
+
+  - "rs": Reed Solomon FEC (default)
+  - "kr": Fire Code FEC
+  - "no": No FEC
+
+  The configuration can only be set once after the board powers up, the
+  firmware will not accept second configuration afterward. So the fec mode
+  will not be changed if the module is reloaded with a different param value.
+
+  The configured value of the fec mode could be queried from sysfs node:
+
+  /sys/bus/dfl/devices/dfl_dev.X/fec_mode
Documentation/fpga/index.rst +1 -0
index f80f956..5fd3c37 100644
@@ -8,6 +8,7 @@ fpga
     :maxdepth: 1
 
     dfl
+    dfl-n3000-nios
 
 .. only::  subproject and html
 
drivers/fpga/Kconfig +7 -0
index 7cd5a29..f820142 100644
@@ -191,6 +191,18 @@ config FPGA_DFL_AFU
 	  to the FPGA infrastructure via a Port. There may be more than one
 	  Port/AFU per DFL based FPGA device.
 
+config FPGA_DFL_N3000_NIOS
+        tristate "FPGA DFL N3000 NIOS Driver"
+        depends on FPGA_DFL
+        select REGMAP
+        help
+	  This is the driver for the nios handshake private feature on Intel
+	  N3000 FPGA Card. This private feature provides a handshake

interface

drivers/fpga/Kconfig +1 -0
+	  to FPGA NIOS firmware, which receives retimer configuration

command

drivers/fpga/Kconfig +1 -0
+	  from host and executes via an internal SPI master. When nios

finished

drivers/fpga/Kconfig +1 -0
+	  the configuration, host takes over the ownership of the SPI master

to

drivers/fpga/Kconfig +2 -0
+	  control an Intel MAX10 BMC Chip on the SPI bus.
+
 config FPGA_DFL_PCI
 	tristate "FPGA DFL PCIe Device Driver"
 	depends on PCI && FPGA_DFL
drivers/fpga/Makefile +2 -0
index d8e21df..27f20f2 100644
@@ -44,5 +44,7 @@ dfl-fme-objs += dfl-fme-perf.o
 dfl-afu-objs := dfl-afu-main.o dfl-afu-region.o dfl-afu-dma-region.o
 dfl-afu-objs += dfl-afu-error.o
 
+obj-$(CONFIG_FPGA_DFL_N3000_NIOS)      += dfl-n3000-nios.o
+
 # Drivers for FPGAs which implement DFL
 obj-$(CONFIG_FPGA_DFL_PCI)		+= dfl-pci.o
drivers/fpga/dfl-n3000-nios.c new file +28 -0
new file mode 100644
index 0000000..b3093a2
@@ -0,0 +1,475 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * DFL device driver for Nios private feature on Intel PAC N3000
+ *
+ * Copyright (C) 2019-2020 Intel Corporation, Inc.
+ *
+ * Authors:
+ *   Wu Hao <[email protected]>
+ *   Xu Yilun <[email protected]>
+ */
+#include <linux/bitfield.h>
+#include <linux/errno.h>
+#include <linux/io.h>
+#include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/stddef.h>
+#include <linux/spi/altera.h>
+#include <linux/spi/spi.h>
+#include <linux/types.h>
+
+#include "dfl.h"
+
+static char *fec_mode = "rs";
+module_param(fec_mode, charp, 0444);
+MODULE_PARM_DESC(fec_mode, "FEC mode of the ethernet retimer on
Quoted 1 line

PAC N3000 FPGA card");

Intel PAC..

Quoted 192 lines
drivers/fpga/dfl-n3000-nios.c new file +134 -0
+
+/* N3000 Nios private feature registers */
+#define NIOS_SPI_PARAM			0x8
+#define PARAM_SHIFT_MODE_MSK		BIT_ULL(1)
+#define PARAM_SHIFT_MODE_MSB		0
+#define PARAM_SHIFT_MODE_LSB		1
+#define PARAM_DATA_WIDTH		GENMASK_ULL(7, 2)
+#define PARAM_NUM_CS			GENMASK_ULL(13, 8)
+#define PARAM_CLK_POL			BIT_ULL(14)
+#define PARAM_CLK_PHASE			BIT_ULL(15)
+#define PARAM_PERIPHERAL_ID		GENMASK_ULL(47, 32)
+
+#define NIOS_SPI_CTRL			0x10
+#define CTRL_WR_DATA			GENMASK_ULL(31, 0)
+#define CTRL_ADDR			GENMASK_ULL(44, 32)
+#define CTRL_CMD_MSK			GENMASK_ULL(63, 62)
+#define CTRL_CMD_NOP			0
+#define CTRL_CMD_RD			1
+#define CTRL_CMD_WR			2
+
+#define NIOS_SPI_STAT			0x18
+#define STAT_RD_DATA			GENMASK_ULL(31, 0)
+#define STAT_RW_VAL			BIT_ULL(32)
+
+/* Nios handshake registers, indirect access */
+#define NIOS_INIT			0x1000
+#define NIOS_INIT_DONE			BIT(0)
+#define NIOS_INIT_START			BIT(1)
+/* Mode for PKVL A, link 0, the same below */
+#define REQ_FEC_MODE_A0_MSK		GENMASK(9, 8)
+#define REQ_FEC_MODE_A1_MSK		GENMASK(11, 10)
+#define REQ_FEC_MODE_A2_MSK		GENMASK(13, 12)
+#define REQ_FEC_MODE_A3_MSK		GENMASK(15, 14)
+#define REQ_FEC_MODE_B0_MSK		GENMASK(17, 16)
+#define REQ_FEC_MODE_B1_MSK		GENMASK(19, 18)
+#define REQ_FEC_MODE_B2_MSK		GENMASK(21, 20)
+#define REQ_FEC_MODE_B3_MSK		GENMASK(23, 22)
+#define REQ_FEC_MODE_NO			0x0
+#define REQ_FEC_MODE_KR			0x1
+#define REQ_FEC_MODE_RS			0x2
+
+#define NIOS_FW_VERSION			0x1004
+#define NIOS_FW_VERSION_PATCH		GENMASK(23, 20)
+#define NIOS_FW_VERSION_MINOR		GENMASK(27, 24)
+#define NIOS_FW_VERSION_MAJOR		GENMASK(31, 28)
+
+#define PKVL_A_MODE_STS			0x1020
+#define PKVL_B_MODE_STS			0x1024
+#define PKVL_MODE_STS_GROUP_MSK		GENMASK(15, 8)
+#define PKVL_MODE_STS_GROUP_OK		0x0
+#define PKVL_MODE_STS_ID_MSK		GENMASK(7, 0)
+
+#define NS_REGBUS_WAIT_TIMEOUT	10000		/* loop count */
+#define NIOS_INIT_TIMEOUT		10000000	/* usec */
+#define NIOS_INIT_TIME_INTV		100000		/* usec */
+
+struct dfl_n3000_nios {
+	void __iomem *base;
+	struct regmap *regmap;
+	struct device *dev;
+	struct platform_device *altera_spi;
+};
+
+static ssize_t nios_fw_version_show(struct device *dev,
+				    struct device_attribute *attr, char *buf)
+{
+	struct dfl_n3000_nios *ns = dev_get_drvdata(dev);
+	unsigned int val;
+	int ret;
+
+	ret = regmap_read(ns->regmap, NIOS_FW_VERSION, &val);
+	if (ret)
+		return ret;
+
+	return sprintf(buf, "%x.%x.%x\n",
+		       (u8)FIELD_GET(NIOS_FW_VERSION_MAJOR, val),
+		       (u8)FIELD_GET(NIOS_FW_VERSION_MINOR, val),
+		       (u8)FIELD_GET(NIOS_FW_VERSION_PATCH, val));
+}
+static DEVICE_ATTR_RO(nios_fw_version);
+
+static ssize_t fec_mode_show(struct device *dev,
+			     struct device_attribute *attr, char *buf)
+{
+	struct dfl_n3000_nios *ns = dev_get_drvdata(dev);
+	unsigned int val, mode;
+	int ret;
+
+	ret = regmap_read(ns->regmap, NIOS_INIT, &val);
+	if (ret)
+		return ret;
+
+	/*
+	 * FEC mode should always be the same for all links, as we set them
+	 * in this way.
+	 */
+	mode = FIELD_GET(REQ_FEC_MODE_A0_MSK, val);
+	if (mode != FIELD_GET(REQ_FEC_MODE_A1_MSK, val) ||
+	    mode != FIELD_GET(REQ_FEC_MODE_A2_MSK, val) ||
+	    mode != FIELD_GET(REQ_FEC_MODE_A3_MSK, val) ||
+	    mode != FIELD_GET(REQ_FEC_MODE_B0_MSK, val) ||
+	    mode != FIELD_GET(REQ_FEC_MODE_B1_MSK, val) ||
+	    mode != FIELD_GET(REQ_FEC_MODE_B2_MSK, val) ||
+	    mode != FIELD_GET(REQ_FEC_MODE_B3_MSK, val))
+		return -EFAULT;
+
+	switch (mode) {
+	case REQ_FEC_MODE_NO:
+		return sprintf(buf, "no\n");
+	case REQ_FEC_MODE_KR:
+		return sprintf(buf, "kr\n");
+	case REQ_FEC_MODE_RS:
+		return sprintf(buf, "rs\n");
+	}
+
+	return -EFAULT;
+}
+static DEVICE_ATTR_RO(fec_mode);
+
+static struct attribute *n3000_nios_attrs[] = {
+	&dev_attr_nios_fw_version.attr,
+	&dev_attr_fec_mode.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(n3000_nios);
+
+static bool init_error_detected(struct dfl_n3000_nios *ns)
+{
+	unsigned int val;
+
+	if (regmap_read(ns->regmap, PKVL_A_MODE_STS, &val))
+		return true;
+
+	if (FIELD_GET(PKVL_MODE_STS_GROUP_MSK, val) ==

PKVL_MODE_STS_GROUP_OK)

drivers/fpga/dfl-n3000-nios.c new file +6 -0
+		return true;
+
+	if (regmap_read(ns->regmap, PKVL_B_MODE_STS, &val))
+		return true;
+
+	if (FIELD_GET(PKVL_MODE_STS_GROUP_MSK, val) ==

PKVL_MODE_STS_GROUP_OK)

drivers/fpga/dfl-n3000-nios.c new file +45 -0
+		return true;
+
+	return false;
+}
+
+static void dump_error_stat(struct dfl_n3000_nios *ns)
+{
+	unsigned int val;
+
+	if (regmap_read(ns->regmap, PKVL_A_MODE_STS, &val))
+		return;
+
+	dev_info(ns->dev, "PKVL_A_MODE_STS %x\n", val);
+
+	if (regmap_read(ns->regmap, PKVL_B_MODE_STS, &val))
+		return;
+
+	dev_info(ns->dev, "PKVL_B_MODE_STS %x\n", val);
+}
+
+static int n3000_nios_init_done_check(struct dfl_n3000_nios *ns)
+{
+	struct device *dev = ns->dev;
+	unsigned int val, mode;
+	int ret;
+
+	/*
+	 * this SPI is shared by Nios core inside FPGA, Nios will use this SPI
+	 * master to do some one time initialization after power up, and then
+	 * release the control to OS. driver needs to poll on INIT_DONE to
+	 * see when driver could take the control.
+	 *
+	 * Please note that after 3.x.x version, INIT_START is introduced, so
+	 * driver needs to trigger START firstly and then check INIT_DONE.
+	 */
+
+	ret = regmap_read(ns->regmap, NIOS_FW_VERSION, &val);
+	if (ret)
+		return ret;
+
+	/*
+	 * If Nios version register is totally uninitialized(== 0x0), then the
+	 * Nios firmware is missing. So host could take control of SPI master
+	 * safely, but initialization work for Nios is not done. This is an
+	 * issue of FPGA image. We didn't error out because we need SPI

master

drivers/fpga/dfl-n3000-nios.c new file +4 -0
+	 * to reprogram a new image.
+	 */
+	if (val == 0) {
+		dev_warn(dev, "Nios version reg = 0x%x, skip INIT_DONE
Quoted 1 line

check, but PKVL may be uninitialized\n",

Seems no place introduced the PKVL or just use retimer to replace PKVL?

Thanks
Hao

Quoted 8 lines
drivers/fpga/dfl-n3000-nios.c new file +8 -0
+			 val);
+		return 0;
+	}
+
+	if (FIELD_GET(NIOS_FW_VERSION_MAJOR, val) >= 3) {
+		/* read NIOS_INIT to check if PKVL INIT done or not */
+		ret = regmap_read(ns->regmap, NIOS_INIT, &val);
+		if (ret)

drivers/fpga/dfl-n3000-nios.c line 235

Quoted 10 lines
drivers/fpga/dfl-n3000-nios.c new file +10 -0
+			return ret;
+
+		/* check if PKVLs are initialized already */
+		if (val & NIOS_INIT_DONE || val & NIOS_INIT_START)
+			goto nios_init_done;
+
+		/* configure FEC mode per module param */
+		val = NIOS_INIT_START;
+
+		/* FEC mode will be ignored by hardware in 10G mode */

So always no fec mode from hardware? Or software should force it in 10G mode? Or 10G mode, below code can be skipped?

Quoted 248 lines
drivers/fpga/dfl-n3000-nios.c new file +45 -0
+		if (!strcmp(fec_mode, "no"))
+			mode = REQ_FEC_MODE_NO;
+		else if (!strcmp(fec_mode, "kr"))
+			mode = REQ_FEC_MODE_KR;
+		else if (!strcmp(fec_mode, "rs"))
+			mode = REQ_FEC_MODE_RS;
+		else
+			return -EINVAL;
+
+		/* set the same FEC mode for all links */
+		val |= FIELD_PREP(REQ_FEC_MODE_A0_MSK, mode) |
+		       FIELD_PREP(REQ_FEC_MODE_A1_MSK, mode) |
+		       FIELD_PREP(REQ_FEC_MODE_A2_MSK, mode) |
+		       FIELD_PREP(REQ_FEC_MODE_A3_MSK, mode) |
+		       FIELD_PREP(REQ_FEC_MODE_B0_MSK, mode) |
+		       FIELD_PREP(REQ_FEC_MODE_B1_MSK, mode) |
+		       FIELD_PREP(REQ_FEC_MODE_B2_MSK, mode) |
+		       FIELD_PREP(REQ_FEC_MODE_B3_MSK, mode);
+
+		ret = regmap_write(ns->regmap, NIOS_INIT, val);
+		if (ret)
+			return ret;
+	}
+
+nios_init_done:
+	/* polls on NIOS_INIT_DONE */
+	ret = regmap_read_poll_timeout(ns->regmap, NIOS_INIT, val,
+				       val & NIOS_INIT_DONE,
+				       NIOS_INIT_TIME_INTV,
+				       NIOS_INIT_TIMEOUT);
+	if (ret) {
+		dev_err(dev, "NIOS_INIT_DONE %s\n",
+			(ret == -ETIMEDOUT) ? "timed out" : "check error");
+		goto dump_sts;
+	}
+
+	/*
+	 * after INIT_DONE is detected, it still needs to check if any ERR
+	 * detected.
+	 * We won't error out here even if error detected. Nios will release
+	 * spi controller when INIT_DONE is set, so driver could continue to
+	 * initialize spi controller device.
+	 */
+	if (init_error_detected(ns)) {
+		dev_warn(dev, "NIOS_INIT_DONE OK, but err found during

init\n");

drivers/fpga/dfl-n3000-nios.c new file +35 -0
+		goto dump_sts;
+	}
+	return 0;
+
+dump_sts:
+	dump_error_stat(ns);
+
+	return ret;
+}
+
+struct spi_board_info m10_n3000_info = {
+	.modalias = "m10-n3000",
+	.max_speed_hz = 12500000,
+	.bus_num = 0,
+	.chip_select = 0,
+};
+
+static int create_altera_spi_controller(struct dfl_n3000_nios *ns)
+{
+	struct altera_spi_platform_data pdata = { 0 };
+	struct platform_device_info pdevinfo = { 0 };
+	void __iomem *base = ns->base;
+	u64 v;
+
+	v = readq(base + NIOS_SPI_PARAM);
+
+	pdata.mode_bits = SPI_CS_HIGH;
+	if (FIELD_GET(PARAM_CLK_POL, v))
+		pdata.mode_bits |= SPI_CPOL;
+	if (FIELD_GET(PARAM_CLK_PHASE, v))
+		pdata.mode_bits |= SPI_CPHA;
+
+	pdata.num_chipselect = FIELD_GET(PARAM_NUM_CS, v);
+	pdata.bits_per_word_mask =
+		SPI_BPW_RANGE_MASK(1, FIELD_GET(PARAM_DATA_WIDTH,

v));

drivers/fpga/dfl-n3000-nios.c new file +32 -0
+
+	pdata.num_devices = 1;
+	pdata.devices = &m10_n3000_info;
+
+	dev_dbg(ns->dev, "%s cs %u bpm 0x%x mode 0x%x\n", __func__,
+		pdata.num_chipselect, pdata.bits_per_word_mask,
+		pdata.mode_bits);
+
+	pdevinfo.name = "subdev_spi_altera";
+	pdevinfo.id = PLATFORM_DEVID_AUTO;
+	pdevinfo.parent = ns->dev;
+	pdevinfo.data = &pdata;
+	pdevinfo.size_data = sizeof(pdata);
+
+	ns->altera_spi = platform_device_register_full(&pdevinfo);
+	return PTR_ERR_OR_ZERO(ns->altera_spi);
+}
+
+static void destroy_altera_spi_controller(struct dfl_n3000_nios *ns)
+{
+	platform_device_unregister(ns->altera_spi);
+}
+
+/* ns is the abbreviation of nios_spi */
+static int ns_poll_stat_timeout(void __iomem *base, u64 *v)
+{
+	int loops = NS_REGBUS_WAIT_TIMEOUT;
+
+	/*
+	 * We don't use the time based timeout here for performance.
+	 *
+	 * The image reprograming engine is on max10 bmc chip, which is the

spi

drivers/fpga/dfl-n3000-nios.c new file +2 -0
+	 * device connected to altera spi controller. So the regbus read/write
+	 * is on the critical path of PAC N3000 image reprograming. And

usually

drivers/fpga/dfl-n3000-nios.c new file +1 -0
+	 * the state changes in few loops, the time based timeout checking

will

drivers/fpga/dfl-n3000-nios.c new file +77 -0
+	 * add too much overhead on it.
+	 *
+	 * Anyway, 10000 times loop is large enough.
+	 */
+	do {
+		*v = readq(base + NIOS_SPI_STAT);
+		if (*v & STAT_RW_VAL)
+			break;
+		cpu_relax();
+	} while (--loops);
+
+	return loops ? 0 : -ETIMEDOUT;
+}
+
+static int ns_reg_write(void *context, unsigned int reg, unsigned int val)
+{
+	struct dfl_n3000_nios *ns = context;
+	u64 v = 0;
+	int ret;
+
+	v |= FIELD_PREP(CTRL_CMD_MSK, CTRL_CMD_WR);
+	v |= FIELD_PREP(CTRL_ADDR, reg);
+	v |= FIELD_PREP(CTRL_WR_DATA, val);
+	writeq(v, ns->base + NIOS_SPI_CTRL);
+
+	ret = ns_poll_stat_timeout(ns->base, &v);
+	if (ret)
+		dev_err(ns->dev, "fail to write reg 0x%x val 0x%x: %d\n",
+			reg, val, ret);
+
+	return ret;
+}
+
+static int ns_reg_read(void *context, unsigned int reg, unsigned int *val)
+{
+	struct dfl_n3000_nios *ns = context;
+	u64 v = 0;
+	int ret;
+
+	v |= FIELD_PREP(CTRL_CMD_MSK, CTRL_CMD_RD);
+	v |= FIELD_PREP(CTRL_ADDR, reg);
+	writeq(v, ns->base + NIOS_SPI_CTRL);
+
+	ret = ns_poll_stat_timeout(ns->base, &v);
+	if (ret)
+		dev_err(ns->dev, "fail to read reg 0x%x: %d\n", reg, ret);
+	else
+		*val = FIELD_GET(STAT_RD_DATA, v);
+
+	return ret;
+}
+
+static const struct regmap_config ns_regbus_cfg = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.fast_io = true,
+
+	.reg_write = ns_reg_write,
+	.reg_read = ns_reg_read,
+};
+
+static int dfl_n3000_nios_probe(struct dfl_device *dfl_dev)
+{
+	struct device *dev = &dfl_dev->dev;
+	struct dfl_n3000_nios *ns;
+	int ret;
+
+	ns = devm_kzalloc(dev, sizeof(*ns), GFP_KERNEL);
+	if (!ns)
+		return -ENOMEM;
+
+	dev_set_drvdata(&dfl_dev->dev, ns);
+
+	ns->dev = dev;
+
+	ns->base = devm_ioremap_resource(&dfl_dev->dev, &dfl_dev-
Quoted 1 line

mmio_res);

drivers/fpga/dfl-n3000-nios.c new file +48 -1
+	if (IS_ERR(ns->base))
+		return PTR_ERR(ns->base);
+
+	ns->regmap = devm_regmap_init(dev, NULL, ns, &ns_regbus_cfg);
+	if (IS_ERR(ns->regmap))
+		return PTR_ERR(ns->regmap);
+
+	ret = n3000_nios_init_done_check(ns);
+	if (ret)
+		return ret;
+
+	ret = create_altera_spi_controller(ns);
+	if (ret)
+		dev_err(dev, "altera spi controller create failed: %d\n", ret);
+
+	return ret;
+}
+
+static void dfl_n3000_nios_remove(struct dfl_device *dfl_dev)
+{
+	struct dfl_n3000_nios *ns = dev_get_drvdata(&dfl_dev->dev);
+
+	destroy_altera_spi_controller(ns);
+}
+
+#define FME_FEATURE_ID_N3000_NIOS	0xd
+
+static const struct dfl_device_id dfl_n3000_nios_ids[] = {
+	{ FME_ID, FME_FEATURE_ID_N3000_NIOS },
+	{ }
+};
+
+static struct dfl_driver dfl_n3000_nios_driver = {
+	.drv	= {
+		.name       = "dfl-n3000-nios",
+		.dev_groups = n3000_nios_groups,
+	},
+	.id_table = dfl_n3000_nios_ids,
+	.probe   = dfl_n3000_nios_probe,
+	.remove  = dfl_n3000_nios_remove,
+};
+
+module_dfl_driver(dfl_n3000_nios_driver);
+
+MODULE_DEVICE_TABLE(dfl, dfl_n3000_nios_ids);
+MODULE_DESCRIPTION("DFL N3000 Nios private feature driver");
+MODULE_AUTHOR("Intel Corporation");
+MODULE_LICENSE("GPL v2");
--

2.7.4

author maintainer
#18 lore raw

Re: [PATCH v4 4/4] fpga: dfl: add support for N3000 Nios private feature

Quoted 208 lines · Hao
Quoted 208 lines

Subject: [PATCH v4 4/4] fpga: dfl: add support for N3000 Nios private feature

This patch adds support for the Nios handshake private feature on Intel N3000 FPGA Card.

The terminology "Nios" stands for the NIOS II - 32 bit embedded soft processor designed for FPGAs. This private feature provides a handshake interface to FPGA Nois firmware, which receives retimer configuration command from host and executes via an internal SPI master. When Nios finishes the configuration, host takes over the ownership of the SPI master to control an Intel MAX10 BMC Chip on the SPI bus.

For Nios firmware handshake part, this driver requests the retimer configuration for Nios with parameters from module param, and adds some sysfs nodes for user to query Nios state.

For SPI part, this driver adds a spi-altera platform device as well as the MAX10 BMC spi slave info. A spi-altera driver will be matched to handle following the SPI work.

---
v3: Add the doc for this driver
Minor fixes for comments from Tom
v4: Move the err log in regmap implementation, and delete
n3000_nios_writel/readl(), they have nothing to wrapper now.
Some minor fixes and comments improvement.

---
 .../ABI/testing/sysfs-bus-dfl-devices-n3000-nios   |  18 +
 Documentation/fpga/dfl-n3000-nios.rst              |  41 ++
 Documentation/fpga/index.rst                       |   1 +
 drivers/fpga/Kconfig                               |  12 +
 drivers/fpga/Makefile                              |   2 +
 drivers/fpga/dfl-n3000-nios.c                      | 475 +++++++++++++++++++++
 6 files changed, 549 insertions(+)
 create mode 100644 Documentation/ABI/testing/sysfs-bus-dfl-devices-
n3000-nios
 create mode 100644 Documentation/fpga/dfl-n3000-nios.rst
 create mode 100644 drivers/fpga/dfl-n3000-nios.c
Documentation/ABI/testing/sysfs-bus-dfl-devices-n3000-nios new file +7 -0
new file mode 100644
index 0000000..4b2b9c5
@@ -0,0 +1,18 @@
+What:/sys/bus/dfl/devices/dfl_dev.X/fec_mode
+Date:July 2020
+KernelVersion:5.10
+Contact:Xu Yilun <[email protected]>
+Description:Read-only. It returns the FEC mode of the ethernet retimer
+configured by NIOS firmware. "rs" for RS FEC mode, "kr" for
+KR FEC mode, "no" FOR NO FEC mode. The FEC mode could

be set

Documentation/ABI/testing/sysfs-bus-dfl-devices-n3000-nios new file +11 -0
+by module parameters, but it could only be set once after the
+board powers up.
+Format: string
+
+What:/sys/bus/dfl/devices/dfl_dev.X/nios_fw_version
+Date:July 2020
+KernelVersion:5.10
+Contact:Xu Yilun <[email protected]>
+Description:Read-only. It returns the NIOS firmware version in FPGA. Its
+format is "major.minor.patch".
+Format: %x.%x.%x
Documentation/fpga/dfl-n3000-nios.rst new file +7 -0
new file mode 100644
index 0000000..ecc14e3
@@ -0,0 +1,41 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=====================================
+DFL N3000 Nios Private Feature Driver
+=====================================
+
+The dfl N3000 Nios driver supports for the Nios handshake private feature

on

Documentation/fpga/dfl-n3000-nios.rst new file +3 -0
+Intel N3000 FPGA Card.
+
+The terminology "Nios" stands for the NIOS II - 32 bit embedded soft

processor
+designed for FPGAs. This private feature provides a handshake interface to FPGA
+Nios firmware, which receives the ethernet retimer configuration command from
+host and does the configuration via an internal SPI master. When Nios finishes
+the configuration, host takes over the ownership of the SPI master to control

Documentation/fpga/dfl-n3000-nios.rst new file +3 -0
+an Intel MAX10 BMC Chip on the SPI bus.
+
+So the driver does 2 major tasks on probe, uses the Nios firmware to

configure
+the ethernet retimer, and then creates a spi master platform device with the

Documentation/fpga/dfl-n3000-nios.rst new file +9 -0
+MAX10 device info in spi_board_info.
+
+Module Parameters
+=================
+
+The dfl N3000 Nios driver supports the following module parameters:
+
+* fec_mode: string
+  Require the Nios firmware to set the FEC (Forward Error Correction) mode

of
+ the ethernet retimer on the PAC N3000 FPGA card. The possible values could

Documentation/fpga/dfl-n3000-nios.rst new file +13 -0
+  be:
+
+  - "rs": Reed Solomon FEC (default)
+  - "kr": Fire Code FEC
+  - "no": No FEC
+
+  The configuration can only be set once after the board powers up, the
+  firmware will not accept second configuration afterward. So the fec mode
+  will not be changed if the module is reloaded with a different param value.
+
+  The configured value of the fec mode could be queried from sysfs node:
+
+  /sys/bus/dfl/devices/dfl_dev.X/fec_mode
Documentation/fpga/index.rst +1 -0
index f80f956..5fd3c37 100644
@@ -8,6 +8,7 @@ fpga
     :maxdepth: 1
 
     dfl
+    dfl-n3000-nios
 
 .. only::  subproject and html
 
drivers/fpga/Kconfig +7 -0
index 7cd5a29..f820142 100644
@@ -191,6 +191,18 @@ config FPGA_DFL_AFU
   to the FPGA infrastructure via a Port. There may be more than one
   Port/AFU per DFL based FPGA device.
 
+config FPGA_DFL_N3000_NIOS
+        tristate "FPGA DFL N3000 NIOS Driver"
+        depends on FPGA_DFL
+        select REGMAP
+        help
+  This is the driver for the nios handshake private feature on Intel
+  N3000 FPGA Card. This private feature provides a handshake

interface
+ to FPGA NIOS firmware, which receives retimer configuration command
+ from host and executes via an internal SPI master. When nios finished
+ the configuration, host takes over the ownership of the SPI master to

drivers/fpga/Kconfig +2 -0
+  control an Intel MAX10 BMC Chip on the SPI bus.
+
 config FPGA_DFL_PCI
 tristate "FPGA DFL PCIe Device Driver"
 depends on PCI && FPGA_DFL
drivers/fpga/Makefile +2 -0
index d8e21df..27f20f2 100644
@@ -44,5 +44,7 @@ dfl-fme-objs += dfl-fme-perf.o
 dfl-afu-objs := dfl-afu-main.o dfl-afu-region.o dfl-afu-dma-region.o
 dfl-afu-objs += dfl-afu-error.o
 
+obj-$(CONFIG_FPGA_DFL_N3000_NIOS)      += dfl-n3000-nios.o
+
 # Drivers for FPGAs which implement DFL
 obj-$(CONFIG_FPGA_DFL_PCI)+= dfl-pci.o
drivers/fpga/dfl-n3000-nios.c new file +23 -0
new file mode 100644
index 0000000..b3093a2
@@ -0,0 +1,475 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * DFL device driver for Nios private feature on Intel PAC N3000
+ *
+ * Copyright (C) 2019-2020 Intel Corporation, Inc.
+ *
+ * Authors:
+ *   Wu Hao <[email protected]>
+ *   Xu Yilun <[email protected]>
+ */
+#include <linux/bitfield.h>
+#include <linux/errno.h>
+#include <linux/io.h>
+#include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/stddef.h>
+#include <linux/spi/altera.h>
+#include <linux/spi/spi.h>
+#include <linux/types.h>
+
Quoted 8 lines
Quoted 6 lines
drivers/fpga/dfl-n3000-nios.c new file +5 -0
+#include "dfl.h"
+
+static char *fec_mode = "rs";
+module_param(fec_mode, charp, 0444);
+MODULE_PARM_DESC(fec_mode, "FEC mode of the ethernet retimer on

PAC N3000 FPGA card");

Intel PAC..

Yes.

Quoted 187 lines
Quoted 187 lines
drivers/fpga/dfl-n3000-nios.c new file +134 -0
+
+/* N3000 Nios private feature registers */
+#define NIOS_SPI_PARAM0x8
+#define PARAM_SHIFT_MODE_MSKBIT_ULL(1)
+#define PARAM_SHIFT_MODE_MSB0
+#define PARAM_SHIFT_MODE_LSB1
+#define PARAM_DATA_WIDTHGENMASK_ULL(7, 2)
+#define PARAM_NUM_CSGENMASK_ULL(13, 8)
+#define PARAM_CLK_POLBIT_ULL(14)
+#define PARAM_CLK_PHASEBIT_ULL(15)
+#define PARAM_PERIPHERAL_IDGENMASK_ULL(47, 32)
+
+#define NIOS_SPI_CTRL0x10
+#define CTRL_WR_DATAGENMASK_ULL(31, 0)
+#define CTRL_ADDRGENMASK_ULL(44, 32)
+#define CTRL_CMD_MSKGENMASK_ULL(63, 62)
+#define CTRL_CMD_NOP0
+#define CTRL_CMD_RD1
+#define CTRL_CMD_WR2
+
+#define NIOS_SPI_STAT0x18
+#define STAT_RD_DATAGENMASK_ULL(31, 0)
+#define STAT_RW_VALBIT_ULL(32)
+
+/* Nios handshake registers, indirect access */
+#define NIOS_INIT0x1000
+#define NIOS_INIT_DONEBIT(0)
+#define NIOS_INIT_STARTBIT(1)
+/* Mode for PKVL A, link 0, the same below */
+#define REQ_FEC_MODE_A0_MSKGENMASK(9, 8)
+#define REQ_FEC_MODE_A1_MSKGENMASK(11, 10)
+#define REQ_FEC_MODE_A2_MSKGENMASK(13, 12)
+#define REQ_FEC_MODE_A3_MSKGENMASK(15, 14)
+#define REQ_FEC_MODE_B0_MSKGENMASK(17, 16)
+#define REQ_FEC_MODE_B1_MSKGENMASK(19, 18)
+#define REQ_FEC_MODE_B2_MSKGENMASK(21, 20)
+#define REQ_FEC_MODE_B3_MSKGENMASK(23, 22)
+#define REQ_FEC_MODE_NO0x0
+#define REQ_FEC_MODE_KR0x1
+#define REQ_FEC_MODE_RS0x2
+
+#define NIOS_FW_VERSION0x1004
+#define NIOS_FW_VERSION_PATCHGENMASK(23, 20)
+#define NIOS_FW_VERSION_MINORGENMASK(27, 24)
+#define NIOS_FW_VERSION_MAJORGENMASK(31, 28)
+
+#define PKVL_A_MODE_STS0x1020
+#define PKVL_B_MODE_STS0x1024
+#define PKVL_MODE_STS_GROUP_MSKGENMASK(15, 8)
+#define PKVL_MODE_STS_GROUP_OK0x0
+#define PKVL_MODE_STS_ID_MSKGENMASK(7, 0)
+
+#define NS_REGBUS_WAIT_TIMEOUT10000/* loop count */
+#define NIOS_INIT_TIMEOUT10000000/* usec */
+#define NIOS_INIT_TIME_INTV100000/* usec */
+
+struct dfl_n3000_nios {
+void __iomem *base;
+struct regmap *regmap;
+struct device *dev;
+struct platform_device *altera_spi;
+};
+
+static ssize_t nios_fw_version_show(struct device *dev,
+    struct device_attribute *attr, char *buf)
+{
+struct dfl_n3000_nios *ns = dev_get_drvdata(dev);
+unsigned int val;
+int ret;
+
+ret = regmap_read(ns->regmap, NIOS_FW_VERSION, &val);
+if (ret)
+return ret;
+
+return sprintf(buf, "%x.%x.%x\n",
+       (u8)FIELD_GET(NIOS_FW_VERSION_MAJOR, val),
+       (u8)FIELD_GET(NIOS_FW_VERSION_MINOR, val),
+       (u8)FIELD_GET(NIOS_FW_VERSION_PATCH, val));
+}
+static DEVICE_ATTR_RO(nios_fw_version);
+
+static ssize_t fec_mode_show(struct device *dev,
+     struct device_attribute *attr, char *buf)
+{
+struct dfl_n3000_nios *ns = dev_get_drvdata(dev);
+unsigned int val, mode;
+int ret;
+
+ret = regmap_read(ns->regmap, NIOS_INIT, &val);
+if (ret)
+return ret;
+
+/*
+ * FEC mode should always be the same for all links, as we set them
+ * in this way.
+ */
+mode = FIELD_GET(REQ_FEC_MODE_A0_MSK, val);
+if (mode != FIELD_GET(REQ_FEC_MODE_A1_MSK, val) ||
+    mode != FIELD_GET(REQ_FEC_MODE_A2_MSK, val) ||
+    mode != FIELD_GET(REQ_FEC_MODE_A3_MSK, val) ||
+    mode != FIELD_GET(REQ_FEC_MODE_B0_MSK, val) ||
+    mode != FIELD_GET(REQ_FEC_MODE_B1_MSK, val) ||
+    mode != FIELD_GET(REQ_FEC_MODE_B2_MSK, val) ||
+    mode != FIELD_GET(REQ_FEC_MODE_B3_MSK, val))
+return -EFAULT;
+
+switch (mode) {
+case REQ_FEC_MODE_NO:
+return sprintf(buf, "no\n");
+case REQ_FEC_MODE_KR:
+return sprintf(buf, "kr\n");
+case REQ_FEC_MODE_RS:
+return sprintf(buf, "rs\n");
+}
+
+return -EFAULT;
+}
+static DEVICE_ATTR_RO(fec_mode);
+
+static struct attribute *n3000_nios_attrs[] = {
+&dev_attr_nios_fw_version.attr,
+&dev_attr_fec_mode.attr,
+NULL,
+};
+ATTRIBUTE_GROUPS(n3000_nios);
+
+static bool init_error_detected(struct dfl_n3000_nios *ns)
+{
+unsigned int val;
+
+if (regmap_read(ns->regmap, PKVL_A_MODE_STS, &val))
+return true;
+
+if (FIELD_GET(PKVL_MODE_STS_GROUP_MSK, val) ==

PKVL_MODE_STS_GROUP_OK)

drivers/fpga/dfl-n3000-nios.c new file +6 -0
+return true;
+
+if (regmap_read(ns->regmap, PKVL_B_MODE_STS, &val))
+return true;
+
+if (FIELD_GET(PKVL_MODE_STS_GROUP_MSK, val) ==

PKVL_MODE_STS_GROUP_OK)

drivers/fpga/dfl-n3000-nios.c new file +45 -0
+return true;
+
+return false;
+}
+
+static void dump_error_stat(struct dfl_n3000_nios *ns)
+{
+unsigned int val;
+
+if (regmap_read(ns->regmap, PKVL_A_MODE_STS, &val))
+return;
+
+dev_info(ns->dev, "PKVL_A_MODE_STS %x\n", val);
+
+if (regmap_read(ns->regmap, PKVL_B_MODE_STS, &val))
+return;
+
+dev_info(ns->dev, "PKVL_B_MODE_STS %x\n", val);
+}
+
+static int n3000_nios_init_done_check(struct dfl_n3000_nios *ns)
+{
+struct device *dev = ns->dev;
+unsigned int val, mode;
+int ret;
+
+/*
+ * this SPI is shared by Nios core inside FPGA, Nios will use this SPI
+ * master to do some one time initialization after power up, and then
+ * release the control to OS. driver needs to poll on INIT_DONE to
+ * see when driver could take the control.
+ *
+ * Please note that after 3.x.x version, INIT_START is introduced, so
+ * driver needs to trigger START firstly and then check INIT_DONE.
+ */
+
+ret = regmap_read(ns->regmap, NIOS_FW_VERSION, &val);
+if (ret)
+return ret;
+
+/*
+ * If Nios version register is totally uninitialized(== 0x0), then the
+ * Nios firmware is missing. So host could take control of SPI master
+ * safely, but initialization work for Nios is not done. This is an
+ * issue of FPGA image. We didn't error out because we need SPI
Quoted 8 lines
Quoted 6 lines

master
+ * to reprogram a new image.
+ */

drivers/fpga/dfl-n3000-nios.c new file +2 -0
+if (val == 0) {
+dev_warn(dev, "Nios version reg = 0x%x, skip INIT_DONE

check, but PKVL may be uninitialized\n",

Seems no place introduced the PKVL or just use retimer to replace PKVL?

Yes.

Quoted 17 lines

Thanks
Hao

Quoted 13 lines

+ val);

drivers/fpga/dfl-n3000-nios.c new file +12 -0
+return 0;
+}
+
+if (FIELD_GET(NIOS_FW_VERSION_MAJOR, val) >= 3) {
+/* read NIOS_INIT to check if PKVL INIT done or not */
+ret = regmap_read(ns->regmap, NIOS_INIT, &val);
+if (ret)
+return ret;
+
+/* check if PKVLs are initialized already */
+if (val & NIOS_INIT_DONE || val & NIOS_INIT_START)
+goto nios_init_done;
Quoted 8 lines

drivers/fpga/dfl-n3000-nios.c line 235

Quoted 5 lines
drivers/fpga/dfl-n3000-nios.c new file +5 -0
+
+/* configure FEC mode per module param */
+val = NIOS_INIT_START;
+
+/* FEC mode will be ignored by hardware in 10G mode */

So always no fec mode from hardware? Or software should force it in 10G mode? Or 10G mode, below code can be skipped?

I'm not sure about the 10G mode behavior since HW ignores the config in this case. I'll check it.

Thanks
Yilun

Quoted 249 lines
Quoted 249 lines
drivers/fpga/dfl-n3000-nios.c new file +45 -0
+if (!strcmp(fec_mode, "no"))
+mode = REQ_FEC_MODE_NO;
+else if (!strcmp(fec_mode, "kr"))
+mode = REQ_FEC_MODE_KR;
+else if (!strcmp(fec_mode, "rs"))
+mode = REQ_FEC_MODE_RS;
+else
+return -EINVAL;
+
+/* set the same FEC mode for all links */
+val |= FIELD_PREP(REQ_FEC_MODE_A0_MSK, mode) |
+       FIELD_PREP(REQ_FEC_MODE_A1_MSK, mode) |
+       FIELD_PREP(REQ_FEC_MODE_A2_MSK, mode) |
+       FIELD_PREP(REQ_FEC_MODE_A3_MSK, mode) |
+       FIELD_PREP(REQ_FEC_MODE_B0_MSK, mode) |
+       FIELD_PREP(REQ_FEC_MODE_B1_MSK, mode) |
+       FIELD_PREP(REQ_FEC_MODE_B2_MSK, mode) |
+       FIELD_PREP(REQ_FEC_MODE_B3_MSK, mode);
+
+ret = regmap_write(ns->regmap, NIOS_INIT, val);
+if (ret)
+return ret;
+}
+
+nios_init_done:
+/* polls on NIOS_INIT_DONE */
+ret = regmap_read_poll_timeout(ns->regmap, NIOS_INIT, val,
+       val & NIOS_INIT_DONE,
+       NIOS_INIT_TIME_INTV,
+       NIOS_INIT_TIMEOUT);
+if (ret) {
+dev_err(dev, "NIOS_INIT_DONE %s\n",
+(ret == -ETIMEDOUT) ? "timed out" : "check error");
+goto dump_sts;
+}
+
+/*
+ * after INIT_DONE is detected, it still needs to check if any ERR
+ * detected.
+ * We won't error out here even if error detected. Nios will release
+ * spi controller when INIT_DONE is set, so driver could continue to
+ * initialize spi controller device.
+ */
+if (init_error_detected(ns)) {
+dev_warn(dev, "NIOS_INIT_DONE OK, but err found during

init\n");

drivers/fpga/dfl-n3000-nios.c new file +35 -0
+goto dump_sts;
+}
+return 0;
+
+dump_sts:
+dump_error_stat(ns);
+
+return ret;
+}
+
+struct spi_board_info m10_n3000_info = {
+.modalias = "m10-n3000",
+.max_speed_hz = 12500000,
+.bus_num = 0,
+.chip_select = 0,
+};
+
+static int create_altera_spi_controller(struct dfl_n3000_nios *ns)
+{
+struct altera_spi_platform_data pdata = { 0 };
+struct platform_device_info pdevinfo = { 0 };
+void __iomem *base = ns->base;
+u64 v;
+
+v = readq(base + NIOS_SPI_PARAM);
+
+pdata.mode_bits = SPI_CS_HIGH;
+if (FIELD_GET(PARAM_CLK_POL, v))
+pdata.mode_bits |= SPI_CPOL;
+if (FIELD_GET(PARAM_CLK_PHASE, v))
+pdata.mode_bits |= SPI_CPHA;
+
+pdata.num_chipselect = FIELD_GET(PARAM_NUM_CS, v);
+pdata.bits_per_word_mask =
+SPI_BPW_RANGE_MASK(1, FIELD_GET(PARAM_DATA_WIDTH,

v));

drivers/fpga/dfl-n3000-nios.c new file +32 -0
+
+pdata.num_devices = 1;
+pdata.devices = &m10_n3000_info;
+
+dev_dbg(ns->dev, "%s cs %u bpm 0x%x mode 0x%x\n", __func__,
+pdata.num_chipselect, pdata.bits_per_word_mask,
+pdata.mode_bits);
+
+pdevinfo.name = "subdev_spi_altera";
+pdevinfo.id = PLATFORM_DEVID_AUTO;
+pdevinfo.parent = ns->dev;
+pdevinfo.data = &pdata;
+pdevinfo.size_data = sizeof(pdata);
+
+ns->altera_spi = platform_device_register_full(&pdevinfo);
+return PTR_ERR_OR_ZERO(ns->altera_spi);
+}
+
+static void destroy_altera_spi_controller(struct dfl_n3000_nios *ns)
+{
+platform_device_unregister(ns->altera_spi);
+}
+
+/* ns is the abbreviation of nios_spi */
+static int ns_poll_stat_timeout(void __iomem *base, u64 *v)
+{
+int loops = NS_REGBUS_WAIT_TIMEOUT;
+
+/*
+ * We don't use the time based timeout here for performance.
+ *
+ * The image reprograming engine is on max10 bmc chip, which is the

spi
+ * device connected to altera spi controller. So the regbus read/write
+ * is on the critical path of PAC N3000 image reprograming. And usually
+ * the state changes in few loops, the time based timeout checking will
+ * add too much overhead on it.
+ *
+ * Anyway, 10000 times loop is large enough.
+ */

drivers/fpga/dfl-n3000-nios.c new file +73 -0
+do {
+*v = readq(base + NIOS_SPI_STAT);
+if (*v & STAT_RW_VAL)
+break;
+cpu_relax();
+} while (--loops);
+
+return loops ? 0 : -ETIMEDOUT;
+}
+
+static int ns_reg_write(void *context, unsigned int reg, unsigned int val)
+{
+struct dfl_n3000_nios *ns = context;
+u64 v = 0;
+int ret;
+
+v |= FIELD_PREP(CTRL_CMD_MSK, CTRL_CMD_WR);
+v |= FIELD_PREP(CTRL_ADDR, reg);
+v |= FIELD_PREP(CTRL_WR_DATA, val);
+writeq(v, ns->base + NIOS_SPI_CTRL);
+
+ret = ns_poll_stat_timeout(ns->base, &v);
+if (ret)
+dev_err(ns->dev, "fail to write reg 0x%x val 0x%x: %d\n",
+reg, val, ret);
+
+return ret;
+}
+
+static int ns_reg_read(void *context, unsigned int reg, unsigned int *val)
+{
+struct dfl_n3000_nios *ns = context;
+u64 v = 0;
+int ret;
+
+v |= FIELD_PREP(CTRL_CMD_MSK, CTRL_CMD_RD);
+v |= FIELD_PREP(CTRL_ADDR, reg);
+writeq(v, ns->base + NIOS_SPI_CTRL);
+
+ret = ns_poll_stat_timeout(ns->base, &v);
+if (ret)
+dev_err(ns->dev, "fail to read reg 0x%x: %d\n", reg, ret);
+else
+*val = FIELD_GET(STAT_RD_DATA, v);
+
+return ret;
+}
+
+static const struct regmap_config ns_regbus_cfg = {
+.reg_bits = 32,
+.reg_stride = 4,
+.val_bits = 32,
+.fast_io = true,
+
+.reg_write = ns_reg_write,
+.reg_read = ns_reg_read,
+};
+
+static int dfl_n3000_nios_probe(struct dfl_device *dfl_dev)
+{
+struct device *dev = &dfl_dev->dev;
+struct dfl_n3000_nios *ns;
+int ret;
+
+ns = devm_kzalloc(dev, sizeof(*ns), GFP_KERNEL);
+if (!ns)
+return -ENOMEM;
+
+dev_set_drvdata(&dfl_dev->dev, ns);
+
+ns->dev = dev;
+
+ns->base = devm_ioremap_resource(&dfl_dev->dev, &dfl_dev-
Quoted 1 line

mmio_res);

drivers/fpga/dfl-n3000-nios.c new file +48 -1
+if (IS_ERR(ns->base))
+return PTR_ERR(ns->base);
+
+ns->regmap = devm_regmap_init(dev, NULL, ns, &ns_regbus_cfg);
+if (IS_ERR(ns->regmap))
+return PTR_ERR(ns->regmap);
+
+ret = n3000_nios_init_done_check(ns);
+if (ret)
+return ret;
+
+ret = create_altera_spi_controller(ns);
+if (ret)
+dev_err(dev, "altera spi controller create failed: %d\n", ret);
+
+return ret;
+}
+
+static void dfl_n3000_nios_remove(struct dfl_device *dfl_dev)
+{
+struct dfl_n3000_nios *ns = dev_get_drvdata(&dfl_dev->dev);
+
+destroy_altera_spi_controller(ns);
+}
+
+#define FME_FEATURE_ID_N3000_NIOS0xd
+
+static const struct dfl_device_id dfl_n3000_nios_ids[] = {
+{ FME_ID, FME_FEATURE_ID_N3000_NIOS },
+{ }
+};
+
+static struct dfl_driver dfl_n3000_nios_driver = {
+.drv= {
+.name       = "dfl-n3000-nios",
+.dev_groups = n3000_nios_groups,
+},
+.id_table = dfl_n3000_nios_ids,
+.probe   = dfl_n3000_nios_probe,
+.remove  = dfl_n3000_nios_remove,
+};
+
+module_dfl_driver(dfl_n3000_nios_driver);
+
+MODULE_DEVICE_TABLE(dfl, dfl_n3000_nios_ids);
+MODULE_DESCRIPTION("DFL N3000 Nios private feature driver");
+MODULE_AUTHOR("Intel Corporation");
+MODULE_LICENSE("GPL v2");
--

2.7.4