Skip to contentSkip to conversation

mlxbf_gige: add BlueField-3 support

PATCH Superseded David Thompson 4 participants 19 messages Nov 9, 2022 – Nov 17, 2022

Subsystems (2)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

v2 cover mlxbf_gige: add BlueField-3 support

This patch series adds driver logic to the "mlxbf_gige" Ethernet driver in order to support the third generation BlueField SoC (BF3). The existing "mlxbf_gige" driver is extended with BF3-specific logic and run-time decisions are made by the driver depending on the SoC generation (BF2 vs. BF3).

The BF3 SoC is similar to BF2 SoC with regards to transmit and receive packet processing:
* Driver rings usage; consumer & producer indices
* Single queue for receive and transmit
* DMA operation

The differences between BF3 and BF2 are:
* In addition to supporting 1Gbps interface speed, the BF3 SoC adds support for 10Mbps and 100Mbps interface speeds
* BF3 requires SerDes config logic to support its SGMII interface
* BF3 adds support for "ethtool -s" for interface speed config
* BF3 utilizes different MDIO logic for accessing the board-level PHY device

Testing
- Successful build of kernel for ARM64, ARM32, X86_64
- Tested ARM64 build on FastModels, Palladium, SoC

v1 -> v2:
- Fixed build failures in "build_32bit" and "build_allmodconfig_warn"
- Removed use of spinlock in BF3 "adjust_link" callback
- Added use of ARRAY_SIZE() where appropriate
- Added "set_link_ksettings" ethtool callback for BF2 and BF3

David Thompson (4):
mlxbf_gige: add MDIO support for BlueField-3
mlxbf_gige: support 10M/100M/1G speeds on BlueField-3
mlxbf_gige: add BlueField-3 Serdes configuration
mlxbf_gige: add "set_link_ksettings" ethtool callback

 .../net/ethernet/mellanox/mlxbf_gige/Makefile |    3 +-
 .../ethernet/mellanox/mlxbf_gige/mlxbf_gige.h |   31 +-
 .../mellanox/mlxbf_gige/mlxbf_gige_ethtool.c  |    1 +
 .../mellanox/mlxbf_gige/mlxbf_gige_main.c     |  164 ++-
 .../mellanox/mlxbf_gige/mlxbf_gige_mdio.c     |  185 +--
 .../mellanox/mlxbf_gige/mlxbf_gige_mdio_bf2.h |   53 +
 .../mellanox/mlxbf_gige/mlxbf_gige_mdio_bf3.h |   54 +
 .../mellanox/mlxbf_gige/mlxbf_gige_regs.h     |   22 +
 .../mellanox/mlxbf_gige/mlxbf_gige_uphy.c     | 1173 +++++++++++++++++
 .../mellanox/mlxbf_gige/mlxbf_gige_uphy.h     |  398 ++++++
 10 files changed, 1987 insertions(+), 97 deletions(-)
 create mode 100644 drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_mdio_bf2.h
 create mode 100644 drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_mdio_bf3.h
 create mode 100644 drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_uphy.c
 create mode 100644 drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_uphy.h
Signature · 1 line
2.30.1
#1 lore raw

PATCH v2 1/4 mlxbf_gige: add MDIO support for BlueField-3

This patch adds initial MDIO support for the BlueField-3 SoC. Separate header files for the BlueField-2 and the BlueField-3 SoCs have been created. These header files hold the SoC-specific MDIO macros since the register offsets and bit fields have changed. Also, in BlueField-3 there is a separate register for writing and reading the MDIO data. Finally, instead of having "if" statements everywhere to differentiate between SoC-specific logic, a mlxbf_gige_mdio_gw_t struct was created for this purpose.

---
 .../ethernet/mellanox/mlxbf_gige/mlxbf_gige.h |  19 ++
 .../mellanox/mlxbf_gige/mlxbf_gige_main.c     |   2 +
 .../mellanox/mlxbf_gige/mlxbf_gige_mdio.c     | 172 +++++++++++++-----
 .../mellanox/mlxbf_gige/mlxbf_gige_mdio_bf2.h |  53 ++++++
 .../mellanox/mlxbf_gige/mlxbf_gige_mdio_bf3.h |  54 ++++++
 .../mellanox/mlxbf_gige/mlxbf_gige_regs.h     |   1 +
 6 files changed, 251 insertions(+), 50 deletions(-)
 create mode 100644 drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_mdio_bf2.h
 create mode 100644 drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_mdio_bf3.h
drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige.h +19 -0
index 5a1027b07215..421a0b1b766c 100644
@@ -67,6 +67,23 @@ struct mlxbf_gige_stats {
 	u64 rx_filter_discard_pkts;
 };
 
+struct mlxbf_gige_reg_param {
+	u32 mask;
+	u32 shift;
+};
+
+struct mlxbf_gige_mdio_gw {
+	u32 gw_address;
+	u32 read_data_address;
+	struct mlxbf_gige_reg_param busy;
+	struct mlxbf_gige_reg_param write_data;
+	struct mlxbf_gige_reg_param read_data;
+	struct mlxbf_gige_reg_param devad;
+	struct mlxbf_gige_reg_param partad;
+	struct mlxbf_gige_reg_param opcode;
+	struct mlxbf_gige_reg_param st1;
+};
+
 struct mlxbf_gige {
 	void __iomem *base;
 	void __iomem *llu_base;
@@ -102,6 +119,8 @@ struct mlxbf_gige {
 	u8 valid_polarity;
 	struct napi_struct napi;
 	struct mlxbf_gige_stats stats;
+	u8 hw_version;
+	struct mlxbf_gige_mdio_gw *mdio_gw;
 };
 
 /* Rx Work Queue Element definitions */
drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_main.c +2 -0
index 2292d63a279c..e08c07e914c1 100644
@@ -315,6 +315,8 @@ static int mlxbf_gige_probe(struct platform_device *pdev)
 
 	spin_lock_init(&priv->lock);
 
+	priv->hw_version = readq(base + MLXBF_GIGE_VERSION);
+
 	/* Attach MDIO device */
 	err = mlxbf_gige_mdio_probe(pdev, priv);
 	if (err)
drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_mdio.c +122 -50

256 lines not on this page · raw message

drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_mdio_bf2.h new file +53 -0

53 lines not on this page · raw message

drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_mdio_bf3.h new file +54 -0

54 lines not on this page · raw message

drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_regs.h +1 -0
index 7be3a793984d..8d52dbef4adf 100644
@@ -10,6 +10,7 @@
 
 #define MLXBF_GIGE_VERSION                            0x0000
 #define MLXBF_GIGE_VERSION_BF2                        0x0
+#define MLXBF_GIGE_VERSION_BF3                        0x1
 #define MLXBF_GIGE_STATUS                             0x0010
 #define MLXBF_GIGE_STATUS_READY                       BIT(0)
 #define MLXBF_GIGE_INT_STATUS                         0x0028
Signature · 1 line
2.30.1
Reviewed-by on v2 1/4
#2 lore raw

Re: [PATCH net-next v2 1/4] mlxbf_gige: add MDIO support for BlueField-3

Quoted 10 lines · David Thompson

This patch adds initial MDIO support for the BlueField-3 SoC. Separate header files for the BlueField-2 and the BlueField-3 SoCs have been created. These header files hold the SoC-specific MDIO macros since the register offsets and bit fields have changed. Also, in BlueField-3 there is a separate register for writing and reading the MDIO data. Finally, instead of having "if" statements everywhere to differentiate between SoC-specific logic, a mlxbf_gige_mdio_gw_t struct was created for this purpose.

Quoted 2 lines

Andrew

#3 lore raw

PATCH v2 2/4 mlxbf_gige: support 10M/100M/1G speeds on BlueField-3

In reply to David Thompson

The BlueField-3 OOB interface supports 10Mbps, 100Mbps, and 1Gbps speeds. The external PHY is responsible for autonegotiating the speed with the link partner. Once the autonegotiation is done, the BlueField PLU needs to be configured accordingly.

This patch does two things:
1) Initialize the advertised control flow/duplex/speed in the probe based on the BlueField SoC generation (2 or 3)
2) Adjust the PLU speed config in the PHY interrupt handler

---
 .../ethernet/mellanox/mlxbf_gige/mlxbf_gige.h |   8 ++
 .../mellanox/mlxbf_gige/mlxbf_gige_main.c     | 105 +++++++++++++++---
 .../mellanox/mlxbf_gige/mlxbf_gige_regs.h     |  21 ++++
 3 files changed, 119 insertions(+), 15 deletions(-)
drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige.h +8 -0
index 421a0b1b766c..a453b9cd9033 100644
@@ -14,6 +14,7 @@
 #include <linux/irqreturn.h>
 #include <linux/netdevice.h>
 #include <linux/irq.h>
+#include <linux/phy.h>
 
 /* The silicon design supports a maximum RX ring size of
  * 32K entries. Based on current testing this maximum size
@@ -84,6 +85,12 @@ struct mlxbf_gige_mdio_gw {
 	struct mlxbf_gige_reg_param st1;
 };
 
+struct mlxbf_gige_link_cfg {
+	void (*set_phy_link_mode)(struct phy_device *phydev);
+	void (*adjust_link)(struct net_device *netdev);
+	phy_interface_t phy_mode;
+};
+
 struct mlxbf_gige {
 	void __iomem *base;
 	void __iomem *llu_base;
@@ -121,6 +128,7 @@ struct mlxbf_gige {
 	struct mlxbf_gige_stats stats;
 	u8 hw_version;
 	struct mlxbf_gige_mdio_gw *mdio_gw;
+	int prev_speed;
 };
 
 /* Rx Work Queue Element definitions */
drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_main.c +90 -15

128 lines not on this page · raw message

drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_regs.h +21 -0
index 8d52dbef4adf..cd0973229c9b 100644
@@ -8,6 +8,8 @@
 #ifndef __MLXBF_GIGE_REGS_H__
 #define __MLXBF_GIGE_REGS_H__
 
+#include <linux/bitfield.h>
+
 #define MLXBF_GIGE_VERSION                            0x0000
 #define MLXBF_GIGE_VERSION_BF2                        0x0
 #define MLXBF_GIGE_VERSION_BF3                        0x1
@@ -78,4 +80,23 @@
  */
 #define MLXBF_GIGE_MMIO_REG_SZ                        (MLXBF_GIGE_MAC_CFG + 8)
 
+#define MLXBF_GIGE_PLU_TX_REG0                        0x80
+#define MLXBF_GIGE_PLU_TX_IPG_SIZE_MASK               GENMASK(11, 0)
+#define MLXBF_GIGE_PLU_TX_SGMII_MODE_MASK             GENMASK(15, 14)
+
+#define MLXBF_GIGE_PLU_RX_REG0                        0x10
+#define MLXBF_GIGE_PLU_RX_SGMII_MODE_MASK             GENMASK(25, 24)
+
+#define MLXBF_GIGE_1G_SGMII_MODE                      0x0
+#define MLXBF_GIGE_10M_SGMII_MODE                     0x1
+#define MLXBF_GIGE_100M_SGMII_MODE                    0x2
+
+/* ipg_size default value for 1G is fixed by HW to 11 + End = 12.
+ * So for 100M it is 12 * 10 - 1 = 119
+ * For 10M, it is 12 * 100 - 1 = 1199
+ */
+#define MLXBF_GIGE_1G_IPG_SIZE                        11
+#define MLXBF_GIGE_100M_IPG_SIZE                      119
+#define MLXBF_GIGE_10M_IPG_SIZE                       1199
+
 #endif /* !defined(__MLXBF_GIGE_REGS_H__) */
Signature · 1 line
2.30.1
Reviewed-by on v2 2/4
#4 lore raw

Re: [PATCH net-next v2 2/4] mlxbf_gige: support 10M/100M/1G speeds on BlueField-3

Quoted 5 lines · David Thompson

The BlueField-3 OOB interface supports 10Mbps, 100Mbps, and 1Gbps speeds. The external PHY is responsible for autonegotiating the speed with the link partner. Once the autonegotiation is done, the BlueField PLU needs to be configured accordingly.

Quoted 7 lines

This patch does two things:
1) Initialize the advertised control flow/duplex/speed in the probe based on the BlueField SoC generation (2 or 3)
2) Adjust the PLU speed config in the PHY interrupt handler

Andrew

#5 lore raw

PATCH v2 3/4 mlxbf_gige: add BlueField-3 Serdes configuration

In reply to David Thompson

The BlueField-3 out-of-band Ethernet interface requires SerDes configuration. There are two aspects to this:

Configuration of PLL:
1) Initialize UPHY registers to values dependent on p1clk clock
2) Load PLL best known values via the gateway register
3) Set the fuses to tune up the SerDes voltage
4) Lock the PLL
5) Get the lanes out of functional reset.
6) Configure the UPHY microcontroller via gateway reads/writes

Configuration of lanes:
1) Configure and open TX lanes
2) Configure and open RX lanes

---
 .../net/ethernet/mellanox/mlxbf_gige/Makefile |    3 +-
 .../ethernet/mellanox/mlxbf_gige/mlxbf_gige.h |    4 +-
 .../mellanox/mlxbf_gige/mlxbf_gige_main.c     |   57 +-
 .../mellanox/mlxbf_gige/mlxbf_gige_mdio.c     |   37 -
 .../mellanox/mlxbf_gige/mlxbf_gige_uphy.c     | 1173 +++++++++++++++++
 .../mellanox/mlxbf_gige/mlxbf_gige_uphy.h     |  398 ++++++
 6 files changed, 1628 insertions(+), 44 deletions(-)
 create mode 100644 drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_uphy.c
 create mode 100644 drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_uphy.h
drivers/net/ethernet/mellanox/mlxbf_gige/Makefile +2 -1
index a97c2bef846b..524af17cad9c 100644
@@ -7,4 +7,5 @@ mlxbf_gige-y := mlxbf_gige_ethtool.o \
 		mlxbf_gige_main.o \
 		mlxbf_gige_mdio.o \
 		mlxbf_gige_rx.o   \
-		mlxbf_gige_tx.o
+		mlxbf_gige_tx.o   \
+		mlxbf_gige_uphy.o
drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige.h +3 -1
index a453b9cd9033..e9bd09ee0b1f 100644
@@ -100,6 +100,7 @@ struct mlxbf_gige {
 	struct platform_device *pdev;
 	void __iomem *mdio_io;
 	void __iomem *clk_io;
+	void __iomem *fuse_gw_io;
 	struct mii_bus *mdiobus;
 	spinlock_t lock;      /* for packet processing indices */
 	u16 rx_q_entries;
@@ -166,7 +167,8 @@ enum mlxbf_gige_res {
 	MLXBF_GIGE_RES_GPIO0,
 	MLXBF_GIGE_RES_LLU,
 	MLXBF_GIGE_RES_PLU,
-	MLXBF_GIGE_RES_CLK
+	MLXBF_GIGE_RES_CLK,
+	MLXBF_GIGE_RES_FUSE_GW
 };
 
 /* Version of register data returned by mlxbf_gige_get_regs() */
drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_main.c +52 -5

92 lines not on this page · raw message

drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_mdio.c +0 -37
index 7ac06fd31011..043edf57e36b 100644
@@ -113,24 +113,6 @@ static struct mlxbf_gige_mdio_gw mlxbf_gige_mdio_gw_t[] = {
 /* Busy bit is set by software and cleared by hardware */
 #define MLXBF_GIGE_MDIO_SET_BUSY	0x1
 
-#define MLXBF_GIGE_BF2_COREPLL_ADDR 0x02800c30
-#define MLXBF_GIGE_BF2_COREPLL_SIZE 0x0000000c
-#define MLXBF_GIGE_BF3_COREPLL_ADDR 0x13409824
-#define MLXBF_GIGE_BF3_COREPLL_SIZE 0x00000010
-
-static struct resource corepll_params[] = {
-	[MLXBF_GIGE_VERSION_BF2] = {
-		.start = MLXBF_GIGE_BF2_COREPLL_ADDR,
-		.end = MLXBF_GIGE_BF2_COREPLL_ADDR + MLXBF_GIGE_BF2_COREPLL_SIZE - 1,
-		.name = "COREPLL_RES"
-	},
-	[MLXBF_GIGE_VERSION_BF3] = {
-		.start = MLXBF_GIGE_BF3_COREPLL_ADDR,
-		.end = MLXBF_GIGE_BF3_COREPLL_ADDR + MLXBF_GIGE_BF3_COREPLL_SIZE - 1,
-		.name = "COREPLL_RES"
-	}
-};
-
 /* Returns core clock i1clk in Hz */
 static u64 calculate_i1clk(struct mlxbf_gige *priv)
 {
@@ -294,31 +276,12 @@ static void mlxbf_gige_mdio_cfg(struct mlxbf_gige *priv)
 int mlxbf_gige_mdio_probe(struct platform_device *pdev, struct mlxbf_gige *priv)
 {
 	struct device *dev = &pdev->dev;
-	struct resource *res;
 	int ret;
 
-	if (priv->hw_version > MLXBF_GIGE_VERSION_BF3)
-		return -ENODEV;
-
 	priv->mdio_io = devm_platform_ioremap_resource(pdev, MLXBF_GIGE_RES_MDIO9);
 	if (IS_ERR(priv->mdio_io))
 		return PTR_ERR(priv->mdio_io);
 
-	/* clk resource shared with other drivers so cannot use
-	 * devm_platform_ioremap_resource
-	 */
-	res = platform_get_resource(pdev, IORESOURCE_MEM, MLXBF_GIGE_RES_CLK);
-	if (!res) {
-		/* For backward compatibility with older ACPI tables, also keep
-		 * CLK resource internal to the driver.
-		 */
-		res = &corepll_params[priv->hw_version];
-	}
-
-	priv->clk_io = devm_ioremap(dev, res->start, resource_size(res));
-	if (!priv->clk_io)
-		return -ENOMEM;
-
 	priv->mdio_gw = &mlxbf_gige_mdio_gw_t[priv->hw_version];
 
 	mlxbf_gige_mdio_cfg(priv);
drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_uphy.c new file +1173 -0

1,173 lines not on this page · raw message

drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_uphy.h new file +398 -0

398 lines not on this page · raw message

Signature · 1 line
2.30.1
#6 lore raw

Re: [PATCH net-next v2 3/4] mlxbf_gige: add BlueField-3 Serdes configuration

Quoted 11 lines · David Thompson

The BlueField-3 out-of-band Ethernet interface requires SerDes configuration. There are two aspects to this:

Configuration of PLL:
1) Initialize UPHY registers to values dependent on p1clk clock
2) Load PLL best known values via the gateway register
3) Set the fuses to tune up the SerDes voltage
4) Lock the PLL
5) Get the lanes out of functional reset.
6) Configure the UPHY microcontroller via gateway reads/writes

Quoted 3 lines

Configuration of lanes:
1) Configure and open TX lanes
2) Configure and open RX lanes

I still don't like all these black magic tables in the driver.

But lets see what others say.

Andrew

maintainer
#7 lore raw

Re: [PATCH net-next v2 3/4] mlxbf_gige: add BlueField-3 Serdes configuration

Quoted 12 lines · On Thu, 10 Nov 2022 14:33:47 +0100 Andrew Lunn wrote:
Quoted 11 lines · David Thompson

The BlueField-3 out-of-band Ethernet interface requires SerDes configuration. There are two aspects to this:

Configuration of PLL:
1) Initialize UPHY registers to values dependent on p1clk clock
2) Load PLL best known values via the gateway register
3) Set the fuses to tune up the SerDes voltage
4) Lock the PLL
5) Get the lanes out of functional reset.
6) Configure the UPHY microcontroller via gateway reads/writes

Quoted 7 lines
Quoted 3 lines

Configuration of lanes:
1) Configure and open TX lanes
2) Configure and open RX lanes

I still don't like all these black magic tables in the driver.

But lets see what others say.

Well, the patch was marked as Changes Requested so it seems that DaveM concurs :) (I'm slightly desensitized to those tables because they happen in WiFi relatively often.)

The recommendation is to come up with a format for a binary file, load it via FW loader and then parse in the kernel?

We did have a recommendation against parsing FW files in the kernel at some point, too, but perhaps this is simple enough to pass.

Should this be shared infra? The problem is fairly common.

#8 lore raw

Re: [PATCH net-next v2 3/4] mlxbf_gige: add BlueField-3 Serdes configuration

Quoted 19 lines · Jakub Kicinski
Quoted 18 lines · On Thu, 10 Nov 2022 14:33:47 +0100 Andrew Lunn wrote:
Quoted 11 lines · David Thompson

The BlueField-3 out-of-band Ethernet interface requires SerDes configuration. There are two aspects to this:

Configuration of PLL:
1) Initialize UPHY registers to values dependent on p1clk clock
2) Load PLL best known values via the gateway register
3) Set the fuses to tune up the SerDes voltage
4) Lock the PLL
5) Get the lanes out of functional reset.
6) Configure the UPHY microcontroller via gateway reads/writes

Quoted 3 lines

Configuration of lanes:
1) Configure and open TX lanes
2) Configure and open RX lanes

I still don't like all these black magic tables in the driver.

Quoted 8 lines
Quoted 1 line

But lets see what others say.

Well, the patch was marked as Changes Requested so it seems that DaveM concurs :) (I'm slightly desensitized to those tables because they happen in WiFi relatively often.)

The recommendation is to come up with a format for a binary file, load it via FW loader and then parse in the kernel?

By FW loader you mean request_firmware() functionality ?

I am not advocating for black magic tables of course :), but how do we avoid them if request_firmware() will be an overkill to configure such a simple device? Express such data in a developer friendly c structures with somewhat sensible field names?

Quoted 5 lines

We did have a recommendation against parsing FW files in the kernel at some point, too, but perhaps this is simple enough to pass.

Should this be shared infra? The problem is fairly common.

Infrastructure to parse vendor Firmware ? we can't get vendors to agree on ethtool interface, you want them to agree on one firmware format :)?

BTW i don't think the issue here is firmware at all, this is device specific config space.

#9 lore raw

Re: [PATCH net-next v2 3/4] mlxbf_gige: add BlueField-3 Serdes configuration

Quoted 9 lines
Quoted 2 lines

The recommendation is to come up with a format for a binary file, load it via FW loader and then parse in the kernel?

By FW loader you mean request_firmware() functionality ?

I am not advocating for black magic tables of course :), but how do we avoid them if request_firmware() will be an overkill to configure such a simple device? Express such data in a developer friendly c structures with somewhat sensible field names?

Do you think anybody other than your company has the ability to change these values? Is there useful documentation about what they do, even if it is under NDA? Why would somebody actually need to change them?

Is here functionally here which you don't support but the community might like to add?

Expressing the data in a developer friendly C structure only really make sense if there is a small collection of developers out there who have the skills, documentation and maybe equipment to actually make meaningful changes.

I don't like making it harder to some clever people to hack new stuff into your drivers, but there are so few contributions from the community to your drivers that it might as well be black magic, and just load the values from a file.

Andrew

maintainer
#10 lore raw

Re: [PATCH net-next v2 3/4] mlxbf_gige: add BlueField-3 Serdes configuration

Quoted 11 lines · On Sat, 12 Nov 2022 16:53:28 +0100 Andrew Lunn wrote:

Do you think anybody other than your company has the ability to change these values? Is there useful documentation about what they do, even if it is under NDA? Why would somebody actually need to change them?

Is here functionally here which you don't support but the community might like to add?

Expressing the data in a developer friendly C structure only really make sense if there is a small collection of developers out there who have the skills, documentation and maybe equipment to actually make meaningful changes.

+1, even if the tables are not "FW as in compiled code" it's still "FW as in opaque blob". Plus if the format is well known and documented modifying the files is not harder than changing the kernel.

I think the applicability would be much wider. I will definitely make the WiFi people use this mechanism:

drivers/net/wireless/realtek/rtw89/rtw8852a_table.c

ugh.

Quoted 4 lines

I don't like making it harder to some clever people to hack new stuff into your drivers, but there are so few contributions from the community to your drivers that it might as well be black magic, and just load the values from a file.

maintainer
#11 lore raw

Re: [PATCH net-next v2 3/4] mlxbf_gige: add BlueField-3 Serdes configuration

In reply to Saeed Mahameed
Quoted 8 lines · On Sat, 12 Nov 2022 01:52:47 -0800 Saeed Mahameed wrote:
Quoted 6 lines

Well, the patch was marked as Changes Requested so it seems that DaveM concurs :) (I'm slightly desensitized to those tables because they happen in WiFi relatively often.)

The recommendation is to come up with a format for a binary file, load it via FW loader and then parse in the kernel?

By FW loader you mean request_firmware() functionality ?

Yes, that's what I meant.

Quoted 4 lines

I am not advocating for black magic tables of course :), but how do we avoid them if request_firmware() will be an overkill to configure such a simple device? Express such data in a developer friendly c structures with somewhat sensible field names?

I don't feel particularly strongly but seems like something worth exploring. A minor advantage is that once the init is done the tables can be discarded from memory.

Quoted 7 lines
Quoted 4 lines

We did have a recommendation against parsing FW files in the kernel at some point, too, but perhaps this is simple enough to pass.

Should this be shared infra? The problem is fairly common.

Infrastructure to parse vendor Firmware ? we can't get vendors to agree on ethtool interface, you want them to agree on one firmware format :)?

We can keep the table format pretty much as is. What I had in mind was basically creating a binary file format with u64 address, and u64 data. Plus file sections to pack multiple tables into one file. Pretty pleasant coding exercise if you ask me :)

Quoted 2 lines

BTW i don't think the issue here is firmware at all, this is device specific config space.

#12 lore raw

Re: [PATCH net-next v2 3/4] mlxbf_gige: add BlueField-3 Serdes configuration

Quoted 8 lines

I am not advocating for black magic tables of course :), but how do we

Quoted 3 lines

avoid them if request_firmware() will be an overkill to configure such a simple device? Express such data in a developer friendly c structures with somewhat sensible field names?

I don't feel particularly strongly but seems like something worth exploring. A minor advantage is that once the init is done the tables can be discarded from memory.

I wondered about that, but i'm not sure initdata works for modules, and for hot pluggable devices like PCIe, you never know when another one might appear and you need the tables.

Andrew

maintainer
#13 lore raw

Re: [PATCH net-next v2 3/4] mlxbf_gige: add BlueField-3 Serdes configuration

Quoted 5 lines · On Tue, 15 Nov 2022 02:06:19 +0100 Andrew Lunn wrote:
Quoted 5 lines

I am not advocating for black magic tables of course :), but how do we

Quoted 3 lines

avoid them if request_firmware() will be an overkill to configure such a simple device? Express such data in a developer friendly c structures with somewhat sensible field names?

Quoted 7 lines
Quoted 3 lines

I don't feel particularly strongly but seems like something worth exploring. A minor advantage is that once the init is done the tables can be discarded from memory.

I wondered about that, but i'm not sure initdata works for modules, and for hot pluggable devices like PCIe, you never know when another one might appear and you need the tables.

Right, I meant that the request_firmware() version can discard the tables. I shouldn't have said tables :)

maintainer
#14 lore raw

Re: [PATCH net-next v2 3/4] mlxbf_gige: add BlueField-3 Serdes configuration

Quoted 11 lines · On Mon, 14 Nov 2022 17:13:05 -0800 Jakub Kicinski wrote:
Quoted 7 lines · On Tue, 15 Nov 2022 02:06:19 +0100 Andrew Lunn wrote:
Quoted 3 lines

I don't feel particularly strongly but seems like something worth exploring. A minor advantage is that once the init is done the tables can be discarded from memory.

I wondered about that, but i'm not sure initdata works for modules, and for hot pluggable devices like PCIe, you never know when another one might appear and you need the tables.

Right, I meant that the request_firmware() version can discard the tables. I shouldn't have said tables :)

Saeed, David, are you looking into this? The problem come up again in a Realtek USB conversation.

The task is so small and well defined I'm pretty sure I can get some aspiring kernel developer at Meta to knock it off in a few days.

FWIW the structure of a file I had in mind would be something like this:

# Section 0 - strings (must be section 0)

 # header
 u32 type:   1   # string section
 u32 length: n   # length excluding header and pads
 u32 name:   0   # offset to the name in str section
 u32 pad:    0   # align to 8B
 # data
 .str\0table_abc\0table_def\0some_other_string\0
 # pad, align to 8B
 \0\0\0\0\0\0\0

# Section 1 - table_abc

 # header
 u32 type:   2   # 32b/32b table
 u32 length: 64  # length excluding header and pads
 u32 name:   5   # offset to the name in str section
 u32 pad:    0
 # data
 [ 0x210, 0xc00ff ]
 [ 0x214, 0xffeee ]
 [ 0x218, 0xdeaddd ]
 [ 0x21c, 0xc4ee5e ]
 [ 0x220, 0xc00ff ]
 [ 0x224, 0xffeee ]
 [ 0x228, 0xdeaddd ]
 [ 0x22c, 0xc4ee5e ]

etc.

Use:

	struct fw_table32 *abc, *def;

	fw = request_firmware("whatever_name.ftb");
	
	abc = fw_table_get(fw, "table_abc");
	/* use abc */

	def = fw_table_get(fw, "table_def");
	/* use def */

	release_firmware(fw)
#15 lore raw

Re: [PATCH net-next v2 3/4] mlxbf_gige: add BlueField-3 Serdes configuration

Quoted 10 lines · Jakub Kicinski
Quoted 9 lines · On Mon, 14 Nov 2022 17:13:05 -0800 Jakub Kicinski wrote:
Quoted 7 lines · On Tue, 15 Nov 2022 02:06:19 +0100 Andrew Lunn wrote:
Quoted 3 lines

I don't feel particularly strongly but seems like something worth exploring. A minor advantage is that once the init is done the tables can be discarded from memory.

I wondered about that, but i'm not sure initdata works for modules, and for hot pluggable devices like PCIe, you never know when another one might appear and you need the tables.

Quoted 9 lines
Quoted 2 lines

Right, I meant that the request_firmware() version can discard the tables. I shouldn't have said tables :)

Saeed, David, are you looking into this? The problem come up again in a Realtek USB conversation.

The task is so small and well defined I'm pretty sure I can get some aspiring kernel developer at Meta to knock it off in a few days.

Give me a couple of days and will let you know what's the verdict. I sent David and Asmaa the links to read up on Request Firmware. Still waiting for a response.

Quoted 31 lines

FWIW the structure of a file I had in mind would be something like this:

# Section 0 - strings (must be section 0)

# header
u32 type:   1   # string section
u32 length: n   # length excluding header and pads
u32 name:   0   # offset to the name in str section
u32 pad:    0   # align to 8B
# data
.str\0table_abc\0table_def\0some_other_string\0
# pad, align to 8B
\0\0\0\0\0\0\0

# Section 1 - table_abc

# header
u32 type:   2   # 32b/32b table
u32 length: 64  # length excluding header and pads
u32 name:   5   # offset to the name in str section
u32 pad:    0
# data
[ 0x210, 0xc00ff ]
[ 0x214, 0xffeee ]
[ 0x218, 0xdeaddd ]
[ 0x21c, 0xc4ee5e ]
[ 0x220, 0xc00ff ]
[ 0x224, 0xffeee ]
[ 0x228, 0xdeaddd ]
[ 0x22c, 0xc4ee5e ]

etc.

Quoted 8 lines

Use:

	struct fw_table32 *abc, *def;

	fw = request_firmware("whatever_name.ftb");
	
	abc = fw_table_get(fw, "table_abc");
	/* use abc */

abc is just a byte buffer ? right ?

Quoted 3 lines
	def = fw_table_get(fw, "table_def");
	/* use def */

And what goes here? any constraints on how the driver must interpret and handle abc/def blobs ?

Quoted 1 line

release_firmware(fw)

What if the same abc blob structure/table format is used to setup dynamic link properties, say via ethtool -s ? Then the whole request firmware will be redundant since "struct abc {};" must be defined in the driver src code.

I like the idea, i am just trying to figure how we are going to define it and how developers will differentiate between when to use this or when to use standard APIs to setup their devices.

maintainer
#16 lore raw

Re: [PATCH net-next v2 3/4] mlxbf_gige: add BlueField-3 Serdes configuration

Quoted 7 lines · On Wed, 16 Nov 2022 18:01:41 -0800 Saeed Mahameed wrote:
Quoted 5 lines
	fw = request_firmware("whatever_name.ftb");
	
	abc = fw_table_get(fw, "table_abc");
	/* use abc */

abc is just a byte buffer ? right ?

pointer to const struct fw_table32, which is just;

struct fw_table32 {
	u32 addr;
	u32 value;
};

Actually we need the length as well, so perhaps this is better:

struct fw_table32 {
	u32 addr;
	u32 val;
};
struct fw_table_result {
	uint cnt;
	union {
		const struct fw_table32 *tb32;
	};
};

User:

	struct fw_table_result tab;

	fw = request_firmware("whatever_name.ftb");
	if (!fw)
		...

	err = fw_table_get(fw, "table_abc", &tab);
	if (err)
		...
	for (i = 0; i < tab.cnt; i++) /* use abc */
		write_or_whatever(hw, tab.tb32[i].addr, tab.tb32[i].val);

	def = fw_table_get(fw, "table_def");
	if (err)
		...
	for (i = 0; i < tab.cnt; i++) /* use def */
		write_or_whatever(hw, tab.tb32[i].addr, tab.tb32[i].val);

	release_firmware(fw)
Quoted 6 lines
Quoted 3 lines
	def = fw_table_get(fw, "table_def");
	/* use def */

And what goes here? any constraints on how the driver must interpret and handle abc/def blobs ?

In the example I assumed a typical buffer with addr / value pairs. But we can define more table types as needed.

Quoted 5 lines
Quoted 1 line

release_firmware(fw)

What if the same abc blob structure/table format is used to setup dynamic link properties, say via ethtool -s ? Then the whole request firmware will be redundant since "struct abc {};" must be defined in the driver src code.

No complex structures, we'd be only targeting register init and small FW blobs (IOW { u32 addr; u32 val; } and { u8 val; }). Stuff which Windows? drivers tend to put into the code as static array.

Quoted 3 lines

I like the idea, i am just trying to figure how we are going to define it and how developers will differentiate between when to use this or when to use standard APIs to setup their devices.

#17 lore raw

PATCH v2 4/4 mlxbf_gige: add "set_link_ksettings" ethtool callback

In reply to David Thompson

This patch extends the "ethtool_ops" data structure to include the "set_link_ksettings" callback. This change enables configuration of the various interface speeds that the BlueField-3 supports (10Mbps, 100Mbps, and 1Gbps).

---
 drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_ethtool.c | 1 +
 drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_main.c    | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_ethtool.c +1 -0
index 41ebef25a930..253d7ad9b809 100644
@@ -135,4 +135,5 @@ const struct ethtool_ops mlxbf_gige_ethtool_ops = {
 	.nway_reset		= phy_ethtool_nway_reset,
 	.get_pauseparam		= mlxbf_gige_get_pauseparam,
 	.get_link_ksettings	= phy_ethtool_get_link_ksettings,
+	.set_link_ksettings	= phy_ethtool_set_link_ksettings,
 };
drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_main.c +1 -1
index 80060a54ba95..a9fa662e0665 100644
@@ -224,7 +224,7 @@ static int mlxbf_gige_stop(struct net_device *netdev)
 }
 
 static int mlxbf_gige_eth_ioctl(struct net_device *netdev,
-			       struct ifreq *ifr, int cmd)
+				struct ifreq *ifr, int cmd)
 {
 	if (!(netif_running(netdev)))
 		return -EINVAL;
Signature · 1 line
2.30.1
Reviewed-by on v2 4/4
#18 lore raw

Re: [PATCH net-next v2 4/4] mlxbf_gige: add "set_link_ksettings" ethtool callback

Quoted 27 lines · David Thompson

This patch extends the "ethtool_ops" data structure to include the "set_link_ksettings" callback. This change enables configuration of the various interface speeds that the BlueField-3 supports (10Mbps, 100Mbps, and 1Gbps).

---
 drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_ethtool.c | 1 +
 drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_main.c    | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_ethtool.c +1 -0
index 41ebef25a930..253d7ad9b809 100644
@@ -135,4 +135,5 @@ const struct ethtool_ops mlxbf_gige_ethtool_ops = {
 	.nway_reset		= phy_ethtool_nway_reset,
 	.get_pauseparam		= mlxbf_gige_get_pauseparam,
 	.get_link_ksettings	= phy_ethtool_get_link_ksettings,
+	.set_link_ksettings	= phy_ethtool_set_link_ksettings,
 };
drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_main.c +0 -0
index 80060a54ba95..a9fa662e0665 100644

drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_main.c static int mlxbf_gige_stop(struct net_device *netdev) · line 230

Quoted 8 lines
drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_main.c +1 -1
@@ -224,7 +224,7 @@ static int mlxbf_gige_stop(struct net_device *netdev)
 }
 
 static int mlxbf_gige_eth_ioctl(struct net_device *netdev,
-			       struct ifreq *ifr, int cmd)
+				struct ifreq *ifr, int cmd)
 {
 	if (!(netif_running(netdev)))
 		return -EINVAL;

White space changes should be in a separate patch.

With this fixed:

Andrew