Skip to contentSkip to conversation

mlx5 next updates 2020-11-20

PATCH In review Saeed Mahameed 6 participants 32 messages Nov 20, 2020 – Nov 30, 2020

Subsystems (5)
lore
Share & tools
lore

Saeed Mahameed

author maintainer
#0 lore raw

cover mlx5 next updates 2020-11-20

Hi,

This series includes trivial updates to mlx5 next branch
1) HW definition for upcoming features
2) Include files and general Cleanups
3) Add the upcoming BlueField-3 device ID
4) Define flow steering priority for VDPA
5) Export missing steering API for ULPs,
will be used later in VDPA driver, to create flow steering domain for VDPA queues.
6) ECPF (Embedded CPU Physical function) minor improvements for BlueField.

In case of no objection this series will be applied to mlx5-next, and sent as a pull request to both net-next and rdma-next trees.

Thanks,
Saeed.

---

Aya Levin (1):
net/mlx5: Expose IP-in-IP TX and RX capability bits

Chris Mi (2):
net/mlx5: Add sample offload hardware bits and structures net/mlx5: Add sampler destination type

Eli Cohen (2):
net/mlx5: Add VDPA priority to NIC RX namespace
net/mlx5: Export steering related functions

Eran Ben Elisha (1):
net/mlx5: Add ts_cqe_to_dest_cqn related bits

Meir Lichtinger (1):
net/mlx5: Update the list of the PCI supported devices

Muhammad Sammar (2):
net/mlx5: Check dr mask size against mlx5_match_param size net/mlx5: Add misc4 to mlx5_ifc_fte_match_param_bits

Parav Pandit (6):
net/mlx5: Avoid exposing driver internal command helpers net/mlx5: Update the hardware interface definition for vhca state net/mlx5: Make API mlx5_core_is_ecpf accept const pointer net/mlx5: Rename peer_pf to host_pf
net/mlx5: Enable host PF HCA after eswitch is initialized net/mlx5: Treat host PF vport as other (non eswitch manager) vport

Yishai Hadas (1):
net/mlx5: Expose other function ifc bits

 drivers/net/ethernet/mellanox/mlx5/core/cmd.c |  3 -
 .../mellanox/mlx5/core/diag/fs_tracepoint.c   |  3 +
 .../net/ethernet/mellanox/mlx5/core/ecpf.c    | 76 ++++++++++-----
 .../net/ethernet/mellanox/mlx5/core/ecpf.h    |  3 +
 .../mellanox/mlx5/core/esw/acl/helper.c       |  5 +-
 .../net/ethernet/mellanox/mlx5/core/eswitch.c | 29 +++++-
 .../net/ethernet/mellanox/mlx5/core/fs_cmd.c  | 57 ++++++-----
 .../net/ethernet/mellanox/mlx5/core/fs_core.c | 27 +++---
 .../net/ethernet/mellanox/mlx5/core/fs_core.h |  2 +-
 .../net/ethernet/mellanox/mlx5/core/main.c    | 19 ++--
 .../ethernet/mellanox/mlx5/core/mlx5_core.h   |  4 +
 .../ethernet/mellanox/mlx5/core/pagealloc.c   | 12 +--
 .../mellanox/mlx5/core/steering/dr_matcher.c  |  2 +-
 .../mellanox/mlx5/core/steering/dr_rule.c     |  3 +-
 .../mellanox/mlx5/core/steering/dr_types.h    |  1 +
 include/linux/mlx5/device.h                   |  8 ++
 include/linux/mlx5/driver.h                   |  8 +-
 include/linux/mlx5/fs.h                       |  7 +-
 include/linux/mlx5/mlx5_ifc.h                 | 94 +++++++++++++++++--
 include/uapi/rdma/mlx5_user_ioctl_cmds.h      |  2 +-
 20 files changed, 260 insertions(+), 105 deletions(-)
Signature · 1 line
2.26.2
author maintainer
#1 lore raw

PATCH 1/16 net/mlx5: Add sample offload hardware bits and structures

From: Chris Mi <[email protected]>

Hardware introduces flow sampler object for packet sampling. Add the offload hardware bits and structures.

---
 include/linux/mlx5/mlx5_ifc.h | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)
include/linux/mlx5/mlx5_ifc.h +29 -0
index 651591a2965d..65ea35af0527 100644
@@ -10657,11 +10657,13 @@ struct mlx5_ifc_affiliated_event_header_bits {
 enum {
 	MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY = BIT(0xc),
 	MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC = BIT(0x13),
+	MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_SAMPLER = BIT(0x20),
 };
 
 enum {
 	MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY = 0xc,
 	MLX5_GENERAL_OBJECT_TYPES_IPSEC = 0x13,
+	MLX5_GENERAL_OBJECT_TYPES_SAMPLER = 0x20,
 };
 
 enum {
@@ -10736,6 +10738,33 @@ struct mlx5_ifc_create_encryption_key_in_bits {
 	struct mlx5_ifc_encryption_key_obj_bits encryption_key_object;
 };
 
+struct mlx5_ifc_sampler_obj_bits {
+	u8         modify_field_select[0x40];
+
+	u8         table_type[0x8];
+	u8         level[0x8];
+	u8         reserved_at_50[0xf];
+	u8         ignore_flow_level[0x1];
+
+	u8         sample_ratio[0x20];
+
+	u8         reserved_at_80[0x8];
+	u8         sample_table_id[0x18];
+
+	u8         reserved_at_a0[0x8];
+	u8         default_table_id[0x18];
+
+	u8         sw_steering_icm_address_rx[0x40];
+	u8         sw_steering_icm_address_tx[0x40];
+
+	u8         reserved_at_140[0xa0];
+};
+
+struct mlx5_ifc_create_sampler_obj_in_bits {
+	struct mlx5_ifc_general_obj_in_cmd_hdr_bits general_obj_in_cmd_hdr;
+	struct mlx5_ifc_sampler_obj_bits sampler_object;
+};
+
 enum {
 	MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_128 = 0x0,
 	MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_256 = 0x1,
Signature · 1 line
2.26.2
author maintainer
#2 lore raw

PATCH 2/16 net/mlx5: Add sampler destination type

In reply to Saeed Mahameed

From: Chris Mi <[email protected]>

The flow sampler object is a new destination type. Add a new member for the flow destination.

---
 drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c | 3 +++
 drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c             | 3 +++
 include/linux/mlx5/fs.h                                      | 1 +
 include/linux/mlx5/mlx5_ifc.h                                | 1 +
 4 files changed, 8 insertions(+)
drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c +3 -0
index a700f3c86899..87d65f6b5310 100644
@@ -247,6 +247,9 @@ const char *parse_fs_dst(struct trace_seq *p,
 	case MLX5_FLOW_DESTINATION_TYPE_TIR:
 		trace_seq_printf(p, "tir=%u\n", dst->tir_num);
 		break;
+	case MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER:
+		trace_seq_printf(p, "sampler_id=%u\n", dst->sampler_id);
+		break;
 	case MLX5_FLOW_DESTINATION_TYPE_COUNTER:
 		trace_seq_printf(p, "counter_id=%u\n", counter_id);
 		break;
drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c +3 -0
index babe3405132a..c2fed9c3d75c 100644
@@ -515,6 +515,9 @@ static int mlx5_cmd_set_fte(struct mlx5_core_dev *dev,
 						 dst->dest_attr.vport.pkt_reformat->id);
 				}
 				break;
+			case MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER:
+				id = dst->dest_attr.sampler_id;
+				break;
 			default:
 				id = dst->dest_attr.tir_num;
 			}
include/linux/mlx5/fs.h +1 -0
index 846d94ad04bc..35d2cc1646d3 100644
@@ -132,6 +132,7 @@ struct mlx5_flow_destination {
 			struct mlx5_pkt_reformat *pkt_reformat;
 			u8		flags;
 		} vport;
+		u32			sampler_id;
 	};
 };
 
include/linux/mlx5/mlx5_ifc.h +1 -0
index 65ea35af0527..2f2add4bd5e1 100644
@@ -1616,6 +1616,7 @@ enum mlx5_flow_destination_type {
 	MLX5_FLOW_DESTINATION_TYPE_VPORT        = 0x0,
 	MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE   = 0x1,
 	MLX5_FLOW_DESTINATION_TYPE_TIR          = 0x2,
+	MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER = 0x6,
 
 	MLX5_FLOW_DESTINATION_TYPE_PORT         = 0x99,
 	MLX5_FLOW_DESTINATION_TYPE_COUNTER      = 0x100,
Signature · 1 line
2.26.2
author maintainer
#3 lore raw

PATCH 3/16 net/mlx5: Check dr mask size against mlx5_match_param size

In reply to Saeed Mahameed

From: Muhammad Sammar <[email protected]>

This is to allow passing misc4 match param from userspace when function like ib_flow_matcher_create is called.

---
 drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c | 2 +-
 drivers/net/ethernet/mellanox/mlx5/core/steering/dr_rule.c    | 3 +--
 drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h   | 1 +
 3 files changed, 3 insertions(+), 3 deletions(-)
drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c +1 -1
index 7df883686d46..1b3b2acd45c5 100644
@@ -630,7 +630,7 @@ static int dr_matcher_init(struct mlx5dr_matcher *matcher,
 	}
 
 	if (mask) {
-		if (mask->match_sz > sizeof(struct mlx5dr_match_param)) {
+		if (mask->match_sz > DR_SZ_MATCH_PARAM) {
 			mlx5dr_err(dmn, "Invalid match size attribute\n");
 			return -EINVAL;
 		}
drivers/net/ethernet/mellanox/mlx5/core/steering/dr_rule.c +1 -2
index b3c9dc032026..6d73719db1f4 100644
@@ -874,8 +874,7 @@ static bool dr_rule_verify(struct mlx5dr_matcher *matcher,
 	u32 s_idx, e_idx;
 
 	if (!value_size ||
-	    (value_size > sizeof(struct mlx5dr_match_param) ||
-	     (value_size % sizeof(u32)))) {
+	    (value_size > DR_SZ_MATCH_PARAM || (value_size % sizeof(u32)))) {
 		mlx5dr_err(matcher->tbl->dmn, "Rule parameters length is incorrect\n");
 		return false;
 	}
drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h +1 -0
index f50f3b107aa3..937f469ec678 100644
@@ -17,6 +17,7 @@
 #define WIRE_PORT 0xFFFF
 #define DR_STE_SVLAN 0x1
 #define DR_STE_CVLAN 0x2
+#define DR_SZ_MATCH_PARAM (MLX5_ST_SZ_DW_MATCH_PARAM * 4)
 
 #define mlx5dr_err(dmn, arg...) mlx5_core_err((dmn)->mdev, ##arg)
 #define mlx5dr_info(dmn, arg...) mlx5_core_info((dmn)->mdev, ##arg)
Signature · 1 line
2.26.2
author maintainer
#4 lore raw

PATCH 4/16 net/mlx5: Add misc4 to mlx5_ifc_fte_match_param_bits

In reply to Saeed Mahameed

From: Muhammad Sammar <[email protected]>

Add misc4 match params to enable matching on prog_sample_fields.

---
 .../net/ethernet/mellanox/mlx5/core/fs_core.h |  2 +-
 include/linux/mlx5/device.h                   |  1 +
 include/linux/mlx5/mlx5_ifc.h                 | 25 ++++++++++++++++++-
 include/uapi/rdma/mlx5_user_ioctl_cmds.h      |  2 +-
 4 files changed, 27 insertions(+), 3 deletions(-)
drivers/net/ethernet/mellanox/mlx5/core/fs_core.h +1 -1
index afe7f0bffb93..b24a9849c45e 100644
@@ -194,7 +194,7 @@ struct mlx5_ft_underlay_qp {
 	u32 qpn;
 };
 
-#define MLX5_FTE_MATCH_PARAM_RESERVED	reserved_at_a00
+#define MLX5_FTE_MATCH_PARAM_RESERVED	reserved_at_c00
 /* Calculate the fte_match_param length and without the reserved length.
  * Make sure the reserved field is the last.
  */
include/linux/mlx5/device.h +1 -0
index cf824366a7d1..e9639c4cf2ed 100644
@@ -1076,6 +1076,7 @@ enum {
 	MLX5_MATCH_INNER_HEADERS	= 1 << 2,
 	MLX5_MATCH_MISC_PARAMETERS_2	= 1 << 3,
 	MLX5_MATCH_MISC_PARAMETERS_3	= 1 << 4,
+	MLX5_MATCH_MISC_PARAMETERS_4	= 1 << 5,
 };
 
 enum {
include/linux/mlx5/mlx5_ifc.h +24 -1
index 2f2add4bd5e1..11c24fafd7f2 100644
@@ -623,6 +623,26 @@ struct mlx5_ifc_fte_match_set_misc3_bits {
 	u8         reserved_at_140[0xc0];
 };
 
+struct mlx5_ifc_fte_match_set_misc4_bits {
+	u8         prog_sample_field_value_0[0x20];
+
+	u8         prog_sample_field_id_0[0x20];
+
+	u8         prog_sample_field_value_1[0x20];
+
+	u8         prog_sample_field_id_1[0x20];
+
+	u8         prog_sample_field_value_2[0x20];
+
+	u8         prog_sample_field_id_2[0x20];
+
+	u8         prog_sample_field_value_3[0x20];
+
+	u8         prog_sample_field_id_3[0x20];
+
+	u8         reserved_at_100[0x100];
+};
+
 struct mlx5_ifc_cmd_pas_bits {
 	u8         pa_h[0x20];
 
@@ -1669,7 +1689,9 @@ struct mlx5_ifc_fte_match_param_bits {
 
 	struct mlx5_ifc_fte_match_set_misc3_bits misc_parameters_3;
 
-	u8         reserved_at_a00[0x600];
+	struct mlx5_ifc_fte_match_set_misc4_bits misc_parameters_4;
+
+	u8         reserved_at_c00[0x400];
 };
 
 enum {
@@ -5462,6 +5484,7 @@ enum {
 	MLX5_QUERY_FLOW_GROUP_OUT_MATCH_CRITERIA_ENABLE_INNER_HEADERS    = 0x2,
 	MLX5_QUERY_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS_2 = 0x3,
 	MLX5_QUERY_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS_3 = 0x4,
+	MLX5_QUERY_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS_4 = 0x5,
 };
 
 struct mlx5_ifc_query_flow_group_out_bits {
include/uapi/rdma/mlx5_user_ioctl_cmds.h +1 -1
index e24d66d278cf..3fd9b380a091 100644
@@ -232,7 +232,7 @@ enum mlx5_ib_device_query_context_attrs {
 	MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX = (1U << UVERBS_ID_NS_SHIFT),
 };
 
-#define MLX5_IB_DW_MATCH_PARAM 0x80
+#define MLX5_IB_DW_MATCH_PARAM 0x90
 
 struct mlx5_ib_match_params {
 	__u32	match_params[MLX5_IB_DW_MATCH_PARAM];
Signature · 1 line
2.26.2
author maintainer
#5 lore raw

PATCH 5/16 net/mlx5: Add ts_cqe_to_dest_cqn related bits

In reply to Saeed Mahameed

From: Eran Ben Elisha <[email protected]>

Add a bit in HCA capabilities layout to indicate if ts_cqe_to_dest_cqn is supported.

In addition, add ts_cqe_to_dest_cqn field to SQ context, for driver to set the actual CQN.

---
 include/linux/mlx5/mlx5_ifc.h | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)
include/linux/mlx5/mlx5_ifc.h +8 -2
index 11c24fafd7f2..632b9a61fda5 100644
@@ -1261,7 +1261,9 @@ struct mlx5_ifc_cmd_hca_cap_bits {
 	u8	   ece_support[0x1];
 	u8	   reserved_at_a4[0x7];
 	u8         log_max_srq[0x5];
-	u8         reserved_at_b0[0x10];
+	u8         reserved_at_b0[0x2];
+	u8         ts_cqe_to_dest_cqn[0x1];
+	u8         reserved_at_b3[0xd];
 
 	u8         max_sgl_for_optimized_performance[0x8];
 	u8         log_max_cq_sz[0x8];
@@ -3312,8 +3314,12 @@ struct mlx5_ifc_sqc_bits {
 	u8         reserved_at_80[0x10];
 	u8         hairpin_peer_vhca[0x10];
 
-	u8         reserved_at_a0[0x50];
+	u8         reserved_at_a0[0x20];
 
+	u8         reserved_at_c0[0x8];
+	u8         ts_cqe_to_dest_cqn[0x18];
+
+	u8         reserved_at_e0[0x10];
 	u8         packet_pacing_rate_limit_index[0x10];
 	u8         tis_lst_sz[0x10];
 	u8         reserved_at_110[0x10];
Signature · 1 line
2.26.2
author maintainer
#6 lore raw

PATCH 6/16 net/mlx5: Avoid exposing driver internal command helpers

In reply to Saeed Mahameed

From: Parav Pandit <[email protected]>

mlx5 command init and cleanup routines are internal to mlx5_core driver. Hence, avoid exporting them and move their definition to mlx5_core driver's internal file mlx5_core.h

---
 drivers/net/ethernet/mellanox/mlx5/core/cmd.c       | 3 ---
 drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h | 4 ++++
 include/linux/mlx5/driver.h                         | 4 ----
 3 files changed, 4 insertions(+), 7 deletions(-)
drivers/net/ethernet/mellanox/mlx5/core/cmd.c +0 -3
index e49387dbef98..50c7b9ee80c3 100644
@@ -2142,7 +2142,6 @@ int mlx5_cmd_init(struct mlx5_core_dev *dev)
 	kvfree(cmd->stats);
 	return err;
 }
-EXPORT_SYMBOL(mlx5_cmd_init);
 
 void mlx5_cmd_cleanup(struct mlx5_core_dev *dev)
 {
@@ -2155,11 +2154,9 @@ void mlx5_cmd_cleanup(struct mlx5_core_dev *dev)
 	dma_pool_destroy(cmd->pool);
 	kvfree(cmd->stats);
 }
-EXPORT_SYMBOL(mlx5_cmd_cleanup);
 
 void mlx5_cmd_set_state(struct mlx5_core_dev *dev,
 			enum mlx5_cmdif_state cmdif_state)
 {
 	dev->cmd.state = cmdif_state;
 }
-EXPORT_SYMBOL(mlx5_cmd_set_state);
drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +4 -0
index 8cec85ab419d..9d00efa9e6bc 100644
@@ -122,6 +122,10 @@ enum mlx5_semaphore_space_address {
 
 int mlx5_query_hca_caps(struct mlx5_core_dev *dev);
 int mlx5_query_board_id(struct mlx5_core_dev *dev);
+int mlx5_cmd_init(struct mlx5_core_dev *dev);
+void mlx5_cmd_cleanup(struct mlx5_core_dev *dev);
+void mlx5_cmd_set_state(struct mlx5_core_dev *dev,
+			enum mlx5_cmdif_state cmdif_state);
 int mlx5_cmd_init_hca(struct mlx5_core_dev *dev, uint32_t *sw_owner_id);
 int mlx5_cmd_teardown_hca(struct mlx5_core_dev *dev);
 int mlx5_cmd_force_teardown_hca(struct mlx5_core_dev *dev);
include/linux/mlx5/driver.h +0 -4
index add85094f9a5..5e84b1d53650 100644
@@ -888,10 +888,6 @@ enum {
 	CMD_ALLOWED_OPCODE_ALL,
 };
 
-int mlx5_cmd_init(struct mlx5_core_dev *dev);
-void mlx5_cmd_cleanup(struct mlx5_core_dev *dev);
-void mlx5_cmd_set_state(struct mlx5_core_dev *dev,
-			enum mlx5_cmdif_state cmdif_state);
 void mlx5_cmd_use_events(struct mlx5_core_dev *dev);
 void mlx5_cmd_use_polling(struct mlx5_core_dev *dev);
 void mlx5_cmd_allowed_opcode(struct mlx5_core_dev *dev, u16 opcode);
Signature · 1 line
2.26.2
author maintainer
#7 lore raw

PATCH 7/16 net/mlx5: Update the list of the PCI supported devices

In reply to Saeed Mahameed

From: Meir Lichtinger <[email protected]>

Add the upcoming BlueField-3 device ID.

---
 drivers/net/ethernet/mellanox/mlx5/core/main.c | 1 +
 1 file changed, 1 insertion(+)
drivers/net/ethernet/mellanox/mlx5/core/main.c +1 -0
index 8ff207aa1479..a9757ccb9d16 100644
@@ -1594,6 +1594,7 @@ static const struct pci_device_id mlx5_core_pci_table[] = {
 	{ PCI_VDEVICE(MELLANOX, 0xa2d2) },			/* BlueField integrated ConnectX-5 network controller */
 	{ PCI_VDEVICE(MELLANOX, 0xa2d3), MLX5_PCI_DEV_IS_VF},	/* BlueField integrated ConnectX-5 network controller VF */
 	{ PCI_VDEVICE(MELLANOX, 0xa2d6) },			/* BlueField-2 integrated ConnectX-6 Dx network controller */
+	{ PCI_VDEVICE(MELLANOX, 0xa2dc) },			/* BlueField-3 integrated ConnectX-7 network controller */
 	{ 0, }
 };
 
Signature · 1 line
2.26.2
author maintainer
#8 lore raw

PATCH 8/16 net/mlx5: Update the hardware interface definition for vhca state

In reply to Saeed Mahameed

From: Parav Pandit <[email protected]>

Update the hardware interface definitions to query and modify vhca state, related EQE and event code.

---
 include/linux/mlx5/device.h   |  7 +++++++
 include/linux/mlx5/mlx5_ifc.h | 17 ++++++++++++++---
 2 files changed, 21 insertions(+), 3 deletions(-)
include/linux/mlx5/device.h +7 -0
index e9639c4cf2ed..f1de49d64a98 100644
@@ -346,6 +346,7 @@ enum mlx5_event {
 	MLX5_EVENT_TYPE_NIC_VPORT_CHANGE   = 0xd,
 
 	MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED = 0xe,
+	MLX5_EVENT_TYPE_VHCA_STATE_CHANGE = 0xf,
 
 	MLX5_EVENT_TYPE_DCT_DRAINED        = 0x1c,
 	MLX5_EVENT_TYPE_DCT_KEY_VIOLATION  = 0x1d,
@@ -717,6 +718,11 @@ struct mlx5_eqe_sync_fw_update {
 	u8 sync_rst_state;
 };
 
+struct mlx5_eqe_vhca_state {
+	__be16 ec_function;
+	__be16 function_id;
+} __packed;
+
 union ev_data {
 	__be32				raw[7];
 	struct mlx5_eqe_cmd		cmd;
@@ -736,6 +742,7 @@ union ev_data {
 	struct mlx5_eqe_temp_warning	temp_warning;
 	struct mlx5_eqe_xrq_err		xrq_err;
 	struct mlx5_eqe_sync_fw_update	sync_fw_update;
+	struct mlx5_eqe_vhca_state	vhca_state;
 } __packed;
 
 struct mlx5_eqe {
include/linux/mlx5/mlx5_ifc.h +14 -3
index 632b9a61fda5..3ace1976514c 100644
@@ -299,6 +299,8 @@ enum {
 	MLX5_CMD_OP_CREATE_UMEM                   = 0xa08,
 	MLX5_CMD_OP_DESTROY_UMEM                  = 0xa0a,
 	MLX5_CMD_OP_SYNC_STEERING                 = 0xb00,
+	MLX5_CMD_OP_QUERY_VHCA_STATE              = 0xb0d,
+	MLX5_CMD_OP_MODIFY_VHCA_STATE             = 0xb0e,
 	MLX5_CMD_OP_MAX
 };
 
@@ -1244,7 +1246,15 @@ enum mlx5_fc_bulk_alloc_bitmask {
 #define MLX5_FC_BULK_NUM_FCS(fc_enum) (MLX5_FC_BULK_SIZE_FACTOR * (fc_enum))
 
 struct mlx5_ifc_cmd_hca_cap_bits {
-	u8         reserved_at_0[0x30];
+	u8         reserved_at_0[0x20];
+
+	u8         reserved_at_20[0x3];
+	u8         event_on_vhca_state_teardown_request[0x1];
+	u8         event_on_vhca_state_in_use[0x1];
+	u8         event_on_vhca_state_active[0x1];
+	u8         event_on_vhca_state_allocated[0x1];
+	u8         event_on_vhca_state_invalid[0x1];
+	u8         reserved_at_28[0x8];
 	u8         vhca_id[0x10];
 
 	u8         reserved_at_40[0x40];
@@ -1534,7 +1544,8 @@ struct mlx5_ifc_cmd_hca_cap_bits {
 	u8         disable_local_lb_uc[0x1];
 	u8         disable_local_lb_mc[0x1];
 	u8         log_min_hairpin_wq_data_sz[0x5];
-	u8         reserved_at_3e8[0x3];
+	u8         reserved_at_3e8[0x2];
+	u8         vhca_state[0x1];
 	u8         log_max_vlan_list[0x5];
 	u8         reserved_at_3f0[0x3];
 	u8         log_max_current_mc_list[0x5];
@@ -1602,7 +1613,7 @@ struct mlx5_ifc_cmd_hca_cap_bits {
 	u8         max_num_of_monitor_counters[0x10];
 	u8         num_ppcnt_monitor_counters[0x10];
 
-	u8         reserved_at_640[0x10];
+	u8         max_num_sf[0x10];
 	u8         num_q_monitor_counters[0x10];
 
 	u8         reserved_at_660[0x20];
Signature · 1 line
2.26.2
author maintainer
#9 lore raw

PATCH 9/16 net/mlx5: Expose IP-in-IP TX and RX capability bits

In reply to Saeed Mahameed

From: Aya Levin <[email protected]>

Expose FW indication that it supports stateless offloads for IP over IP tunneled packets per direction. In some HW like ConnectX-4 IP-in-IP support is not symmetric, it supports steering on the inner header but it doesn't TX-Checksum and TSO. Add IP-in-IP capability per direction to cover this case as well.

Note: only if both indications are turned on, the global tunnel_stateless_ip_over_ip is on too.

---
 include/linux/mlx5/mlx5_ifc.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)
include/linux/mlx5/mlx5_ifc.h +4 -1
index 3ace1976514c..96888f9f822d 100644
@@ -913,7 +913,10 @@ struct mlx5_ifc_per_protocol_networking_offload_caps_bits {
 	u8         tunnel_stateless_ipv4_over_vxlan[0x1];
 	u8         tunnel_stateless_ip_over_ip[0x1];
 	u8         insert_trailer[0x1];
-	u8         reserved_at_2b[0x5];
+	u8         reserved_at_2b[0x1];
+	u8         tunnel_stateless_ip_over_ip_rx[0x1];
+	u8         tunnel_stateless_ip_over_ip_tx[0x1];
+	u8         reserved_at_2e[0x2];
 	u8         max_vxlan_udp_ports[0x8];
 	u8         reserved_at_38[0x6];
 	u8         max_geneve_opt_len[0x1];
Signature · 1 line
2.26.2
maintainer
#10 lore raw

Re: [PATCH mlx5-next 09/16] net/mlx5: Expose IP-in-IP TX and RX capability bits

Quoted 7 lines · On Fri, 20 Nov 2020 15:03:32 -0800 Saeed Mahameed wrote:

From: Aya Levin <[email protected]>

Expose FW indication that it supports stateless offloads for IP over IP tunneled packets per direction. In some HW like ConnectX-4 IP-in-IP support is not symmetric, it supports steering on the inner header but it doesn't TX-Checksum and TSO. Add IP-in-IP capability per direction to cover this case as well.

What's the use for the rx capability in Linux? We don't have an API to configure that AFAIK.

#11 lore raw

Re: [PATCH mlx5-next 09/16] net/mlx5: Expose IP-in-IP TX and RX capability bits

Quoted 12 lines · Jakub Kicinski
Quoted 7 lines · On Fri, 20 Nov 2020 15:03:32 -0800 Saeed Mahameed wrote:

From: Aya Levin <[email protected]>

Expose FW indication that it supports stateless offloads for IP over IP tunneled packets per direction. In some HW like ConnectX-4 IP-in-IP support is not symmetric, it supports steering on the inner header but it doesn't TX-Checksum and TSO. Add IP-in-IP capability per direction to cover this case as well.

What's the use for the rx capability in Linux? We don't have an API to configure that AFAIK.

Correct, the rx capability bit is used by the driver to allow flow steering on the inner header.

#12 lore raw

Re: [PATCH mlx5-next 09/16] net/mlx5: Expose IP-in-IP TX and RX capability bits

Quoted 15 lines · Aya Levin
Quoted 13 lines · Jakub Kicinski
Quoted 11 lines · On Fri, 20 Nov 2020 15:03:32 -0800 Saeed Mahameed wrote:

From: Aya Levin <[email protected]>

Expose FW indication that it supports stateless offloads for IP over IP
tunneled packets per direction. In some HW like ConnectX-4 IP-in- IP
support is not symmetric, it supports steering on the inner header but
it doesn't TX-Checksum and TSO. Add IP-in-IP capability per direction to
cover this case as well.

Quoted 6 lines
Quoted 4 lines

What's the use for the rx capability in Linux? We don't have an API to
configure that AFAIK.

Correct, the rx capability bit is used by the driver to allow flow steering on the inner header.

Currently we use the global HW capability to enable flow steering on inner header for RSS. in upcoming patch to net-next we will relax the dependency on the global capability and will use the dedicated rx cap instead.

author maintainer
#13 lore raw

PATCH 10/16 net/mlx5: Expose other function ifc bits

In reply to Saeed Mahameed

From: Yishai Hadas <[email protected]>

Expose other function ifc bits to enable setting HCA caps on behalf of other function.

In addition, expose vhca_resource_manager bit to control whether the other function functionality is supported by firmware.

---
 include/linux/mlx5/mlx5_ifc.h | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)
include/linux/mlx5/mlx5_ifc.h +7 -2
index 96888f9f822d..3e337386faa8 100644
@@ -1249,7 +1249,8 @@ enum mlx5_fc_bulk_alloc_bitmask {
 #define MLX5_FC_BULK_NUM_FCS(fc_enum) (MLX5_FC_BULK_SIZE_FACTOR * (fc_enum))
 
 struct mlx5_ifc_cmd_hca_cap_bits {
-	u8         reserved_at_0[0x20];
+	u8         reserved_at_0[0x1f];
+	u8         vhca_resource_manager[0x1];
 
 	u8         reserved_at_20[0x3];
 	u8         event_on_vhca_state_teardown_request[0x1];
@@ -4247,7 +4248,11 @@ struct mlx5_ifc_set_hca_cap_in_bits {
 	u8         reserved_at_20[0x10];
 	u8         op_mod[0x10];
 
-	u8         reserved_at_40[0x40];
+	u8         other_function[0x1];
+	u8         reserved_at_41[0xf];
+	u8         function_id[0x10];
+
+	u8         reserved_at_60[0x20];
 
 	union mlx5_ifc_hca_cap_union_bits capability;
 };
Signature · 1 line
2.26.2
author maintainer
#14 lore raw

PATCH 11/16 net/mlx5: Add VDPA priority to NIC RX namespace

In reply to Saeed Mahameed

From: Eli Cohen <[email protected]>

Add a new namespace type to the NIC RX root namespace to allow for inserting VDPA rules before regular NIC but after bypass, thus allowing DPDK to have precedence in packet processing.

---
 drivers/net/ethernet/mellanox/mlx5/core/fs_core.c | 10 +++++++++-
 include/linux/mlx5/fs.h                           |  1 +
 2 files changed, 10 insertions(+), 1 deletion(-)
drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +9 -1
index 16091838bfcf..e095c5968e67 100644
@@ -118,6 +118,10 @@
 #define ANCHOR_NUM_PRIOS 1
 #define ANCHOR_MIN_LEVEL (BY_PASS_MIN_LEVEL + 1)
 
+#define VDPA_PRIO_NUM_LEVELS 1
+#define VDPA_NUM_PRIOS 1
+#define VDPA_MIN_LEVEL 1
+
 #define OFFLOADS_MAX_FT 2
 #define OFFLOADS_NUM_PRIOS 2
 #define OFFLOADS_MIN_LEVEL (ANCHOR_MIN_LEVEL + OFFLOADS_NUM_PRIOS)
@@ -147,7 +151,7 @@ static struct init_tree_node {
 	enum mlx5_flow_table_miss_action def_miss_action;
 } root_fs = {
 	.type = FS_TYPE_NAMESPACE,
-	.ar_size = 7,
+	.ar_size = 8,
 	  .children = (struct init_tree_node[]){
 		  ADD_PRIO(0, BY_PASS_MIN_LEVEL, 0, FS_CHAINING_CAPS,
 			   ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
@@ -165,6 +169,10 @@ static struct init_tree_node {
 			   ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
 				  ADD_MULTIPLE_PRIO(ETHTOOL_NUM_PRIOS,
 						    ETHTOOL_PRIO_NUM_LEVELS))),
+		  ADD_PRIO(0, VDPA_MIN_LEVEL, 0, FS_CHAINING_CAPS,
+			   ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+				  ADD_MULTIPLE_PRIO(VDPA_NUM_PRIOS,
+						    VDPA_PRIO_NUM_LEVELS))),
 		  ADD_PRIO(0, KERNEL_MIN_LEVEL, 0, {},
 			   ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
 				  ADD_MULTIPLE_PRIO(KERNEL_NIC_TC_NUM_PRIOS,
include/linux/mlx5/fs.h +1 -0
index 35d2cc1646d3..97176d623d74 100644
@@ -67,6 +67,7 @@ enum mlx5_flow_namespace_type {
 	MLX5_FLOW_NAMESPACE_LAG,
 	MLX5_FLOW_NAMESPACE_OFFLOADS,
 	MLX5_FLOW_NAMESPACE_ETHTOOL,
+	MLX5_FLOW_NAMESPACE_VDPA,
 	MLX5_FLOW_NAMESPACE_KERNEL,
 	MLX5_FLOW_NAMESPACE_LEFTOVERS,
 	MLX5_FLOW_NAMESPACE_ANCHOR,
Signature · 1 line
2.26.2
maintainer
#15 lore raw

Re: [PATCH mlx5-next 11/16] net/mlx5: Add VDPA priority to NIC RX namespace

Quoted 5 lines · On Fri, 20 Nov 2020 15:03:34 -0800 Saeed Mahameed wrote:

From: Eli Cohen <[email protected]>

Add a new namespace type to the NIC RX root namespace to allow for inserting VDPA rules before regular NIC but after bypass, thus allowing DPDK to have precedence in packet processing.

How does DPDK and VDPA relate in this context?

#16 lore raw

Re: [PATCH mlx5-next 11/16] net/mlx5: Add VDPA priority to NIC RX namespace

Quoted 8 lines · Jakub Kicinski
Quoted 5 lines · On Fri, 20 Nov 2020 15:03:34 -0800 Saeed Mahameed wrote:

From: Eli Cohen <[email protected]>

Add a new namespace type to the NIC RX root namespace to allow for inserting VDPA rules before regular NIC but after bypass, thus allowing DPDK to have precedence in packet processing.

How does DPDK and VDPA relate in this context?

mlx5 steering is hierarchical and defines precedence amongst namespaces. Up till now, the VDPA implementation would insert a rule into the MLX5_FLOW_NAMESPACE_BYPASS hierarchy which is used by DPDK thus taking all the incoming traffic.

The MLX5_FLOW_NAMESPACE_VDPA hirerachy comes after MLX5_FLOW_NAMESPACE_BYPASS.

maintainer
#17 lore raw

Re: [PATCH mlx5-next 11/16] net/mlx5: Add VDPA priority to NIC RX namespace

Quoted 10 lines · On Sun, 22 Nov 2020 08:41:58 +0200 Eli Cohen wrote:
Quoted 8 lines · Jakub Kicinski
Quoted 5 lines · On Fri, 20 Nov 2020 15:03:34 -0800 Saeed Mahameed wrote:

From: Eli Cohen <[email protected]>

Add a new namespace type to the NIC RX root namespace to allow for inserting VDPA rules before regular NIC but after bypass, thus allowing DPDK to have precedence in packet processing.

How does DPDK and VDPA relate in this context?

Quoted 7 lines

mlx5 steering is hierarchical and defines precedence amongst namespaces. Up till now, the VDPA implementation would insert a rule into the MLX5_FLOW_NAMESPACE_BYPASS hierarchy which is used by DPDK thus taking all the incoming traffic.

The MLX5_FLOW_NAMESPACE_VDPA hirerachy comes after MLX5_FLOW_NAMESPACE_BYPASS.

Our policy was no DPDK driver bifurcation. There's no asterisk saying "unless you pretend you need flow filters for RDMA, get them upstream and then drop the act".

What do you expect me to do?

maintainer
#18 lore raw

Re: [PATCH mlx5-next 11/16] net/mlx5: Add VDPA priority to NIC RX namespace

Quoted 16 lines · Jakub Kicinski
Quoted 15 lines · On Sun, 22 Nov 2020 08:41:58 +0200 Eli Cohen wrote:
Quoted 8 lines · Jakub Kicinski
Quoted 5 lines · On Fri, 20 Nov 2020 15:03:34 -0800 Saeed Mahameed wrote:

From: Eli Cohen <[email protected]>

Add a new namespace type to the NIC RX root namespace to allow for inserting VDPA rules before regular NIC but after bypass, thus allowing DPDK to have precedence in packet processing.

How does DPDK and VDPA relate in this context?

mlx5 steering is hierarchical and defines precedence amongst namespaces. Up till now, the VDPA implementation would insert a rule into the MLX5_FLOW_NAMESPACE_BYPASS hierarchy which is used by DPDK thus taking all the incoming traffic.

Quoted 6 lines
Quoted 2 lines

The MLX5_FLOW_NAMESPACE_VDPA hirerachy comes after MLX5_FLOW_NAMESPACE_BYPASS.

Our policy was no DPDK driver bifurcation. There's no asterisk saying "unless you pretend you need flow filters for RDMA, get them upstream and then drop the act".

Huh?

mlx5 DPDK is an *RDMA* userspace application. It links to libibverbs. It runs on the RDMA stack. It uses RDMA flow filtering and RDMA raw ethernet QPs. It has been like this for years, it is not some "act".

It is long standing uABI that accelerators like RDMA/etc get to take the traffic before netdev. This cannot be reverted. I don't really understand what you are expecting here?

Jason

maintainer
#19 lore raw

Re: [PATCH mlx5-next 11/16] net/mlx5: Add VDPA priority to NIC RX namespace

Quoted 20 lines · On Tue, 24 Nov 2020 14:02:10 -0400 Jason Gunthorpe wrote:
Quoted 19 lines · Jakub Kicinski
Quoted 17 lines · On Sun, 22 Nov 2020 08:41:58 +0200 Eli Cohen wrote:
Quoted 8 lines · Jakub Kicinski
Quoted 5 lines · On Fri, 20 Nov 2020 15:03:34 -0800 Saeed Mahameed wrote:

From: Eli Cohen <[email protected]>

Add a new namespace type to the NIC RX root namespace to allow for inserting VDPA rules before regular NIC but after bypass, thus allowing DPDK to have precedence in packet processing.

How does DPDK and VDPA relate in this context?

mlx5 steering is hierarchical and defines precedence amongst namespaces. Up till now, the VDPA implementation would insert a rule into the MLX5_FLOW_NAMESPACE_BYPASS hierarchy which is used by DPDK thus taking all the incoming traffic.

The MLX5_FLOW_NAMESPACE_VDPA hirerachy comes after MLX5_FLOW_NAMESPACE_BYPASS.

Quoted 7 lines
Quoted 3 lines

Our policy was no DPDK driver bifurcation. There's no asterisk saying "unless you pretend you need flow filters for RDMA, get them upstream and then drop the act".

Huh?

mlx5 DPDK is an *RDMA* userspace application.

Forgive me for my naiveté.

Here I thought the RDMA subsystem is for doing RDMA.

I'm sure if you start doing crypto over ibverbs crypto people will want to have a look.

Quoted 2 lines

libibverbs. It runs on the RDMA stack. It uses RDMA flow filtering and RDMA raw ethernet QPs.

I'm not saying that's not the case. I'm saying I don't think this was something that netdev developers signed-off on. And our policy on DPDK is pretty widely known.

Would you mind pointing us to the introduction of raw Ethernet QPs?

Is there any production use for that without DPDK?

Quoted 5 lines

It has been like this for years, it is not some "act".

It is long standing uABI that accelerators like RDMA/etc get to take the traffic before netdev. This cannot be reverted. I don't really understand what you are expecting here?

Same. I don't really know what you expect me to do either. I don't think I can sign-off on kernel changes needed for DPDK.

maintainer
#20 lore raw

Re: [PATCH mlx5-next 11/16] net/mlx5: Add VDPA priority to NIC RX namespace

Quoted 25 lines · Jakub Kicinski
Quoted 24 lines · On Tue, 24 Nov 2020 14:02:10 -0400 Jason Gunthorpe wrote:
Quoted 22 lines · Jakub Kicinski
Quoted 10 lines · On Sun, 22 Nov 2020 08:41:58 +0200 Eli Cohen wrote:
Quoted 8 lines · Jakub Kicinski
Quoted 5 lines · On Fri, 20 Nov 2020 15:03:34 -0800 Saeed Mahameed wrote:

From: Eli Cohen <[email protected]>

Add a new namespace type to the NIC RX root namespace to allow for inserting VDPA rules before regular NIC but after bypass, thus allowing DPDK to have precedence in packet processing.

How does DPDK and VDPA relate in this context?

Quoted 7 lines

mlx5 steering is hierarchical and defines precedence amongst namespaces. Up till now, the VDPA implementation would insert a rule into the MLX5_FLOW_NAMESPACE_BYPASS hierarchy which is used by DPDK thus taking all the incoming traffic.

The MLX5_FLOW_NAMESPACE_VDPA hirerachy comes after MLX5_FLOW_NAMESPACE_BYPASS.

Our policy was no DPDK driver bifurcation. There's no asterisk saying "unless you pretend you need flow filters for RDMA, get them upstream and then drop the act".

Quoted 7 lines
Quoted 3 lines

Huh?

mlx5 DPDK is an *RDMA* userspace application.

Forgive me for my naiveté.

Here I thought the RDMA subsystem is for doing RDMA.

RDMA covers a wide range of accelerated networking these days.. Where else are you going to put this stuff in the kernel?

Quoted 2 lines

I'm sure if you start doing crypto over ibverbs crypto people will want to have a look.

Well, RDMA has crypto transforms for a few years now too. Why would crypto subsystem people be involved? It isn't using or duplicating their APIs.

Quoted 5 lines
Quoted 2 lines

libibverbs. It runs on the RDMA stack. It uses RDMA flow filtering and RDMA raw ethernet QPs.

I'm not saying that's not the case. I'm saying I don't think this was something that netdev developers signed-off on.

Part of the point of the subsystem split was to end the fighting that started all of it. It was very clear during the whole iWarp and TCP Offload Engine buisness in the mid 2000's that netdev wanted nothing to do with the accelerator world.

So why would netdev need sign off on any accelerator stuff? Do you want to start co-operating now? I'm willing to talk about how to do that.

Quoted 1 line

And our policy on DPDK is pretty widely known.

I honestly have no idea on the netdev DPDK policy, I'm maintaining the RDMA subsystem not DPDK :)

Quoted 3 lines

Would you mind pointing us to the introduction of raw Ethernet QPs?

Is there any production use for that without DPDK?

Hmm.. It is very old. RAW (InfiniBand) QPs were part of the original IBA specification cira 2000. When RoCE was defined (around 2010) they were naturally carried forward to Ethernet. The "flow steering" concept to make raw ethernet QP useful was added to verbs around 2012
- 2013. It officially made it upstream in commit 436f2ad05a0b ("IB/core: Export ib_create/destroy_flow through uverbs")

If I recall properly the first real application was ultra low latency ethernet processing for financial applications.

dpdk later adopted the first mlx4 PMD using this libibverbs API around 2015. Interestingly the mlx4 PMD was made through an open source process with minimal involvment from Mellanox, based on the pre-existing RDMA work.

Currently there are many projects, and many open source, built on top of the RDMA raw ethernet QP and RDMA flow steering model. It is now long established kernel ABI.

Quoted 8 lines
Quoted 5 lines

It has been like this for years, it is not some "act".

It is long standing uABI that accelerators like RDMA/etc get to take the traffic before netdev. This cannot be reverted. I don't really understand what you are expecting here?

Same. I don't really know what you expect me to do either. I don't think I can sign-off on kernel changes needed for DPDK.

This patch is fine tuning the shared logic that splits the traffic to accelerator subsystems, I don't think netdev should have a veto here. This needs to be consensus among the various communities and subsystems that rely on this.

Eli did not explain this well in his commit message. When he said DPDK he means RDMA which is the owner of the FLOW_NAMESPACE. Each accelerator subsystem gets hooked into this, so here VPDA is getting its own hook because re-using the the same hook between two kernel subsystems is buggy.

Jason

#21 lore raw

Re: [PATCH mlx5-next 11/16] net/mlx5: Add VDPA priority to NIC RX namespace

Quoted 101 lines · Jason Gunthorpe
Quoted 25 lines · Jakub Kicinski
Quoted 24 lines · On Tue, 24 Nov 2020 14:02:10 -0400 Jason Gunthorpe wrote:
Quoted 22 lines · Jakub Kicinski
Quoted 10 lines · On Sun, 22 Nov 2020 08:41:58 +0200 Eli Cohen wrote:
Quoted 8 lines · Jakub Kicinski
Quoted 5 lines · On Fri, 20 Nov 2020 15:03:34 -0800 Saeed Mahameed wrote:

From: Eli Cohen <[email protected]>

Add a new namespace type to the NIC RX root namespace to allow for inserting VDPA rules before regular NIC but after bypass, thus allowing DPDK to have precedence in packet processing.

How does DPDK and VDPA relate in this context?

Quoted 7 lines

mlx5 steering is hierarchical and defines precedence amongst namespaces. Up till now, the VDPA implementation would insert a rule into the MLX5_FLOW_NAMESPACE_BYPASS hierarchy which is used by DPDK thus taking all the incoming traffic.

The MLX5_FLOW_NAMESPACE_VDPA hirerachy comes after MLX5_FLOW_NAMESPACE_BYPASS.

Our policy was no DPDK driver bifurcation. There's no asterisk saying "unless you pretend you need flow filters for RDMA, get them upstream and then drop the act".

Quoted 7 lines
Quoted 3 lines

Huh?

mlx5 DPDK is an *RDMA* userspace application.

Forgive me for my naiveté.

Here I thought the RDMA subsystem is for doing RDMA.

RDMA covers a wide range of accelerated networking these days.. Where else are you going to put this stuff in the kernel?

Quoted 2 lines

I'm sure if you start doing crypto over ibverbs crypto people will want to have a look.

Well, RDMA has crypto transforms for a few years now too. Why would crypto subsystem people be involved? It isn't using or duplicating their APIs.

Quoted 5 lines
Quoted 2 lines

libibverbs. It runs on the RDMA stack. It uses RDMA flow filtering and RDMA raw ethernet QPs.

I'm not saying that's not the case. I'm saying I don't think this was something that netdev developers signed-off on.

Part of the point of the subsystem split was to end the fighting that started all of it. It was very clear during the whole iWarp and TCP Offload Engine buisness in the mid 2000's that netdev wanted nothing to do with the accelerator world.

So why would netdev need sign off on any accelerator stuff? Do you want to start co-operating now? I'm willing to talk about how to do that.

Quoted 1 line

And our policy on DPDK is pretty widely known.

I honestly have no idea on the netdev DPDK policy, I'm maintaining the RDMA subsystem not DPDK :)

Quoted 3 lines

Would you mind pointing us to the introduction of raw Ethernet QPs?

Is there any production use for that without DPDK?

Hmm.. It is very old. RAW (InfiniBand) QPs were part of the original IBA specification cira 2000. When RoCE was defined (around 2010) they were naturally carried forward to Ethernet. The "flow steering" concept to make raw ethernet QP useful was added to verbs around 2012
- 2013. It officially made it upstream in commit 436f2ad05a0b ("IB/core: Export ib_create/destroy_flow through uverbs")

If I recall properly the first real application was ultra low latency ethernet processing for financial applications.

dpdk later adopted the first mlx4 PMD using this libibverbs API around 2015. Interestingly the mlx4 PMD was made through an open source process with minimal involvment from Mellanox, based on the pre-existing RDMA work.

Currently there are many projects, and many open source, built on top of the RDMA raw ethernet QP and RDMA flow steering model. It is now long established kernel ABI.

Quoted 8 lines
Quoted 5 lines

It has been like this for years, it is not some "act".

It is long standing uABI that accelerators like RDMA/etc get to take the traffic before netdev. This cannot be reverted. I don't really understand what you are expecting here?

Same. I don't really know what you expect me to do either. I don't think I can sign-off on kernel changes needed for DPDK.

This patch is fine tuning the shared logic that splits the traffic to accelerator subsystems, I don't think netdev should have a veto here. This needs to be consensus among the various communities and subsystems that rely on this.

Quoted 5 lines

Eli did not explain this well in his commit message. When he said DPDK he means RDMA which is the owner of the FLOW_NAMESPACE. Each accelerator subsystem gets hooked into this, so here VPDA is getting its own hook because re-using the the same hook between two kernel subsystems is buggy.

I agree, RDMA should have been used here. DPDK is just one, though widely used, accelerator using RDMA interfaces to flow steering.

I will push submit another patch with a modified change log.

Quoted 2 lines

Jason

#22 lore raw

Re: [PATCH mlx5-next 11/16] net/mlx5: Add VDPA priority to NIC RX namespace

Quoted 137 lines · Eli Cohen
Quoted 128 lines · Jason Gunthorpe
Quoted 34 lines · Jakub Kicinski
Quoted 33 lines · On Tue, 24 Nov 2020 14:02:10 -0400 Jason Gunthorpe wrote:
Quoted 31 lines · Jakub Kicinski
Quoted 22 lines · On Sun, 22 Nov 2020 08:41:58 +0200 Eli Cohen wrote:
Quoted 11 lines · Jakub Kicinski
Quoted 7 lines · On Fri, 20 Nov 2020 15:03:34 -0800 Saeed Mahameed wrote:

From: Eli Cohen <[email protected]>

Add a new namespace type to the NIC RX root namespace to allow for
inserting VDPA rules before regular NIC but after bypass, thus allowing
DPDK to have precedence in packet processing.

How does DPDK and VDPA relate in this context?

mlx5 steering is hierarchical and defines precedence amongst namespaces.
Up till now, the VDPA implementation would insert a rule into the
MLX5_FLOW_NAMESPACE_BYPASS hierarchy which is used by DPDK thus taking
all the incoming traffic.

Quoted 2 lines

The MLX5_FLOW_NAMESPACE_VDPA hirerachy comes after MLX5_FLOW_NAMESPACE_BYPASS.

Our policy was no DPDK driver bifurcation. There's no asterisk saying
"unless you pretend you need flow filters for RDMA, get them upstream
and then drop the act".

Quoted 7 lines
Quoted 3 lines

Huh?

mlx5 DPDK is an *RDMA* userspace application.

Forgive me for my naiveté.

Here I thought the RDMA subsystem is for doing RDMA.

RDMA covers a wide range of accelerated networking these days.. Where
else are you going to put this stuff in the kernel?

Quoted 3 lines

I'm sure if you start doing crypto over ibverbs crypto people will want
to have a look.

Well, RDMA has crypto transforms for a few years now too. Why would crypto subsystem people be involved? It isn't using or duplicating their APIs.

Quoted 7 lines
Quoted 3 lines

libibverbs. It runs on the RDMA stack. It uses RDMA flow filtering and
RDMA raw ethernet QPs.

I'm not saying that's not the case. I'm saying I don't think this was
something that netdev developers signed-off on.

Part of the point of the subsystem split was to end the fighting that
started all of it. It was very clear during the whole iWarp and TCP Offload Engine buisness in the mid 2000's that netdev wanted nothing
to do with the accelerator world.

So why would netdev need sign off on any accelerator stuff? Do you want to start co-operating now? I'm willing to talk about how to do that.

Quoted 1 line

And our policy on DPDK is pretty widely known.

I honestly have no idea on the netdev DPDK policy, I'm maintaining the
RDMA subsystem not DPDK :)

Quoted 4 lines

Would you mind pointing us to the introduction of raw Ethernet QPs?

Is there any production use for that without DPDK?

Hmm.. It is very old. RAW (InfiniBand) QPs were part of the original
IBA specification cira 2000. When RoCE was defined (around 2010) they
were naturally carried forward to Ethernet. The "flow steering" concept to make raw ethernet QP useful was added to verbs around 2012
- 2013. It officially made it upstream in commit 436f2ad05a0b ("IB/core: Export ib_create/destroy_flow through uverbs")

If I recall properly the first real application was ultra low latency
ethernet processing for financial applications.

dpdk later adopted the first mlx4 PMD using this libibverbs API around
2015. Interestingly the mlx4 PMD was made through an open source process with minimal involvment from Mellanox, based on the pre-existing RDMA work.

Currently there are many projects, and many open source, built on top
of the RDMA raw ethernet QP and RDMA flow steering model. It is now long established kernel ABI.

Quoted 11 lines
Quoted 7 lines

It has been like this for years, it is not some "act".

It is long standing uABI that accelerators like RDMA/etc get to take
the traffic before netdev. This cannot be reverted. I don't really
understand what you are expecting here?

Same. I don't really know what you expect me to do either. I don't
think I can sign-off on kernel changes needed for DPDK.

This patch is fine tuning the shared logic that splits the traffic to
accelerator subsystems, I don't think netdev should have a veto here. This needs to be consensus among the various communities and subsystems that rely on this.

Quoted 7 lines

Eli did not explain this well in his commit message. When he said DPDK
he means RDMA which is the owner of the FLOW_NAMESPACE. Each accelerator subsystem gets hooked into this, so here VPDA is getting
its own hook because re-using the the same hook between two kernel subsystems is buggy.

Quoted 4 lines

I agree, RDMA should have been used here. DPDK is just one, though widely used, accelerator using RDMA interfaces to flow steering.

I will push submit another patch with a modified change log.

Hi Jakub, Given that this patch is just defining the HW domain of vDPA to separate it from the RDMA domain, it has no actual functionality, just the mlx5_core low level hw definition, can i take this patch to mlx5-next and move on to my next series ?

Thanks,
Saeed.

maintainer
#23 lore raw

Re: [PATCH mlx5-next 11/16] net/mlx5: Add VDPA priority to NIC RX namespace

In reply to Jason Gunthorpe
Quoted 28 lines · On Tue, 24 Nov 2020 15:44:13 -0400 Jason Gunthorpe wrote:
Quoted 27 lines · Jakub Kicinski
Quoted 26 lines · On Tue, 24 Nov 2020 14:02:10 -0400 Jason Gunthorpe wrote:
Quoted 16 lines · Jakub Kicinski
Quoted 15 lines · On Sun, 22 Nov 2020 08:41:58 +0200 Eli Cohen wrote:
Quoted 8 lines · Jakub Kicinski
Quoted 5 lines · On Fri, 20 Nov 2020 15:03:34 -0800 Saeed Mahameed wrote:

From: Eli Cohen <[email protected]>

Add a new namespace type to the NIC RX root namespace to allow for inserting VDPA rules before regular NIC but after bypass, thus allowing DPDK to have precedence in packet processing.

How does DPDK and VDPA relate in this context?

mlx5 steering is hierarchical and defines precedence amongst namespaces. Up till now, the VDPA implementation would insert a rule into the MLX5_FLOW_NAMESPACE_BYPASS hierarchy which is used by DPDK thus taking all the incoming traffic.

Quoted 6 lines
Quoted 2 lines

The MLX5_FLOW_NAMESPACE_VDPA hirerachy comes after MLX5_FLOW_NAMESPACE_BYPASS.

Our policy was no DPDK driver bifurcation. There's no asterisk saying "unless you pretend you need flow filters for RDMA, get them upstream and then drop the act".

Huh?

Quoted 8 lines
Quoted 5 lines
Quoted 1 line

mlx5 DPDK is an *RDMA* userspace application.

Forgive me for my naiveté.

Here I thought the RDMA subsystem is for doing RDMA.

RDMA covers a wide range of accelerated networking these days.. Where else are you going to put this stuff in the kernel?

IDK what else you got in there :) It's probably a case by case answer.

IMHO even using libibverbs is no strong reason for things to fall under RDMA exclusively. Client drivers of virtio don't get silently funneled through a separate tree just because they use a certain spec.

Quoted 4 lines
Quoted 2 lines

I'm sure if you start doing crypto over ibverbs crypto people will want to have a look.

Well, RDMA has crypto transforms for a few years now too.

Are you talking about RDMA traffic being encrypted? That's a different case.

My example was alluding to access to a generic crypto accelerator over ibverbs. I hope you'd let crypto people know when merging such a thing...

Quoted 6 lines

Why would crypto subsystem people be involved? It isn't using or duplicating their APIs.

Quoted 3 lines
Quoted 2 lines

libibverbs. It runs on the RDMA stack. It uses RDMA flow filtering and RDMA raw ethernet QPs.

Quoted 7 lines
Quoted 2 lines

I'm not saying that's not the case. I'm saying I don't think this was something that netdev developers signed-off on.

Part of the point of the subsystem split was to end the fighting that started all of it. It was very clear during the whole iWarp and TCP Offload Engine buisness in the mid 2000's that netdev wanted nothing to do with the accelerator world.

I was in middle school at the time, not sure what exactly went down :) But I'm going by common sense here. Perhaps there was an agreement I'm not aware of?

Quoted 1 line

So why would netdev need sign off on any accelerator stuff?

I'm not sure why you keep saying accelerators!

What is accelerated in raw Ethernet frame access??

Quoted 2 lines

Do you want to start co-operating now? I'm willing to talk about how to do that.

IDK how that's even in question. I always try to bump all RDMA-looking stuff to linux-rdma when it's not CCed there. That's the bare minimum of cooperation I'd expect from anyone.

Quoted 5 lines
Quoted 1 line

And our policy on DPDK is pretty widely known.

I honestly have no idea on the netdev DPDK policy,

I'm maintaining the RDMA subsystem not DPDK :)

That's what I thought, but turns out DPDK is your important user.

Quoted 37 lines
Quoted 3 lines

Would you mind pointing us to the introduction of raw Ethernet QPs?

Is there any production use for that without DPDK?

Hmm.. It is very old. RAW (InfiniBand) QPs were part of the original IBA specification cira 2000. When RoCE was defined (around 2010) they were naturally carried forward to Ethernet. The "flow steering" concept to make raw ethernet QP useful was added to verbs around 2012
- 2013. It officially made it upstream in commit 436f2ad05a0b ("IB/core: Export ib_create/destroy_flow through uverbs")

If I recall properly the first real application was ultra low latency ethernet processing for financial applications.

dpdk later adopted the first mlx4 PMD using this libibverbs API around 2015. Interestingly the mlx4 PMD was made through an open source process with minimal involvment from Mellanox, based on the pre-existing RDMA work.

Currently there are many projects, and many open source, built on top of the RDMA raw ethernet QP and RDMA flow steering model. It is now long established kernel ABI.

Quoted 8 lines
Quoted 5 lines

It has been like this for years, it is not some "act".

It is long standing uABI that accelerators like RDMA/etc get to take the traffic before netdev. This cannot be reverted. I don't really understand what you are expecting here?

Same. I don't really know what you expect me to do either. I don't think I can sign-off on kernel changes needed for DPDK.

This patch is fine tuning the shared logic that splits the traffic to accelerator subsystems, I don't think netdev should have a veto here. This needs to be consensus among the various communities and subsystems that rely on this.

Quoted 5 lines

Eli did not explain this well in his commit message. When he said DPDK he means RDMA which is the owner of the FLOW_NAMESPACE. Each accelerator subsystem gets hooked into this, so here VPDA is getting its own hook because re-using the the same hook between two kernel subsystems is buggy.

I'm not so sure about this.

The switchdev modeling is supposed to give users control over flow of traffic in a sane, well defined way, as opposed to magic flow filtering of the early SR-IOV implementations which every vendor had their own twist on.

Now IIUC you're tapping traffic for DPDK/raw QPs _before_ all switching happens in the NIC? That breaks the switchdev model. We're back to per-vendor magic.

And why do you need a separate VDPA table in the first place? Forwarding to a VDPA device has different semantics than forwarding to any other VF/SF?

#24 lore raw

Re: [PATCH mlx5-next 11/16] net/mlx5: Add VDPA priority to NIC RX namespace

Quoted 3 lines · Jakub Kicinski
Quoted 2 lines · On Tue, 24 Nov 2020 15:44:13 -0400 Jason Gunthorpe wrote:
Quoted 1 line · Jakub Kicinski

On Tue, 24 Nov 2020 14:02:10 -0400 Jason Gunthorpe wrote:

[snip]

Quoted 34 lines
Quoted 18 lines
Quoted 11 lines
Quoted 7 lines
Quoted 1 line
Quoted 1 line

It has been like this for years, it is not some "act".

It is long standing uABI that accelerators like RDMA/etc get to take the traffic before netdev. This cannot be reverted. I don't
really understand what you are expecting here?

Same. I don't really know what you expect me to do either. I don't
think I can sign-off on kernel changes needed for DPDK.

This patch is fine tuning the shared logic that splits the traffic to
accelerator subsystems, I don't think netdev should have a veto here. This needs to be consensus among the various communities and subsystems that rely on this.

Quoted 7 lines

Eli did not explain this well in his commit message. When he said DPDK
he means RDMA which is the owner of the FLOW_NAMESPACE. Each accelerator subsystem gets hooked into this, so here VPDA is getting
its own hook because re-using the the same hook between two kernel subsystems is buggy.

I'm not so sure about this.

The switchdev modeling is supposed to give users control over flow of traffic in a sane, well defined way, as opposed to magic flow filtering
of the early SR-IOV implementations which every vendor had their own twist on.

Quoted 4 lines

Now IIUC you're tapping traffic for DPDK/raw QPs _before_ all switching
happens in the NIC? That breaks the switchdev model. We're back to per-vendor magic.

No this is after switching, nothing can precede switching! after switching and forwarding to the correct function/vport, The HW deumx rdma to rdma and eth(rest) to netdev.

Quoted 5 lines

And why do you need a separate VDPA table in the first place? Forwarding to a VDPA device has different semantics than forwarding to
any other VF/SF?

VDPA is yet another "RDMA" Application, similar to raw qp, it is different than VF/SF.

switching can only forward to PF/VF/SF, it doesn't know or care about the end point app (netdev/rdma).

Jakub, this is how rdma works and has been working for the past 20 years :), Jason is well aware of the lack of visibility, and i am sure rdma folks will improve this, they have been improving a lot lately, take rdma_tool for example.

Bottom line the switching model is not the answer for rdma, another model is required, rdma by definition is HW oriented from day one, you can't think of it as an offloaded SW model. ( also in a real switch you can't define if a port is rdma or eth :) )

Anyway you have very valid points that Jason already raised in the past, but the challenge is more complicated than the challenges we have in netdev, simply because RDMA is RDMA, where the leading model is the HW model and the rdma spec and not the SW ..

maintainer
#25 lore raw

Re: [PATCH mlx5-next 11/16] net/mlx5: Add VDPA priority to NIC RX namespace

In reply to Jakub Kicinski
Quoted 4 lines · Jakub Kicinski
Quoted 2 lines

RDMA covers a wide range of accelerated networking these days.. Where else are you going to put this stuff in the kernel?

IDK what else you got in there :) It's probably a case by case answer.

Hmm, yes, it seems endless sometimes :(

Quoted 3 lines

IMHO even using libibverbs is no strong reason for things to fall under RDMA exclusively. Client drivers of virtio don't get silently funneled through a separate tree just because they use a certain spec.

I'm not sure I understand this, libibverbs is the user library to interface with the kernel RDMA subsystem. I don't care what apps people build on top of it, it doesn't matter to me that netdev and DPDK have some kind of feud.

Quoted 7 lines
Quoted 4 lines
Quoted 2 lines

I'm sure if you start doing crypto over ibverbs crypto people will want to have a look.

Well, RDMA has crypto transforms for a few years now too.

Are you talking about RDMA traffic being encrypted? That's a different case.

That too, but in general, anything netdev can do can be done via RDMA in userspace. So all the kTLS and IPSEC xfrm HW offloads mlx5 supports are all available in userspace too.

Quoted 6 lines
Quoted 4 lines

Part of the point of the subsystem split was to end the fighting that started all of it. It was very clear during the whole iWarp and TCP Offload Engine buisness in the mid 2000's that netdev wanted nothing to do with the accelerator world.

I was in middle school at the time, not sure what exactly went down :)

Ah, it was quite the thing. Microsoft and Co were heavilly pushing TOE technology (Microsoft Chimney!) as the next most certain thing and I recall DaveM&co was completely against it in Linux.

I will admit at the time I was doubtful, but in hindsight this was the correct choice. netdev would not look like it does today if it had been shackled by the HW implementations of the day. Instead all this HW stuff ended up largely in RDMA and some in block with the iSCSI mania of old. It is quite evident to me the mess being tied to HW has caused to a SW ecosystem. DRM and RDMA both have a very similiar kind of suffering due to this.

However - over the last 20 years it has been steadfast that there is *always* a compelling reason for certain applications to use something from the accelerator side. It is not for everyone, but the specialized applications that need it, *really need it*.

For instance, it is the difference between being able to get a COVID simulation result in a few week vs .. well.. never.

Quoted 2 lines

But I'm going by common sense here. Perhaps there was an agreement I'm not aware of?

The resolution to the argument above was to split them in Linux. Thus what logically is networking was split up in the kernel between netdev and the accelerator subsystems (iscsi, rdma, and so on).

The general notion is netdev doesn't have to accomodate anything an accelerator does. If you choose to run them then you do not get to complain that your ethtool counters are wrong, your routing tables and tc don't work, firewalling doesn't work. Etc.

That is all broken by design.

In turn, the accelerators do their own thing, tap the traffic before it hits netdev and so on. netdev does not care what goes on over there and is not responsible.

I would say this is the basic unspoken agreement of the last 15 years.

Both have a right to exist in Linux. Both have a right to use the physical ethernet port.

Quoted 5 lines
Quoted 1 line

So why would netdev need sign off on any accelerator stuff?

I'm not sure why you keep saying accelerators!

What is accelerated in raw Ethernet frame access??

The nature of the traffic is not relavent.

It goes through RDMA, it is accelerator traffic (vs netdev traffic, which goes to netdev). Even if you want to be pedantic, in the raw ethernet area there is lots of HW special accelerated stuff going on. Mellanox has some really neat hard realtime networking technology that works on raw ethernet packets, for instance.

And of course raw ethernet is a fraction of what RDMA covers. iWarp and RoCE are much more like you might imagine when you hear the word accelerator.

Quoted 6 lines
Quoted 2 lines

Do you want to start co-operating now? I'm willing to talk about how to do that.

IDK how that's even in question. I always try to bump all RDMA-looking stuff to linux-rdma when it's not CCed there. That's the bare minimum of cooperation I'd expect from anyone.

I mean co-operate in the sense of defining a scheme where the two worlds are not completely seperated and isolated.

Quoted 7 lines
Quoted 5 lines
Quoted 1 line

And our policy on DPDK is pretty widely known.

I honestly have no idea on the netdev DPDK policy,

I'm maintaining the RDMA subsystem not DPDK :)

That's what I thought, but turns out DPDK is your important user.

Nonsense.

I don't have stats but the majority of people I work with using RDMA are not using DPDK. DPDK serves two somewhat niche markets of NFV and certain hyperscalers - RDMA covers the entire scientific computing community and a big swath of the classic "Big Iron" enterprise stuff, like databases and storage.

Quoted 3 lines

Now IIUC you're tapping traffic for DPDK/raw QPs _before_ all switching happens in the NIC? That breaks the switchdev model. We're back to per-vendor magic.

No, as I explained before, the switchdev completely contains the SF/VF and all applications running on a mlx5_core are trapped by it. This includes netdev, RDMA and VDPA.

Quoted 3 lines

And why do you need a separate VDPA table in the first place? Forwarding to a VDPA device has different semantics than forwarding to any other VF/SF?

The VDPA table is not switchdev. Go back to my overly long email about VDPA, here we are talking about the "selector" that chooses which subsystem the traffic will go to. The selector is after switchdev but before netdev, VDPA, RDMA.

Each accelerator subsystem gets a table. RDMA, VDPA, and netdev all get one. It is some part of the HW to make the selectoring work.

Jason

author maintainer
#26 lore raw

PATCH 12/16 net/mlx5: Export steering related functions

In reply to Saeed Mahameed

From: Eli Cohen <[email protected]>

Export
mlx5_create_flow_table()
mlx5_create_flow_group()
mlx5_destroy_flow_group().

These symbols are required by the VDPA implementation to create rules that consume VDPA specific traffic.

We do not deal with putting the prototypes in a header file since they already exist in include/linux/mlx5/fs.h.

---
 drivers/net/ethernet/mellanox/mlx5/core/fs_core.c | 3 +++
 1 file changed, 3 insertions(+)
drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +3 -0
index e095c5968e67..9feab81ab919 100644
@@ -1153,6 +1153,7 @@ struct mlx5_flow_table *mlx5_create_flow_table(struct mlx5_flow_namespace *ns,
 {
 	return __mlx5_create_flow_table(ns, ft_attr, FS_FT_OP_MOD_NORMAL, 0);
 }
+EXPORT_SYMBOL(mlx5_create_flow_table);
 
 struct mlx5_flow_table *mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns,
 						     int prio, int max_fte,
@@ -1244,6 +1245,7 @@ struct mlx5_flow_group *mlx5_create_flow_group(struct mlx5_flow_table *ft,
 
 	return fg;
 }
+EXPORT_SYMBOL(mlx5_create_flow_group);
 
 static struct mlx5_flow_rule *alloc_rule(struct mlx5_flow_destination *dest)
 {
@@ -2146,6 +2148,7 @@ void mlx5_destroy_flow_group(struct mlx5_flow_group *fg)
 		mlx5_core_warn(get_dev(&fg->node), "Flow group %d wasn't destroyed, refcount > 1\n",
 			       fg->id);
 }
+EXPORT_SYMBOL(mlx5_destroy_flow_group);
 
 struct mlx5_flow_namespace *mlx5_get_fdb_sub_ns(struct mlx5_core_dev *dev,
 						int n)
Signature · 1 line
2.26.2
author maintainer
#27 lore raw

PATCH 13/16 net/mlx5: Make API mlx5_core_is_ecpf accept const pointer

In reply to Saeed Mahameed

From: Parav Pandit <[email protected]>

Subsequent patch implements helper API which has mlx5_core_dev as const pointer, make its caller API too const *.

---
 include/linux/mlx5/driver.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/mlx5/driver.h +1 -1
index 5e84b1d53650..d6ef3068d7d3 100644
@@ -1133,7 +1133,7 @@ static inline bool mlx5_core_is_vf(const struct mlx5_core_dev *dev)
 	return dev->coredev_type == MLX5_COREDEV_VF;
 }
 
-static inline bool mlx5_core_is_ecpf(struct mlx5_core_dev *dev)
+static inline bool mlx5_core_is_ecpf(const struct mlx5_core_dev *dev)
 {
 	return dev->caps.embedded_cpu;
 }
Signature · 1 line
2.26.2
author maintainer
#28 lore raw

PATCH 14/16 net/mlx5: Rename peer_pf to host_pf

In reply to Saeed Mahameed

From: Parav Pandit <[email protected]>

To match the hardware spec, rename peer_pf to host_pf.

---
 .../net/ethernet/mellanox/mlx5/core/ecpf.c    | 51 ++++++++++++-------
 .../ethernet/mellanox/mlx5/core/pagealloc.c   | 12 ++---
 include/linux/mlx5/driver.h                   |  2 +-
 3 files changed, 40 insertions(+), 25 deletions(-)
drivers/net/ethernet/mellanox/mlx5/core/ecpf.c +33 -18
index 3dc9dd3f24dc..68ca0e2b26cd 100644
@@ -8,37 +8,52 @@ bool mlx5_read_embedded_cpu(struct mlx5_core_dev *dev)
 	return (ioread32be(&dev->iseg->initializing) >> MLX5_ECPU_BIT_NUM) & 1;
 }
 
-static int mlx5_peer_pf_init(struct mlx5_core_dev *dev)
+static int mlx5_cmd_host_pf_enable_hca(struct mlx5_core_dev *dev)
 {
-	u32 in[MLX5_ST_SZ_DW(enable_hca_in)] = {};
-	int err;
+	u32 out[MLX5_ST_SZ_DW(enable_hca_out)] = {};
+	u32 in[MLX5_ST_SZ_DW(enable_hca_in)]   = {};
 
 	MLX5_SET(enable_hca_in, in, opcode, MLX5_CMD_OP_ENABLE_HCA);
-	err = mlx5_cmd_exec_in(dev, enable_hca, in);
+	MLX5_SET(enable_hca_in, in, function_id, 0);
+	MLX5_SET(enable_hca_in, in, embedded_cpu_function, 0);
+	return mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
+}
+
+static int mlx5_cmd_host_pf_disable_hca(struct mlx5_core_dev *dev)
+{
+	u32 out[MLX5_ST_SZ_DW(disable_hca_out)] = {};
+	u32 in[MLX5_ST_SZ_DW(disable_hca_in)]   = {};
+
+	MLX5_SET(disable_hca_in, in, opcode, MLX5_CMD_OP_DISABLE_HCA);
+	MLX5_SET(disable_hca_in, in, function_id, 0);
+	MLX5_SET(disable_hca_in, in, embedded_cpu_function, 0);
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+}
+
+static int mlx5_host_pf_init(struct mlx5_core_dev *dev)
+{
+	int err;
+
+	err = mlx5_cmd_host_pf_enable_hca(dev);
 	if (err)
-		mlx5_core_err(dev, "Failed to enable peer PF HCA err(%d)\n",
-			      err);
+		mlx5_core_err(dev, "Failed to enable external host PF HCA err(%d)\n", err);
 
 	return err;
 }
 
-static void mlx5_peer_pf_cleanup(struct mlx5_core_dev *dev)
+static void mlx5_host_pf_cleanup(struct mlx5_core_dev *dev)
 {
-	u32 in[MLX5_ST_SZ_DW(disable_hca_in)] = {};
 	int err;
 
-	MLX5_SET(disable_hca_in, in, opcode, MLX5_CMD_OP_DISABLE_HCA);
-	err = mlx5_cmd_exec_in(dev, disable_hca, in);
+	err = mlx5_cmd_host_pf_disable_hca(dev);
 	if (err) {
-		mlx5_core_err(dev, "Failed to disable peer PF HCA err(%d)\n",
-			      err);
+		mlx5_core_err(dev, "Failed to disable external host PF HCA err(%d)\n", err);
 		return;
 	}
 
-	err = mlx5_wait_for_pages(dev, &dev->priv.peer_pf_pages);
+	err = mlx5_wait_for_pages(dev, &dev->priv.host_pf_pages);
 	if (err)
-		mlx5_core_warn(dev, "Timeout reclaiming peer PF pages err(%d)\n",
-			       err);
+		mlx5_core_warn(dev, "Timeout reclaiming external host PF pages err(%d)\n", err);
 }
 
 int mlx5_ec_init(struct mlx5_core_dev *dev)
@@ -46,10 +61,10 @@ int mlx5_ec_init(struct mlx5_core_dev *dev)
 	if (!mlx5_core_is_ecpf(dev))
 		return 0;
 
-	/* ECPF shall enable HCA for peer PF in the same way a PF
+	/* ECPF shall enable HCA for host PF in the same way a PF
 	 * does this for its VFs.
 	 */
-	return mlx5_peer_pf_init(dev);
+	return mlx5_host_pf_init(dev);
 }
 
 void mlx5_ec_cleanup(struct mlx5_core_dev *dev)
@@ -57,5 +72,5 @@ void mlx5_ec_cleanup(struct mlx5_core_dev *dev)
 	if (!mlx5_core_is_ecpf(dev))
 		return;
 
-	mlx5_peer_pf_cleanup(dev);
+	mlx5_host_pf_cleanup(dev);
 }
drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c +6 -6
index 150638814517..539baea358bf 100644
@@ -374,7 +374,7 @@ static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages,
 	if (func_id)
 		dev->priv.vfs_pages += npages;
 	else if (mlx5_core_is_ecpf(dev) && !ec_function)
-		dev->priv.peer_pf_pages += npages;
+		dev->priv.host_pf_pages += npages;
 
 	mlx5_core_dbg(dev, "npages %d, ec_function %d, func_id 0x%x, err %d\n",
 		      npages, ec_function, func_id, err);
@@ -416,7 +416,7 @@ static void release_all_pages(struct mlx5_core_dev *dev, u32 func_id,
 	if (func_id)
 		dev->priv.vfs_pages -= npages;
 	else if (mlx5_core_is_ecpf(dev) && !ec_function)
-		dev->priv.peer_pf_pages -= npages;
+		dev->priv.host_pf_pages -= npages;
 
 	mlx5_core_dbg(dev, "npages %d, ec_function %d, func_id 0x%x\n",
 		      npages, ec_function, func_id);
@@ -506,7 +506,7 @@ static int reclaim_pages(struct mlx5_core_dev *dev, u32 func_id, int npages,
 	if (func_id)
 		dev->priv.vfs_pages -= num_claimed;
 	else if (mlx5_core_is_ecpf(dev) && !ec_function)
-		dev->priv.peer_pf_pages -= num_claimed;
+		dev->priv.host_pf_pages -= num_claimed;
 
 out_free:
 	kvfree(out);
@@ -661,9 +661,9 @@ int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev)
 	WARN(dev->priv.vfs_pages,
 	     "VFs FW pages counter is %d after reclaiming all pages\n",
 	     dev->priv.vfs_pages);
-	WARN(dev->priv.peer_pf_pages,
-	     "Peer PF FW pages counter is %d after reclaiming all pages\n",
-	     dev->priv.peer_pf_pages);
+	WARN(dev->priv.host_pf_pages,
+	     "External host PF FW pages counter is %d after reclaiming all pages\n",
+	     dev->priv.host_pf_pages);
 
 	return 0;
 }
include/linux/mlx5/driver.h +1 -1
index d6ef3068d7d3..8e9bcb3bfd77 100644
@@ -547,7 +547,7 @@ struct mlx5_priv {
 	atomic_t		reg_pages;
 	struct list_head	free_list;
 	int			vfs_pages;
-	int			peer_pf_pages;
+	int			host_pf_pages;
 
 	struct mlx5_core_health health;
 
Signature · 1 line
2.26.2
author maintainer
#29 lore raw

PATCH 15/16 net/mlx5: Enable host PF HCA after eswitch is initialized

In reply to Saeed Mahameed

From: Parav Pandit <[email protected]>

Currently ECPF enables external host PF too early in the initialization sequence for Ethernet links when ECPF is eswitch manager.

Due to this, when external host PF driver is loaded, host PF's HCA CAP has inner_ip_version supported by NIC RX flow table.
This capability is later updated by firmware after ECPF driver enables ENCAP/DECAP as eswitch manager.

This results into a timing race condition, where CREATE_TIR command fails with a below syndrome on host PF.

mlx5_cmd_check:775:(pid 510): CREATE_TIR(0x900) op_mod(0x0) failed, status bad parameter(0x3), syndrome (0x562b00)

Hence, enable the external host PF after necessary eswitch and per vport initialization is completed.
Continue to enable host PF when eswitch manager capability is off for a ECPF.

---
 .../net/ethernet/mellanox/mlx5/core/ecpf.c    | 35 ++++++++++++++-----
 .../net/ethernet/mellanox/mlx5/core/ecpf.h    |  3 ++
 .../net/ethernet/mellanox/mlx5/core/eswitch.c | 29 ++++++++++++++-
 .../net/ethernet/mellanox/mlx5/core/main.c    | 18 +++++-----
 4 files changed, 66 insertions(+), 19 deletions(-)
drivers/net/ethernet/mellanox/mlx5/core/ecpf.c +26 -9
index 68ca0e2b26cd..464eb3a18450 100644
@@ -8,7 +8,16 @@ bool mlx5_read_embedded_cpu(struct mlx5_core_dev *dev)
 	return (ioread32be(&dev->iseg->initializing) >> MLX5_ECPU_BIT_NUM) & 1;
 }
 
-static int mlx5_cmd_host_pf_enable_hca(struct mlx5_core_dev *dev)
+static bool mlx5_ecpf_esw_admins_host_pf(const struct mlx5_core_dev *dev)
+{
+	/* In separate host mode, PF enables itself.
+	 * When ECPF is eswitch manager, eswitch enables host PF after
+	 * eswitch is setup.
+	 */
+	return mlx5_core_is_ecpf_esw_manager(dev);
+}
+
+int mlx5_cmd_host_pf_enable_hca(struct mlx5_core_dev *dev)
 {
 	u32 out[MLX5_ST_SZ_DW(enable_hca_out)] = {};
 	u32 in[MLX5_ST_SZ_DW(enable_hca_in)]   = {};
@@ -19,7 +28,7 @@ static int mlx5_cmd_host_pf_enable_hca(struct mlx5_core_dev *dev)
 	return mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
 }
 
-static int mlx5_cmd_host_pf_disable_hca(struct mlx5_core_dev *dev)
+int mlx5_cmd_host_pf_disable_hca(struct mlx5_core_dev *dev)
 {
 	u32 out[MLX5_ST_SZ_DW(disable_hca_out)] = {};
 	u32 in[MLX5_ST_SZ_DW(disable_hca_in)]   = {};
@@ -34,6 +43,12 @@ static int mlx5_host_pf_init(struct mlx5_core_dev *dev)
 {
 	int err;
 
+	if (mlx5_ecpf_esw_admins_host_pf(dev))
+		return 0;
+
+	/* ECPF shall enable HCA for host PF in the same way a PF
+	 * does this for its VFs when ECPF is not a eswitch manager.
+	 */
 	err = mlx5_cmd_host_pf_enable_hca(dev);
 	if (err)
 		mlx5_core_err(dev, "Failed to enable external host PF HCA err(%d)\n", err);
@@ -45,15 +60,14 @@ static void mlx5_host_pf_cleanup(struct mlx5_core_dev *dev)
 {
 	int err;
 
+	if (mlx5_ecpf_esw_admins_host_pf(dev))
+		return;
+
 	err = mlx5_cmd_host_pf_disable_hca(dev);
 	if (err) {
 		mlx5_core_err(dev, "Failed to disable external host PF HCA err(%d)\n", err);
 		return;
 	}
-
-	err = mlx5_wait_for_pages(dev, &dev->priv.host_pf_pages);
-	if (err)
-		mlx5_core_warn(dev, "Timeout reclaiming external host PF pages err(%d)\n", err);
 }
 
 int mlx5_ec_init(struct mlx5_core_dev *dev)
@@ -61,16 +75,19 @@ int mlx5_ec_init(struct mlx5_core_dev *dev)
 	if (!mlx5_core_is_ecpf(dev))
 		return 0;
 
-	/* ECPF shall enable HCA for host PF in the same way a PF
-	 * does this for its VFs.
-	 */
 	return mlx5_host_pf_init(dev);
 }
 
 void mlx5_ec_cleanup(struct mlx5_core_dev *dev)
 {
+	int err;
+
 	if (!mlx5_core_is_ecpf(dev))
 		return;
 
 	mlx5_host_pf_cleanup(dev);
+
+	err = mlx5_wait_for_pages(dev, &dev->priv.host_pf_pages);
+	if (err)
+		mlx5_core_warn(dev, "Timeout reclaiming external host PF pages err(%d)\n", err);
 }
drivers/net/ethernet/mellanox/mlx5/core/ecpf.h +3 -0
index d3d7a00a02ac..40b6ad76dca6 100644
@@ -17,6 +17,9 @@ bool mlx5_read_embedded_cpu(struct mlx5_core_dev *dev);
 int mlx5_ec_init(struct mlx5_core_dev *dev);
 void mlx5_ec_cleanup(struct mlx5_core_dev *dev);
 
+int mlx5_cmd_host_pf_enable_hca(struct mlx5_core_dev *dev);
+int mlx5_cmd_host_pf_disable_hca(struct mlx5_core_dev *dev);
+
 #else  /* CONFIG_MLX5_ESWITCH */
 
 static inline bool
drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +28 -1
index 6e6a9a563992..dcd8946a843c 100644
@@ -1469,6 +1469,26 @@ int mlx5_eswitch_load_vf_vports(struct mlx5_eswitch *esw, u16 num_vfs,
 	return err;
 }
 
+static int host_pf_enable_hca(struct mlx5_core_dev *dev)
+{
+	if (!mlx5_core_is_ecpf(dev))
+		return 0;
+
+	/* Once vport and representor are ready, take out the external host PF
+	 * out of initializing state. Enabling HCA clears the iser->initializing
+	 * bit and host PF driver loading can progress.
+	 */
+	return mlx5_cmd_host_pf_enable_hca(dev);
+}
+
+static void host_pf_disable_hca(struct mlx5_core_dev *dev)
+{
+	if (!mlx5_core_is_ecpf(dev))
+		return;
+
+	mlx5_cmd_host_pf_disable_hca(dev);
+}
+
 /* mlx5_eswitch_enable_pf_vf_vports() enables vports of PF, ECPF and VFs
  * whichever are present on the eswitch.
  */
@@ -1483,6 +1503,11 @@ mlx5_eswitch_enable_pf_vf_vports(struct mlx5_eswitch *esw,
 	if (ret)
 		return ret;
 
+	/* Enable external host PF HCA */
+	ret = host_pf_enable_hca(esw->dev);
+	if (ret)
+		goto pf_hca_err;
+
 	/* Enable ECPF vport */
 	if (mlx5_ecpf_vport_exists(esw->dev)) {
 		ret = mlx5_eswitch_load_vport(esw, MLX5_VPORT_ECPF, enabled_events);
@@ -1500,8 +1525,9 @@ mlx5_eswitch_enable_pf_vf_vports(struct mlx5_eswitch *esw,
 vf_err:
 	if (mlx5_ecpf_vport_exists(esw->dev))
 		mlx5_eswitch_unload_vport(esw, MLX5_VPORT_ECPF);
-
 ecpf_err:
+	host_pf_disable_hca(esw->dev);
+pf_hca_err:
 	mlx5_eswitch_unload_vport(esw, MLX5_VPORT_PF);
 	return ret;
 }
@@ -1516,6 +1542,7 @@ void mlx5_eswitch_disable_pf_vf_vports(struct mlx5_eswitch *esw)
 	if (mlx5_ecpf_vport_exists(esw->dev))
 		mlx5_eswitch_unload_vport(esw, MLX5_VPORT_ECPF);
 
+	host_pf_disable_hca(esw->dev);
 	mlx5_eswitch_unload_vport(esw, MLX5_VPORT_PF);
 }
 
drivers/net/ethernet/mellanox/mlx5/core/main.c +9 -9
index a9757ccb9d16..d86f06f14cd3 100644
@@ -1126,23 +1126,23 @@ static int mlx5_load(struct mlx5_core_dev *dev)
 		goto err_sriov;
 	}
 
-	err = mlx5_sriov_attach(dev);
-	if (err) {
-		mlx5_core_err(dev, "sriov init failed %d\n", err);
-		goto err_sriov;
-	}
-
 	err = mlx5_ec_init(dev);
 	if (err) {
 		mlx5_core_err(dev, "Failed to init embedded CPU\n");
 		goto err_ec;
 	}
 
+	err = mlx5_sriov_attach(dev);
+	if (err) {
+		mlx5_core_err(dev, "sriov init failed %d\n", err);
+		goto err_sriov;
+	}
+
 	return 0;
 
-err_ec:
-	mlx5_sriov_detach(dev);
 err_sriov:
+	mlx5_ec_cleanup(dev);
+err_ec:
 	mlx5_cleanup_fs(dev);
 err_fs:
 	mlx5_accel_tls_cleanup(dev);
@@ -1168,8 +1168,8 @@ static int mlx5_load(struct mlx5_core_dev *dev)
 
 static void mlx5_unload(struct mlx5_core_dev *dev)
 {
-	mlx5_ec_cleanup(dev);
 	mlx5_sriov_detach(dev);
+	mlx5_ec_cleanup(dev);
 	mlx5_cleanup_fs(dev);
 	mlx5_accel_ipsec_cleanup(dev);
 	mlx5_accel_tls_cleanup(dev);
Signature · 1 line
2.26.2
author maintainer
#30 lore raw

PATCH 16/16 net/mlx5: Treat host PF vport as other (non eswitch manager) vport

In reply to Saeed Mahameed

From: Parav Pandit <[email protected]>

When eswitch manager is running on ECPF, host PF should be treated as non eswitch manager port, similar to other VF vports. Fail to do so, results in firmware treating PF's vport as ECPF vport for eswitch ACL tables.
Non zero check to figure out if a given vport is other vport or not is not sufficient becase PF vport number = 0 on ECPF. Hence, create esw acl tables with an attribute of other vport.

---
 .../mellanox/mlx5/core/esw/acl/helper.c       |  5 +-
 .../net/ethernet/mellanox/mlx5/core/fs_cmd.c  | 54 +++++++++----------
 .../net/ethernet/mellanox/mlx5/core/fs_core.c | 14 ++---
 include/linux/mlx5/fs.h                       |  5 +-
 4 files changed, 34 insertions(+), 44 deletions(-)
drivers/net/ethernet/mellanox/mlx5/core/esw/acl/helper.c +4 -1
index 22f4c1c28006..4a369669e51e 100644
@@ -8,6 +8,7 @@
 struct mlx5_flow_table *
 esw_acl_table_create(struct mlx5_eswitch *esw, u16 vport_num, int ns, int size)
 {
+	struct mlx5_flow_table_attr ft_attr = {};
 	struct mlx5_core_dev *dev = esw->dev;
 	struct mlx5_flow_namespace *root_ns;
 	struct mlx5_flow_table *acl;
@@ -33,7 +34,9 @@ esw_acl_table_create(struct mlx5_eswitch *esw, u16 vport_num, int ns, int size)
 		return ERR_PTR(-EOPNOTSUPP);
 	}
 
-	acl = mlx5_create_vport_flow_table(root_ns, 0, size, 0, vport_num);
+	ft_attr.max_fte = size;
+	ft_attr.flags = MLX5_FLOW_TABLE_OTHER_VPORT;
+	acl = mlx5_create_vport_flow_table(root_ns, &ft_attr, vport_num);
 	if (IS_ERR(acl)) {
 		err = PTR_ERR(acl);
 		esw_warn(dev, "vport[%d] create %s ACL table, err(%d)\n", vport_num,
drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c +24 -30

102 lines not on this page · raw message

drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +4 -10
index 9feab81ab919..761581232139 100644
@@ -1155,17 +1155,11 @@ struct mlx5_flow_table *mlx5_create_flow_table(struct mlx5_flow_namespace *ns,
 }
 EXPORT_SYMBOL(mlx5_create_flow_table);
 
-struct mlx5_flow_table *mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns,
-						     int prio, int max_fte,
-						     u32 level, u16 vport)
+struct mlx5_flow_table *
+mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns,
+			     struct mlx5_flow_table_attr *ft_attr, u16 vport)
 {
-	struct mlx5_flow_table_attr ft_attr = {};
-
-	ft_attr.max_fte = max_fte;
-	ft_attr.level   = level;
-	ft_attr.prio    = prio;
-
-	return __mlx5_create_flow_table(ns, &ft_attr, FS_FT_OP_MOD_NORMAL, vport);
+	return __mlx5_create_flow_table(ns, ft_attr, FS_FT_OP_MOD_NORMAL, vport);
 }
 
 struct mlx5_flow_table*
include/linux/mlx5/fs.h +2 -3
index 97176d623d74..12d84e99ff63 100644
@@ -50,6 +50,7 @@ enum {
 	MLX5_FLOW_TABLE_TUNNEL_EN_DECAP = BIT(1),
 	MLX5_FLOW_TABLE_TERMINATION = BIT(2),
 	MLX5_FLOW_TABLE_UNMANAGED = BIT(3),
+	MLX5_FLOW_TABLE_OTHER_VPORT = BIT(4),
 };
 
 #define LEFTOVERS_RULE_NUM	 2
@@ -175,9 +176,7 @@ mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns,
 
 struct mlx5_flow_table *
 mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns,
-			     int prio,
-			     int num_flow_table_entries,
-			     u32 level, u16 vport);
+			     struct mlx5_flow_table_attr *ft_attr, u16 vport);
 struct mlx5_flow_table *mlx5_create_lag_demux_flow_table(
 					       struct mlx5_flow_namespace *ns,
 					       int prio, u32 level);
Signature · 1 line
2.26.2
#31 lore raw
In reply to Saeed Mahameed
Quoted 14 lines · Saeed Mahameed

Hi,

This series includes trivial updates to mlx5 next branch
1) HW definition for upcoming features
2) Include files and general Cleanups
3) Add the upcoming BlueField-3 device ID
4) Define flow steering priority for VDPA
5) Export missing steering API for ULPs,
will be used later in VDPA driver, to create flow steering domain
for
VDPA queues.
6) ECPF (Embedded CPU Physical function) minor improvements for BlueField.

Series applied to mlx5-next without the VDPA patch.

Jakub, please let me know if you still have concerns regarding that patch, i will eventually need to apply it, regardless of the outcome of the RDMA vs ETH discussion, it is just how we configure the HW :).

Thanks,
Saeed.