ENA release 2.15.0

index f0936a6262..b1fe0bcdf7 100644
@@ -613,6 +613,149 @@ static int ena_com_wait_and_process_admin_cq_polling(struct ena_comp_ctx *comp_c
 	return ret;
 }
 
+static int ena_com_wait_and_process_admin_cq_interrupts(struct ena_comp_ctx *comp_ctx,
+							struct ena_com_admin_queue *admin_queue)
+{
+	unsigned long flags = 0;
+	int ret;
+
+	ENA_WAIT_EVENT_WAIT(comp_ctx->wait_event,
+			    admin_queue->completion_timeout);
+
+	/* In case the command wasn't completed find out the root cause.
+	 * There might be 2 kinds of errors
+	 * 1) No completion (timeout reached)
+	 * 2) There is completion but the device didn't get any msi-x interrupt.
+	 */
+	if (unlikely(comp_ctx->status == ENA_CMD_SUBMITTED)) {
+		ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);
+		ena_com_handle_admin_completion(admin_queue);
+		admin_queue->stats.no_completion++;
+		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);
+
+		ret = ENA_COM_TIMER_EXPIRED;
+		/* Now that the admin queue has been polled, check whether the
+		 * request was fulfilled by the device
+		 */
+		if (comp_ctx->status != ENA_CMD_COMPLETED) {
+			ena_trc_err(admin_queue->ena_dev,
+				    "The ena device didn't send a completion for the admin cmd %d status %d\n",
+				    comp_ctx->cmd_opcode, comp_ctx->status);
+			goto close_aq;
+		}
+
+		admin_queue->is_missing_admin_interrupt = true;
+
+		ena_trc_err(admin_queue->ena_dev,
+			    "The ena device sent a completion but the driver didn't receive a MSI-X interrupt (cmd %d), autopolling mode is %s\n",
+			    comp_ctx->cmd_opcode, admin_queue->auto_polling ? "ON" : "OFF");
+		/* If fallback to polling is not enabled, missing an interrupt
+		 * is considered an error
+		 */
+		if (!admin_queue->auto_polling)
+			goto close_aq;
+
+		ena_com_set_admin_polling_mode(admin_queue->ena_dev, true);
+	} else if (unlikely(comp_ctx->status == ENA_CMD_ABORTED)) {
+		ena_trc_err(admin_queue->ena_dev, "Command was aborted\n");
+		ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);
+		admin_queue->stats.aborted_cmd++;
+		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);
+		ret = ENA_COM_NO_DEVICE;
+		goto err;
+	}
+
+	ret = ena_com_admin_status_to_errno(admin_queue, comp_ctx);
+	comp_ctxt_release(admin_queue, comp_ctx);
+
+	return ret;
+close_aq:
+	admin_queue->running_state = false;
+err:
+	comp_ctxt_release(admin_queue, comp_ctx);
+
+	return ret;
+}
+
+/* There are two types to wait for completion.
+ * Polling mode - wait until the completion is available.
+ * Async mode - wait on wait queue until the completion is ready
+ * (or the timeout expired).
+ * It is expected that the IRQ called ena_com_handle_admin_completion
+ * to mark the completions.
+ */
+static int ena_com_wait_and_process_admin_cq(struct ena_comp_ctx *comp_ctx,
+					     struct ena_com_admin_queue *admin_queue)
+{
+	if (admin_queue->polling)
+		return ena_com_wait_and_process_admin_cq_polling(comp_ctx,
+								 admin_queue);
+
+	return ena_com_wait_and_process_admin_cq_interrupts(comp_ctx,
+							    admin_queue);
+}
+
+/* ena_com_execute_admin_command - Execute admin command
+ * @admin_queue: admin queue.
+ * @cmd: the admin command to execute.
+ * @cmd_size: the command size.
+ * @cmd_completion: command completion return value.
+ * @cmd_comp_size: command completion size.
+
+ * Submit an admin command and then wait until the device returns a
+ * completion.
+ * The completion will be copied into cmd_comp.
+ *
+ * @return - 0 on success, negative value on failure.
+ */
+int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue,
+				  struct ena_admin_aq_entry *cmd,
+				  size_t cmd_size,
+				  struct ena_admin_acq_entry *comp,
+				  size_t comp_size)
+{
+	struct ena_comp_ctx *comp_ctx;
+	int ret;
+
+	comp_ctx = ena_com_submit_admin_cmd(admin_queue, cmd, cmd_size,
+					    comp, comp_size);
+	if (IS_ERR(comp_ctx)) {
+		ret = PTR_ERR(comp_ctx);
+		if (ret != ENA_COM_NO_DEVICE)
+			ena_trc_err(admin_queue->ena_dev,
+				    "Failed to submit command [%d]\n",
+				    ret);
+
+		return ret;
+	}
+
+	ret = ena_com_wait_and_process_admin_cq(comp_ctx, admin_queue);
+	if (unlikely(ret)) {
+		if (admin_queue->running_state)
+			ena_trc_err(admin_queue->ena_dev,
+				    "Failed to process command [%d]\n",
+				    ret);
+	}
+	return ret;
+}
+
+/* ena_com_execute_admin_command_no_resp - Execute admin command without response payload
+ * @admin_queue: admin queue.
+ * @cmd: the admin command to execute.
+ * @cmd_size: the command size.
+ *
+ * Submit an admin command and then wait until the device returns a
+ * completion, no return payload is expected.
+ *
+ * @return - 0 on success, negative value on failure.
+ */
+static int ena_com_execute_admin_command_no_resp(struct ena_com_admin_queue *admin_queue,
+						 struct ena_admin_aq_entry *cmd,
+						 size_t cmd_size)
+{
+	return ena_com_execute_admin_command(admin_queue, cmd, cmd_size, NULL, 0);
+}
+
 /*
  * Set the LLQ configurations of the firmware
  *

@@ -623,7 +766,6 @@ static int ena_com_set_llq(struct ena_com_dev *ena_dev)
 {
 	struct ena_com_llq_info *llq_info = &ena_dev->llq_info;
 	struct ena_com_admin_queue *admin_queue;
-	struct ena_admin_set_feat_resp resp;
 	struct ena_admin_set_feat_cmd cmd;
 	int ret;
 

@@ -642,11 +784,9 @@ static int ena_com_set_llq(struct ena_com_dev *ena_dev)
 		BIT(ENA_ADMIN_DISABLE_META_CACHING) |
 		BIT(ENA_ADMIN_LIMIT_TX_BURST);
 
-	ret = ena_com_execute_admin_command(admin_queue,
-					    (struct ena_admin_aq_entry *)&cmd,
-					    sizeof(cmd),
-					    (struct ena_admin_acq_entry *)&resp,
-					    sizeof(resp));
+	ret = ena_com_execute_admin_command_no_resp(admin_queue,
+						    (struct ena_admin_aq_entry *)&cmd,
+						    sizeof(cmd));
 
 	if (unlikely(ret))
 		ena_trc_err(ena_dev, "Failed to set LLQ configurations: %d\n", ret);

@@ -782,62 +922,6 @@ static int ena_com_config_llq_info(struct ena_com_dev *ena_dev,
 	return rc;
 }
 
-static int ena_com_wait_and_process_admin_cq_interrupts(struct ena_comp_ctx *comp_ctx,
-							struct ena_com_admin_queue *admin_queue)
-{
-	unsigned long flags = 0;
-	int ret;
-
-	ENA_WAIT_EVENT_WAIT(comp_ctx->wait_event,
-			    admin_queue->completion_timeout);
-
-	/* In case the command wasn't completed find out the root cause.
-	 * There might be 2 kinds of errors
-	 * 1) No completion (timeout reached)
-	 * 2) There is completion but the device didn't get any msi-x interrupt.
-	 */
-	if (unlikely(comp_ctx->status == ENA_CMD_SUBMITTED)) {
-		ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);
-		ena_com_handle_admin_completion(admin_queue);
-		admin_queue->stats.no_completion++;
-		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);
-
-		if (comp_ctx->status == ENA_CMD_COMPLETED) {
-			admin_queue->is_missing_admin_interrupt = true;
-			ena_trc_err(admin_queue->ena_dev,
-				    "The ena device sent a completion but the driver didn't receive a MSI-X interrupt (cmd %d), autopolling mode is %s\n",
-				    comp_ctx->cmd_opcode, admin_queue->auto_polling ? "ON" : "OFF");
-			/* Check if fallback to polling is enabled */
-			if (admin_queue->auto_polling)
-				admin_queue->polling = true;
-		} else {
-			ena_trc_err(admin_queue->ena_dev,
-				    "The ena device didn't send a completion for the admin cmd %d status %d\n",
-				    comp_ctx->cmd_opcode, comp_ctx->status);
-		}
-		/* Check if shifted to polling mode.
-		 * This will happen if there is a completion without an interrupt
-		 * and autopolling mode is enabled. Continuing normal execution in such case
-		 */
-		if (!admin_queue->polling) {
-			admin_queue->running_state = false;
-			ret = ENA_COM_TIMER_EXPIRED;
-			goto err;
-		}
-	} else if (unlikely(comp_ctx->status == ENA_CMD_ABORTED)) {
-		ena_trc_err(admin_queue->ena_dev, "Command was aborted\n");
-		ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);
-		admin_queue->stats.aborted_cmd++;
-		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);
-		ret = ENA_COM_NO_DEVICE;
-		goto err;
-	}
-	ret = ena_com_admin_status_to_errno(admin_queue, comp_ctx);
-err:
-	comp_ctxt_release(admin_queue, comp_ctx);
-	return ret;
-}
-
 /* This method read the hardware device register through posting writes
  * and waiting for response
  * On timeout the function will return ENA_MMIO_READ_TIMEOUT

@@ -903,29 +987,10 @@ static u32 ena_com_reg_bar_read32(struct ena_com_dev *ena_dev, u16 offset)
 	return ret;
 }
 
-/* There are two types to wait for completion.
- * Polling mode - wait until the completion is available.
- * Async mode - wait on wait queue until the completion is ready
- * (or the timeout expired).
- * It is expected that the IRQ called ena_com_handle_admin_completion
- * to mark the completions.
- */
-static int ena_com_wait_and_process_admin_cq(struct ena_comp_ctx *comp_ctx,
-					     struct ena_com_admin_queue *admin_queue)
-{
-	if (admin_queue->polling)
-		return ena_com_wait_and_process_admin_cq_polling(comp_ctx,
-								 admin_queue);
-
-	return ena_com_wait_and_process_admin_cq_interrupts(comp_ctx,
-							    admin_queue);
-}
-
 static int ena_com_destroy_io_sq(struct ena_com_dev *ena_dev,
 				 struct ena_com_io_sq *io_sq)
 {
 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
-	struct ena_admin_acq_destroy_sq_resp_desc destroy_resp;
 	struct ena_admin_aq_destroy_sq_cmd destroy_cmd;
 	u8 direction;
 	int ret;

@@ -945,11 +1010,9 @@ static int ena_com_destroy_io_sq(struct ena_com_dev *ena_dev,
 	destroy_cmd.sq.sq_idx = io_sq->idx;
 	destroy_cmd.aq_common_descriptor.opcode = ENA_ADMIN_DESTROY_SQ;
 
-	ret = ena_com_execute_admin_command(admin_queue,
-					    (struct ena_admin_aq_entry *)&destroy_cmd,
-					    sizeof(destroy_cmd),
-					    (struct ena_admin_acq_entry *)&destroy_resp,
-					    sizeof(destroy_resp));
+	ret = ena_com_execute_admin_command_no_resp(admin_queue,
+						    (struct ena_admin_aq_entry *)&destroy_cmd,
+						    sizeof(destroy_cmd));
 
 	if (unlikely(ret && (ret != ENA_COM_NO_DEVICE)))
 		ena_trc_err(ena_dev, "Failed to destroy io sq error: %d\n", ret);

@@ -1152,7 +1215,6 @@ int ena_com_set_hw_timestamping_configuration(struct ena_com_dev *ena_dev,
 					      u8 tx_enable,
 					      u8 rx_enable)
 {
-	struct ena_admin_set_feat_resp resp;
 	struct ena_admin_set_feat_cmd cmd;
 	int ret;
 

@@ -1170,11 +1232,9 @@ int ena_com_set_hw_timestamping_configuration(struct ena_com_dev *ena_dev,
 	cmd.u.hw_ts.rx = rx_enable ? ENA_ADMIN_HW_TIMESTAMP_RX_SUPPORT_ALL :
 				     ENA_ADMIN_HW_TIMESTAMP_RX_SUPPORT_NONE;
 
-	ret = ena_com_execute_admin_command(&ena_dev->admin_queue,
-					    (struct ena_admin_aq_entry *)&cmd,
-					    sizeof(cmd),
-					    (struct ena_admin_acq_entry *)&resp,
-					    sizeof(resp));
+	ret = ena_com_execute_admin_command_no_resp(&ena_dev->admin_queue,
+						    (struct ena_admin_aq_entry *)&cmd,
+						    sizeof(cmd));
 	if (unlikely(ret)) {
 		ena_trc_err(ena_dev,
 			    "Failed to set HW timestamping configuration, error: %d\n",

@@ -1469,37 +1529,6 @@ static void ena_com_update_intr_delay_resolution(struct ena_com_dev *ena_dev,
 /*******************************      API       ******************************/
 /*****************************************************************************/
 
-int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue,
-				  struct ena_admin_aq_entry *cmd,
-				  size_t cmd_size,
-				  struct ena_admin_acq_entry *comp,
-				  size_t comp_size)
-{
-	struct ena_comp_ctx *comp_ctx;
-	int ret;
-
-	comp_ctx = ena_com_submit_admin_cmd(admin_queue, cmd, cmd_size,
-					    comp, comp_size);
-	if (IS_ERR(comp_ctx)) {
-		ret = PTR_ERR(comp_ctx);
-		if (ret != ENA_COM_NO_DEVICE)
-			ena_trc_err(admin_queue->ena_dev,
-				    "Failed to submit command [%d]\n",
-				    ret);
-
-		return ret;
-	}
-
-	ret = ena_com_wait_and_process_admin_cq(comp_ctx, admin_queue);
-	if (unlikely(ret)) {
-		if (admin_queue->running_state)
-			ena_trc_err(admin_queue->ena_dev,
-				    "Failed to process command [%d]\n",
-				    ret);
-	}
-	return ret;
-}
-
 int ena_com_create_io_cq(struct ena_com_dev *ena_dev,
 			 struct ena_com_io_cq *io_cq)
 {

@@ -1608,7 +1637,6 @@ int ena_com_destroy_io_cq(struct ena_com_dev *ena_dev,
 			  struct ena_com_io_cq *io_cq)
 {
 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
-	struct ena_admin_acq_destroy_cq_resp_desc destroy_resp;
 	struct ena_admin_aq_destroy_cq_cmd destroy_cmd;
 	int ret;
 

@@ -1617,11 +1645,9 @@ int ena_com_destroy_io_cq(struct ena_com_dev *ena_dev,
 	destroy_cmd.cq_idx = io_cq->idx;
 	destroy_cmd.aq_common_descriptor.opcode = ENA_ADMIN_DESTROY_CQ;
 
-	ret = ena_com_execute_admin_command(admin_queue,
-					    (struct ena_admin_aq_entry *)&destroy_cmd,
-					    sizeof(destroy_cmd),
-					    (struct ena_admin_acq_entry *)&destroy_resp,
-					    sizeof(destroy_resp));
+	ret = ena_com_execute_admin_command_no_resp(admin_queue,
+						    (struct ena_admin_aq_entry *)&destroy_cmd,
+						    sizeof(destroy_cmd));
 
 	if (unlikely(ret && (ret != ENA_COM_NO_DEVICE)))
 		ena_trc_err(ena_dev, "Failed to destroy IO CQ. error: %d\n", ret);

@@ -1660,7 +1686,6 @@ int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag)
 {
 	struct ena_com_admin_queue *admin_queue;
 	struct ena_admin_get_feat_resp get_resp;
-	struct ena_admin_set_feat_resp resp;
 	struct ena_admin_set_feat_cmd cmd;
 	int ret;
 

@@ -1685,11 +1710,9 @@ int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag)
 	cmd.feat_common.feature_id = ENA_ADMIN_AENQ_CONFIG;
 	cmd.u.aenq.enabled_groups = groups_flag;
 
-	ret = ena_com_execute_admin_command(admin_queue,
-					    (struct ena_admin_aq_entry *)&cmd,
-					    sizeof(cmd),
-					    (struct ena_admin_acq_entry *)&resp,
-					    sizeof(resp));
+	ret = ena_com_execute_admin_command_no_resp(admin_queue,
+						    (struct ena_admin_aq_entry *)&cmd,
+						    sizeof(cmd));
 
 	if (unlikely(ret))
 		ena_trc_err(ena_dev, "Failed to config AENQ ret: %d\n", ret);

@@ -1879,7 +1902,6 @@ int ena_com_phc_config(struct ena_com_dev *ena_dev)
 {
 	struct ena_com_phc_info *phc = &ena_dev->phc;
 	struct ena_admin_get_feat_resp get_feat_resp;
-	struct ena_admin_set_feat_resp set_feat_resp;
 	struct ena_admin_set_feat_cmd set_feat_cmd;
 	int ret = 0;
 

@@ -1934,11 +1956,9 @@ int ena_com_phc_config(struct ena_com_dev *ena_dev)
 	}
 
 	/* Send PHC feature command to the device */
-	ret = ena_com_execute_admin_command(&ena_dev->admin_queue,
-					    (struct ena_admin_aq_entry *)&set_feat_cmd,
-					    sizeof(set_feat_cmd),
-					    (struct ena_admin_acq_entry *)&set_feat_resp,
-					    sizeof(set_feat_resp));
+	ret = ena_com_execute_admin_command_no_resp(&ena_dev->admin_queue,
+						    (struct ena_admin_aq_entry *)&set_feat_cmd,
+						    sizeof(set_feat_cmd));
 
 	if (unlikely(ret)) {
 		ena_trc_err(ena_dev, "Failed to enable PHC, error: %d\n", ret);

@@ -2822,7 +2842,6 @@ int ena_com_get_customer_metrics(struct ena_com_dev *ena_dev, char *buffer, u32
 int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, u32 mtu)
 {
 	struct ena_com_admin_queue *admin_queue;
-	struct ena_admin_set_feat_resp resp;
 	struct ena_admin_set_feat_cmd cmd;
 	int ret;
 

@@ -2839,11 +2858,9 @@ int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, u32 mtu)
 	cmd.feat_common.feature_id = ENA_ADMIN_MTU;
 	cmd.u.mtu.mtu = mtu;
 
-	ret = ena_com_execute_admin_command(admin_queue,
-					    (struct ena_admin_aq_entry *)&cmd,
-					    sizeof(cmd),
-					    (struct ena_admin_acq_entry *)&resp,
-					    sizeof(resp));
+	ret = ena_com_execute_admin_command_no_resp(admin_queue,
+						    (struct ena_admin_aq_entry *)&cmd,
+						    sizeof(cmd));
 
 	if (unlikely(ret))
 		ena_trc_err(ena_dev, "Failed to set mtu %d. error: %d\n", mtu, ret);

@@ -2859,7 +2876,6 @@ int ena_com_set_hash_function(struct ena_com_dev *ena_dev)
 {
 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
 	struct ena_admin_get_feat_resp get_resp;
-	struct ena_admin_set_feat_resp resp;
 	struct ena_rss *rss = &ena_dev->rss;
 	struct ena_admin_set_feat_cmd cmd;
 	int ret;

@@ -2902,11 +2918,9 @@ int ena_com_set_hash_function(struct ena_com_dev *ena_dev)
 
 	cmd.control_buffer.length = sizeof(*rss->hash_key);
 
-	ret = ena_com_execute_admin_command(admin_queue,
-					    (struct ena_admin_aq_entry *)&cmd,
-					    sizeof(cmd),
-					    (struct ena_admin_acq_entry *)&resp,
-					    sizeof(resp));
+	ret = ena_com_execute_admin_command_no_resp(admin_queue,
+						    (struct ena_admin_aq_entry *)&cmd,
+						    sizeof(cmd));
 	if (unlikely(ret)) {
 		ena_trc_err(ena_dev, "Failed to set hash function %d. error: %d\n",
 			    rss->hash_func, ret);

@@ -3031,7 +3045,6 @@ int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev)
 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
 	struct ena_admin_feature_rss_hash_control *hash_ctrl;
 	struct ena_rss *rss = &ena_dev->rss;
-	struct ena_admin_set_feat_resp resp;
 	struct ena_admin_set_feat_cmd cmd;
 	int ret;
 

@@ -3063,11 +3076,9 @@ int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev)
 	}
 	cmd.control_buffer.length = sizeof(*hash_ctrl);
 
-	ret = ena_com_execute_admin_command(admin_queue,
-					    (struct ena_admin_aq_entry *)&cmd,
-					    sizeof(cmd),
-					    (struct ena_admin_acq_entry *)&resp,
-					    sizeof(resp));
+	ret = ena_com_execute_admin_command_no_resp(admin_queue,
+						    (struct ena_admin_aq_entry *)&cmd,
+						    sizeof(cmd));
 	if (unlikely(ret))
 		ena_trc_err(ena_dev, "Failed to set hash input. error: %d\n", ret);
 

@@ -3195,7 +3206,6 @@ int ena_com_indirect_table_set(struct ena_com_dev *ena_dev)
 {
 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
 	struct ena_rss *rss = &ena_dev->rss;
-	struct ena_admin_set_feat_resp resp;
 	struct ena_admin_set_feat_cmd cmd;
 	int ret;
 

@@ -3232,11 +3242,9 @@ int ena_com_indirect_table_set(struct ena_com_dev *ena_dev)
 	cmd.control_buffer.length = (1ULL << rss->tbl_log_size) *
 		sizeof(struct ena_admin_rss_ind_table_entry);
 
-	ret = ena_com_execute_admin_command(admin_queue,
-					    (struct ena_admin_aq_entry *)&cmd,
-					    sizeof(cmd),
-					    (struct ena_admin_acq_entry *)&resp,
-					    sizeof(resp));
+	ret = ena_com_execute_admin_command_no_resp(admin_queue,
+						    (struct ena_admin_aq_entry *)&cmd,
+						    sizeof(cmd));
 
 	if (unlikely(ret))
 		ena_trc_err(ena_dev, "Failed to set indirect table. error: %d\n", ret);

@@ -3420,7 +3428,6 @@ int ena_com_set_host_attributes(struct ena_com_dev *ena_dev)
 {
 	struct ena_host_attribute *host_attr = &ena_dev->host_attr;
 	struct ena_com_admin_queue *admin_queue;
-	struct ena_admin_set_feat_resp resp;
 	struct ena_admin_set_feat_cmd cmd;
 	int ret;
 

@@ -3452,11 +3459,9 @@ int ena_com_set_host_attributes(struct ena_com_dev *ena_dev)
 
 	cmd.u.host_attr.debug_area_size = host_attr->debug_area_size;
 
-	ret = ena_com_execute_admin_command(admin_queue,
-					    (struct ena_admin_aq_entry *)&cmd,
-					    sizeof(cmd),
-					    (struct ena_admin_acq_entry *)&resp,
-					    sizeof(resp));
+	ret = ena_com_execute_admin_command_no_resp(admin_queue,
+						    (struct ena_admin_aq_entry *)&cmd,
+						    sizeof(cmd));
 
 	if (unlikely(ret))
 		ena_trc_err(ena_dev, "Failed to set host attributes: %d\n", ret);

@@ -3579,7 +3584,6 @@ int ena_com_config_dev_mode(struct ena_com_dev *ena_dev,
 
 int ena_com_set_frag_bypass(struct ena_com_dev *ena_dev, bool enable)
 {
-	struct ena_admin_set_feat_resp set_feat_resp;
 	struct ena_com_admin_queue *admin_queue;
 	struct ena_admin_set_feat_cmd cmd;
 	int ret;

@@ -3599,11 +3603,9 @@ int ena_com_set_frag_bypass(struct ena_com_dev *ena_dev, bool enable)
 	cmd.feat_common.feature_version = ENA_ADMIN_FRAG_BYPASS_FEATURE_VERSION_0;
 	cmd.u.frag_bypass.enable = (u8)enable;
 
-	ret = ena_com_execute_admin_command(admin_queue,
-					    (struct ena_admin_aq_entry *)&cmd,
-					    sizeof(cmd),
-					    (struct ena_admin_acq_entry *)&set_feat_resp,
-					    sizeof(set_feat_resp));
+	ret = ena_com_execute_admin_command_no_resp(admin_queue,
+						    (struct ena_admin_aq_entry *)&cmd,
+						    sizeof(cmd));
 
 	if (unlikely(ret))
 		ena_trc_err(ena_dev, "Failed to enable frag bypass. error: %d\n", ret);

index b1fe0bcdf7..737626bcfa 100644
@@ -1996,20 +1996,23 @@ void ena_com_phc_destroy(struct ena_com_dev *ena_dev)
 
 int ena_com_phc_get_timestamp(struct ena_com_dev *ena_dev, u64 *timestamp)
 {
-	volatile struct ena_admin_phc_resp *resp = ena_dev->phc.virt_addr;
 	const ena_time_high_res_t zero_system_time = ENA_TIME_INIT_HIGH_RES();
 	struct ena_com_phc_info *phc = &ena_dev->phc;
+	volatile struct ena_admin_phc_resp *resp;
 	ena_time_high_res_t expire_time;
 	ena_time_high_res_t block_time;
 	unsigned long flags = 0;
 	int ret = ENA_COM_OK;
 
+	ENA_SPINLOCK_LOCK(phc->lock, flags);
+
 	if (!phc->active) {
+		ENA_SPINLOCK_UNLOCK(phc->lock, flags);
 		ena_trc_err(ena_dev, "PHC feature is not active in the device\n");
 		return ENA_COM_UNSUPPORTED;
 	}
 
-	ENA_SPINLOCK_LOCK(phc->lock, flags);
+	resp = ena_dev->phc.virt_addr;
 
 	/* Check if PHC is in blocked state */
 	if (unlikely(ENA_TIME_COMPARE_HIGH_RES(phc->system_time, zero_system_time))) {

index 737626bcfa..49818c4bd0 100644
@@ -2037,27 +2037,34 @@ int ena_com_phc_get_timestamp(struct ena_com_dev *ena_dev, u64 *timestamp)
 				    "PHC get time request 0x%x failed (device error)\n",
 				    phc->req_id);
 			phc->stats.phc_err_dv++;
-		} else if (resp->error_flags & ENA_PHC_ERROR_FLAGS) {
-			/* Device updated req_id during blocking time but got
-			 * a PHC error, this occurs if device:
-			 * - exceeded the get time request limit
-			 * - received an invalid timestamp
-			 * - received an excessively high error bound
-			 * - received an invalid error bound
-			 */
-			ena_trc_err(ena_dev,
-				    "PHC get time request 0x%x failed (error 0x%x)\n",
-				    phc->req_id,
-				    resp->error_flags);
-			phc->stats.phc_err_ts += !!(resp->error_flags &
-				ENA_ADMIN_PHC_ERROR_FLAG_TIMESTAMP);
-			phc->stats.phc_err_eb += !!(resp->error_flags &
-				ENA_ADMIN_PHC_ERROR_FLAG_ERROR_BOUND);
 		} else {
-			/* Device updated req_id during blocking time
-			 * with valid timestamp and error bound
+			/* Ensure PHC payload is read after req_id update
+			 * is observed
 			 */
-			phc->stats.phc_exp++;
+			dma_rmb();
+
+			if (resp->error_flags & ENA_PHC_ERROR_FLAGS) {
+				/* Device updated req_id during blocking time
+				 * but got a PHC error, this occurs if device:
+				 * - exceeded the get time request limit
+				 * - received an invalid timestamp
+				 * - received an excessively high error bound
+				 * - received an invalid error bound
+				 */
+				ena_trc_err(ena_dev,
+					    "PHC get time request 0x%x failed (error 0x%x)\n",
+					    phc->req_id,
+					    resp->error_flags);
+				phc->stats.phc_err_ts += !!(resp->error_flags &
+					ENA_ADMIN_PHC_ERROR_FLAG_TIMESTAMP);
+				phc->stats.phc_err_eb += !!(resp->error_flags &
+					ENA_ADMIN_PHC_ERROR_FLAG_ERROR_BOUND);
+			} else {
+				/* Device updated req_id during blocking time
+				 * with valid timestamp and error bound
+				 */
+				phc->stats.phc_exp++;
+			}
 		}
 	}
 

@@ -2100,6 +2107,11 @@ int ena_com_phc_get_timestamp(struct ena_com_dev *ena_dev, u64 *timestamp)
 			continue;
 		}
 
+		/* Ensure PHC payload (timestamp, error_bound, error_flags)
+		 * is read after req_id update is observed
+		 */
+		dma_rmb();
+
 		/* req_id was updated by the device which indicates that
 		 * PHC timestamp, error_bound and error_flags are updated too,
 		 * checking errors before retrieving timestamp and

index 49818c4bd0..e9b9de21cb 100644
@@ -391,9 +391,9 @@ static int ena_com_init_io_sq(struct ena_com_dev *ena_dev,
 		       sizeof(io_sq->llq_info));
 
 		/* Initiate the first bounce buffer */
-		io_sq->llq_buf_ctrl.curr_bounce_buf =
+		io_sq->llq_buf_ctrl.curr_llq_entry =
 			ena_com_get_next_bounce_buffer(&io_sq->bounce_buf_ctrl);
-		memset(io_sq->llq_buf_ctrl.curr_bounce_buf,
+		memset(io_sq->llq_buf_ctrl.curr_llq_entry,
 		       0x0, io_sq->llq_info.desc_list_entry_size);
 		io_sq->llq_buf_ctrl.descs_left_in_line =
 			io_sq->llq_info.descs_num_before_header;

index 1d024cfea0..bd395fd90c 100644
@@ -138,7 +138,7 @@ struct ena_com_io_bounce_buffer_control {
 
 /* This struct is to keep tracking the current location of the next llq entry */
 struct ena_com_llq_pkt_ctrl {
-	u8 *curr_bounce_buf;
+	u8 *curr_llq_entry;
 	u16 idx;
 	u16 descs_left_in_line;
 };

index bd395fd90c..c62fecdef2 100644
@@ -1139,6 +1139,11 @@ static inline struct ena_com_dev *ena_com_io_cq_to_ena_dev(struct ena_com_io_cq
 	return container_of(io_cq, struct ena_com_dev, io_cq_queues[io_cq->qid]);
 }
 
+static inline bool ena_com_sq_is_llq(struct ena_com_io_sq *io_sq)
+{
+	return io_sq->mem_queue_type != ENA_ADMIN_PLACEMENT_POLICY_HOST;
+}
+
 static inline bool ena_com_get_adaptive_moderation_enabled(struct ena_com_dev *ena_dev)
 {
 	return ena_dev->adaptive_coalescing;

160 lines not on this page · raw message

index f773699467..503cf45cf7 100644
@@ -110,7 +110,7 @@ static inline bool ena_com_sq_have_enough_space(struct ena_com_io_sq *io_sq,
 {
 	int temp;
 
-	if (unlikely(io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST))
+	if (unlikely(!ena_com_sq_is_llq(io_sq)))
 		return ena_com_free_q_entries(io_sq) >= required_buffers;
 
 	/* This calculation doesn't need to be 100% accurate. So to reduce

@@ -136,8 +136,8 @@ static inline bool ena_com_meta_desc_changed(struct ena_com_io_sq *io_sq,
 
 static inline bool is_llq_max_tx_burst_exists(struct ena_com_io_sq *io_sq)
 {
-	return (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) &&
-	       io_sq->llq_info.max_entries_in_tx_burst > 0;
+	return likely(ena_com_sq_is_llq(io_sq)) &&
+	       (io_sq->llq_info.max_entries_in_tx_burst > 0);
 }
 
 static inline bool ena_com_is_doorbell_needed(struct ena_com_io_sq *io_sq,

index e9b9de21cb..d3f1063b6c 100644
@@ -784,6 +784,8 @@ static int ena_com_set_llq(struct ena_com_dev *ena_dev)
 		BIT(ENA_ADMIN_DISABLE_META_CACHING) |
 		BIT(ENA_ADMIN_LIMIT_TX_BURST);
 
+	cmd.feat_common.feature_version = llq_info->feature_version;
+
 	ret = ena_com_execute_admin_command_no_resp(admin_queue,
 						    (struct ena_admin_aq_entry *)&cmd,
 						    sizeof(cmd));

@@ -805,6 +807,8 @@ static int ena_com_config_llq_info(struct ena_com_dev *ena_dev,
 
 	memset(llq_info, 0, sizeof(*llq_info));
 
+	llq_info->feature_version = llq_features->feature_version;
+
 	supported_feat = llq_features->header_location_ctrl_supported;
 
 	if (likely(supported_feat & llq_default_cfg->llq_header_location)) {

index c62fecdef2..7884d0dfc3 100644
@@ -93,6 +93,7 @@ struct ena_com_llq_info {
 	u16 descs_per_entry;
 	u16 max_entries_in_tx_burst;
 	bool disable_meta_caching;
+	u8 feature_version;
 };
 
 struct ena_com_io_cq {

index e3b2a753c4..ca15306859 100644
@@ -204,6 +204,29 @@ static void *get_sq_desc_llq(struct ena_com_io_sq *io_sq)
 	return sq_desc;
 }
 
+static int ena_com_write_and_advance_llq_entry(struct ena_com_io_sq *io_sq)
+{
+	struct ena_com_llq_pkt_ctrl *pkt_ctrl = &io_sq->llq_buf_ctrl;
+	struct ena_com_llq_info *llq_info = &io_sq->llq_info;
+	int rc;
+
+	rc = ena_com_write_bounce_buffer_to_dev(io_sq,
+						pkt_ctrl->curr_llq_entry);
+	if (unlikely(rc)) {
+		ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq),
+			    "Failed to write bounce buffer to device\n");
+		return rc;
+	}
+
+	pkt_ctrl->curr_llq_entry =
+		ena_com_get_next_bounce_buffer(&io_sq->bounce_buf_ctrl);
+
+	memset(io_sq->llq_buf_ctrl.curr_llq_entry,
+	       0x0, llq_info->desc_list_entry_size);
+
+	return ENA_COM_OK;
+}
+
 static int ena_com_close_llq_entry(struct ena_com_io_sq *io_sq)
 {
 	struct ena_com_llq_pkt_ctrl *pkt_ctrl = &io_sq->llq_buf_ctrl;

@@ -215,18 +238,9 @@ static int ena_com_close_llq_entry(struct ena_com_io_sq *io_sq)
 
 	/* bounce buffer was used, so write it and get a new one */
 	if (likely(pkt_ctrl->idx)) {
-		rc = ena_com_write_bounce_buffer_to_dev(io_sq,
-							pkt_ctrl->curr_llq_entry);
-		if (unlikely(rc)) {
-			ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq),
-				    "Failed to write bounce buffer to device\n");
+		rc = ena_com_write_and_advance_llq_entry(io_sq);
+		if (unlikely(rc))
 			return rc;
-		}
-
-		pkt_ctrl->curr_llq_entry =
-			ena_com_get_next_bounce_buffer(&io_sq->bounce_buf_ctrl);
-		memset(io_sq->llq_buf_ctrl.curr_llq_entry,
-		       0x0, llq_info->desc_list_entry_size);
 	}
 
 	pkt_ctrl->idx = 0;

@@ -258,18 +272,9 @@ static int ena_com_sq_update_llq_tail(struct ena_com_io_sq *io_sq)
 	int rc;
 
 	if (!pkt_ctrl->descs_left_in_line) {
-		rc = ena_com_write_bounce_buffer_to_dev(io_sq,
-							pkt_ctrl->curr_llq_entry);
-		if (unlikely(rc)) {
-			ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq),
-				    "Failed to write bounce buffer to device\n");
+		rc = ena_com_write_and_advance_llq_entry(io_sq);
+		if (unlikely(rc))
 			return rc;
-		}
-
-		pkt_ctrl->curr_llq_entry =
-			ena_com_get_next_bounce_buffer(&io_sq->bounce_buf_ctrl);
-		memset(io_sq->llq_buf_ctrl.curr_llq_entry,
-		       0x0, llq_info->desc_list_entry_size);
 
 		pkt_ctrl->idx = 0;
 		pkt_ctrl->descs_left_in_line = llq_info->descs_per_entry;

index d3f1063b6c..51bcf1b321 100644
@@ -397,8 +397,6 @@ static int ena_com_init_io_sq(struct ena_com_dev *ena_dev,
 		       0x0, io_sq->llq_info.desc_list_entry_size);
 		io_sq->llq_buf_ctrl.descs_left_in_line =
 			io_sq->llq_info.descs_num_before_header;
-		io_sq->disable_meta_caching =
-			io_sq->llq_info.disable_meta_caching;
 
 		if (io_sq->llq_info.max_entries_in_tx_burst > 0)
 			io_sq->entries_in_tx_burst_left =

index 7884d0dfc3..063a8f021b 100644
@@ -153,7 +153,6 @@ struct ena_com_io_sq {
 	enum queue_direction direction;
 	enum ena_admin_placement_policy_type mem_queue_type;
 
-	bool disable_meta_caching;
 
 	u32 msix_vector;
 	struct ena_com_tx_meta cached_tx_meta;

index ca15306859..63624d169b 100644
@@ -439,7 +439,7 @@ static int ena_com_create_and_store_tx_meta_desc(struct ena_com_io_sq *io_sq,
 	/* When disable meta caching is set, don't bother to save the meta and
 	 * compare it to the stored version, just create the meta
 	 */
-	if (io_sq->disable_meta_caching) {
+	if (io_sq->llq_info.disable_meta_caching) {
 		*have_meta = true;
 		return ena_com_create_meta(io_sq, ena_meta);
 	}

index 51bcf1b321..1ed73c1890 100644
@@ -367,7 +367,7 @@ static int ena_com_init_io_sq(struct ena_com_dev *ena_dev,
 	} else {
 		/* Allocate bounce buffers */
 		io_sq->bounce_buf_ctrl.buffer_size =
-			ena_dev->llq_info.desc_list_entry_size;
+			ena_dev->llq_info.common.desc_list_entry_size;
 		io_sq->bounce_buf_ctrl.buffers_num =
 			ENA_COM_BOUNCE_BUFFER_CNTRL_CNT;
 		io_sq->bounce_buf_ctrl.next_to_use = 0;

@@ -387,20 +387,20 @@ static int ena_com_init_io_sq(struct ena_com_dev *ena_dev,
 			return ENA_COM_NO_MEM;
 		}
 
-		memcpy(&io_sq->llq_info, &ena_dev->llq_info,
-		       sizeof(io_sq->llq_info));
+		memcpy(&io_sq->llq_runtime.common, &ena_dev->llq_info.common,
+		       sizeof(io_sq->llq_runtime.common));
 
 		/* Initiate the first bounce buffer */
 		io_sq->llq_buf_ctrl.curr_llq_entry =
 			ena_com_get_next_bounce_buffer(&io_sq->bounce_buf_ctrl);
 		memset(io_sq->llq_buf_ctrl.curr_llq_entry,
-		       0x0, io_sq->llq_info.desc_list_entry_size);
+		       0x0, io_sq->llq_runtime.common.desc_list_entry_size);
 		io_sq->llq_buf_ctrl.descs_left_in_line =
-			io_sq->llq_info.descs_num_before_header;
+			io_sq->llq_runtime.common.descs_num_before_header;
 
-		if (io_sq->llq_info.max_entries_in_tx_burst > 0)
-			io_sq->entries_in_tx_burst_left =
-				io_sq->llq_info.max_entries_in_tx_burst;
+		if (io_sq->llq_runtime.common.max_entries_in_tx_burst > 0)
+			io_sq->llq_runtime.entries_in_tx_burst_left =
+				io_sq->llq_runtime.common.max_entries_in_tx_burst;
 	}
 
 	io_sq->tail = 0;

@@ -775,7 +775,7 @@ static int ena_com_set_llq(struct ena_com_dev *ena_dev)
 
 	cmd.u.llq.header_location_ctrl_enabled = llq_info->header_location_ctrl;
 	cmd.u.llq.entry_size_ctrl_enabled = llq_info->desc_list_entry_size_ctrl;
-	cmd.u.llq.desc_num_before_header_enabled = llq_info->descs_num_before_header;
+	cmd.u.llq.desc_num_before_header_enabled = llq_info->common.descs_num_before_header;
 	cmd.u.llq.descriptors_stride_ctrl_enabled = llq_info->desc_stride_ctrl;
 
 	cmd.u.llq.accel_mode.u.set.enabled_flags =

@@ -845,17 +845,17 @@ static int ena_com_config_llq_info(struct ena_com_dev *ena_dev,
 	supported_feat = llq_features->entry_size_ctrl_supported;
 	if (likely(supported_feat & llq_default_cfg->llq_ring_entry_size)) {
 		llq_info->desc_list_entry_size_ctrl = llq_default_cfg->llq_ring_entry_size;
-		llq_info->desc_list_entry_size = llq_default_cfg->llq_ring_entry_size_value;
+		llq_info->common.desc_list_entry_size = llq_default_cfg->llq_ring_entry_size_value;
 	} else {
 		if (supported_feat & ENA_ADMIN_LIST_ENTRY_SIZE_128B) {
 			llq_info->desc_list_entry_size_ctrl = ENA_ADMIN_LIST_ENTRY_SIZE_128B;
-			llq_info->desc_list_entry_size = 128;
+			llq_info->common.desc_list_entry_size = 128;
 		} else if (supported_feat & ENA_ADMIN_LIST_ENTRY_SIZE_192B) {
 			llq_info->desc_list_entry_size_ctrl = ENA_ADMIN_LIST_ENTRY_SIZE_192B;
-			llq_info->desc_list_entry_size = 192;
+			llq_info->common.desc_list_entry_size = 192;
 		} else if (supported_feat & ENA_ADMIN_LIST_ENTRY_SIZE_256B) {
 			llq_info->desc_list_entry_size_ctrl = ENA_ADMIN_LIST_ENTRY_SIZE_256B;
-			llq_info->desc_list_entry_size = 256;
+			llq_info->common.desc_list_entry_size = 256;
 		} else {
 			ena_trc_err(ena_dev, "Invalid entry_size_ctrl, supported: 0x%x\n",
 				    supported_feat);

@@ -865,35 +865,40 @@ static int ena_com_config_llq_info(struct ena_com_dev *ena_dev,
 		ena_trc_err(ena_dev, "Default llq ring entry size is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n",
 			    llq_default_cfg->llq_ring_entry_size,
 			    supported_feat,
-			    llq_info->desc_list_entry_size);
+			    llq_info->common.desc_list_entry_size);
 	}
-	if (unlikely(llq_info->desc_list_entry_size & 0x7)) {
+	if (unlikely(llq_info->common.desc_list_entry_size & 0x7)) {
 		/* The desc list entry size should be whole multiply of 8
 		 * This requirement comes from __iowrite64_copy()
 		 */
 		ena_trc_err(ena_dev, "Illegal entry size %d\n",
-			    llq_info->desc_list_entry_size);
+			    llq_info->common.desc_list_entry_size);
 		return ENA_COM_INVAL;
 	}
 
 	if (llq_info->desc_stride_ctrl == ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY)
-		llq_info->descs_per_entry = llq_info->desc_list_entry_size /
+		llq_info->common.descs_per_entry = llq_info->common.desc_list_entry_size /
 			sizeof(struct ena_eth_io_tx_desc);
 	else
-		llq_info->descs_per_entry = 1;
+		llq_info->common.descs_per_entry = 1;
 
 	supported_feat = llq_features->desc_num_before_header_supported;
 	if (likely(supported_feat & llq_default_cfg->llq_num_decs_before_header)) {
-		llq_info->descs_num_before_header = llq_default_cfg->llq_num_decs_before_header;
+		llq_info->common.descs_num_before_header =
+		llq_default_cfg->llq_num_decs_before_header;
 	} else {
 		if (supported_feat & ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2) {
-			llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2;
+			llq_info->common.descs_num_before_header =
+			ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2;
 		} else if (supported_feat & ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_1) {
-			llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_1;
+			llq_info->common.descs_num_before_header =
+			ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_1;
 		} else if (supported_feat & ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_4) {
-			llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_4;
+			llq_info->common.descs_num_before_header =
+			ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_4;
 		} else if (supported_feat & ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_8) {
-			llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_8;
+			llq_info->common.descs_num_before_header =
+			ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_8;
 		} else {
 			ena_trc_err(ena_dev, "Invalid descs_num_before_header, supported: 0x%x\n",
 				    supported_feat);

@@ -903,19 +908,19 @@ static int ena_com_config_llq_info(struct ena_com_dev *ena_dev,
 		ena_trc_err(ena_dev, "Default llq num descs before header is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n",
 			    llq_default_cfg->llq_num_decs_before_header,
 			    supported_feat,
-			    llq_info->descs_num_before_header);
+			    llq_info->common.descs_num_before_header);
 	}
 	/* Check for accelerated queue supported */
 	llq_accel_mode_get = llq_features->accel_mode.u.get;
 
-	llq_info->disable_meta_caching =
+	llq_info->common.disable_meta_caching =
 		!!(llq_accel_mode_get.supported_flags &
 		   BIT(ENA_ADMIN_DISABLE_META_CACHING));
 
 	if (llq_accel_mode_get.supported_flags & BIT(ENA_ADMIN_LIMIT_TX_BURST))
-		llq_info->max_entries_in_tx_burst =
+		llq_info->common.max_entries_in_tx_burst =
 			llq_accel_mode_get.max_tx_burst_size /
-			llq_default_cfg->llq_ring_entry_size_value;
+			llq_info->common.desc_list_entry_size;
 
 	rc = ena_com_set_llq(ena_dev);
 	if (unlikely(rc))

@@ -1055,7 +1060,8 @@ static void ena_com_io_queue_free(struct ena_com_dev *ena_dev,
 	if (io_sq->bounce_buf_ctrl.base_buffer) {
 		ENA_MEM_FREE(ena_dev->dmadev,
 			     io_sq->bounce_buf_ctrl.base_buffer,
-			     (io_sq->llq_info.desc_list_entry_size * ENA_COM_BOUNCE_BUFFER_CNTRL_CNT));
+			     (io_sq->llq_runtime.common.desc_list_entry_size *
+				ENA_COM_BOUNCE_BUFFER_CNTRL_CNT));
 		io_sq->bounce_buf_ctrl.base_buffer = NULL;
 	}
 }

@@ -3586,8 +3592,8 @@ int ena_com_config_dev_mode(struct ena_com_dev *ena_dev,
 	if (unlikely(rc))
 		return rc;
 
-	ena_dev->tx_max_header_size = llq_info->desc_list_entry_size -
-		(llq_info->descs_num_before_header * sizeof(struct ena_eth_io_tx_desc));
+	ena_dev->tx_max_header_size = llq_info->common.desc_list_entry_size -
+		(llq_info->common.descs_num_before_header * sizeof(struct ena_eth_io_tx_desc));
 
 	if (unlikely(ena_dev->tx_max_header_size == 0)) {
 		ena_trc_err(ena_dev, "The size of the LLQ entry is smaller than needed\n");

index 063a8f021b..94aa285cd8 100644
@@ -84,18 +84,27 @@ struct ena_com_tx_meta {
 	u16 l4_hdr_len; /* In words */
 };
 
-struct ena_com_llq_info {
-	u16 header_location_ctrl;
-	u16 desc_stride_ctrl;
-	u16 desc_list_entry_size_ctrl;
+struct ena_com_llq_info_common {
 	u16 desc_list_entry_size;
 	u16 descs_num_before_header;
 	u16 descs_per_entry;
 	u16 max_entries_in_tx_burst;
 	bool disable_meta_caching;
+};
+
+struct ena_com_llq_info {
+	struct ena_com_llq_info_common common;
+	u16 header_location_ctrl;
+	u16 desc_stride_ctrl;
+	u16 desc_list_entry_size_ctrl;
 	u8 feature_version;
 };
 
+struct ena_com_llq_info_runtime {
+	struct ena_com_llq_info_common common;
+	u16 entries_in_tx_burst_left;
+};
+
 struct ena_com_io_cq {
 	struct ena_com_io_desc_addr cdesc_addr;
 	void *bus;

@@ -156,7 +165,7 @@ struct ena_com_io_sq {
 
 	u32 msix_vector;
 	struct ena_com_tx_meta cached_tx_meta;
-	struct ena_com_llq_info llq_info;
+	struct ena_com_llq_info_runtime llq_runtime;
 	struct ena_com_llq_pkt_ctrl llq_buf_ctrl;
 	struct ena_com_io_bounce_buffer_control bounce_buf_ctrl;
 

@@ -171,7 +180,6 @@ struct ena_com_io_sq {
 	u8 phase;
 	u8 desc_entry_size;
 	u8 dma_addr_bits;
-	u16 entries_in_tx_burst_left;
 } ____cacheline_aligned;
 
 struct ena_com_admin_cq {

index 63624d169b..ea0fb374db 100644
@@ -112,7 +112,7 @@ static void *get_sq_desc_regular_queue(struct ena_com_io_sq *io_sq)
 static int ena_com_write_bounce_buffer_to_dev(struct ena_com_io_sq *io_sq,
 						     u8 *bounce_buffer)
 {
-	struct ena_com_llq_info *llq_info = &io_sq->llq_info;
+	struct ena_com_llq_info_common *llq_info = &io_sq->llq_runtime.common;
 	u16 dst_tail_mask;
 	u32 dst_offset;
 

@@ -120,16 +120,16 @@ static int ena_com_write_bounce_buffer_to_dev(struct ena_com_io_sq *io_sq,
 	dst_offset = dst_tail_mask * llq_info->desc_list_entry_size;
 
 	if (is_llq_max_tx_burst_exists(io_sq)) {
-		if (unlikely(!io_sq->entries_in_tx_burst_left)) {
+		if (unlikely(!io_sq->llq_runtime.entries_in_tx_burst_left)) {
 			ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq),
 				    "Error: trying to send more packets than tx burst allows\n");
 			return ENA_COM_NO_SPACE;
 		}
 
-		io_sq->entries_in_tx_burst_left--;
+		io_sq->llq_runtime.entries_in_tx_burst_left--;
 		ena_trc_dbg(ena_com_io_sq_to_ena_dev(io_sq),
 			    "Decreasing entries_in_tx_burst_left of queue %u to %u\n",
-			    io_sq->qid, io_sq->entries_in_tx_burst_left);
+			    io_sq->qid, io_sq->llq_runtime.entries_in_tx_burst_left);
 	}
 
 	/* Make sure everything was written into the bounce buffer before

@@ -156,7 +156,7 @@ static int ena_com_write_header_to_llq_entry(struct ena_com_io_sq *io_sq,
 					  u16 header_len)
 {
 	struct ena_com_llq_pkt_ctrl *pkt_ctrl = &io_sq->llq_buf_ctrl;
-	struct ena_com_llq_info *llq_info = &io_sq->llq_info;
+	struct ena_com_llq_info_common *llq_info = &io_sq->llq_runtime.common;
 	u8 *curr_llq_entry = pkt_ctrl->curr_llq_entry;
 	u16 header_offset;
 

@@ -207,7 +207,7 @@ static void *get_sq_desc_llq(struct ena_com_io_sq *io_sq)
 static int ena_com_write_and_advance_llq_entry(struct ena_com_io_sq *io_sq)
 {
 	struct ena_com_llq_pkt_ctrl *pkt_ctrl = &io_sq->llq_buf_ctrl;
-	struct ena_com_llq_info *llq_info = &io_sq->llq_info;
+	struct ena_com_llq_info_common *llq_info = &io_sq->llq_runtime.common;
 	int rc;
 
 	rc = ena_com_write_bounce_buffer_to_dev(io_sq,

@@ -230,7 +230,7 @@ static int ena_com_write_and_advance_llq_entry(struct ena_com_io_sq *io_sq)
 static int ena_com_close_llq_entry(struct ena_com_io_sq *io_sq)
 {
 	struct ena_com_llq_pkt_ctrl *pkt_ctrl = &io_sq->llq_buf_ctrl;
-	struct ena_com_llq_info *llq_info = &io_sq->llq_info;
+	struct ena_com_llq_info_common *llq_info = &io_sq->llq_runtime.common;
 	int rc;
 
 	if (unlikely(!ena_com_sq_is_llq(io_sq)))

@@ -268,7 +268,7 @@ static void *get_tx_sq_desc(struct ena_com_io_sq *io_sq)
 static int ena_com_sq_update_llq_tail(struct ena_com_io_sq *io_sq)
 {
 	struct ena_com_llq_pkt_ctrl *pkt_ctrl = &io_sq->llq_buf_ctrl;
-	struct ena_com_llq_info *llq_info = &io_sq->llq_info;
+	struct ena_com_llq_info_common *llq_info = &io_sq->llq_runtime.common;
 	int rc;
 
 	if (!pkt_ctrl->descs_left_in_line) {

@@ -439,7 +439,7 @@ static int ena_com_create_and_store_tx_meta_desc(struct ena_com_io_sq *io_sq,
 	/* When disable meta caching is set, don't bother to save the meta and
 	 * compare it to the stored version, just create the meta
 	 */
-	if (io_sq->llq_info.disable_meta_caching) {
+	if (io_sq->llq_runtime.common.disable_meta_caching) {
 		*have_meta = true;
 		return ena_com_create_meta(io_sq, ena_meta);
 	}

index 503cf45cf7..514df12e9d 100644
@@ -118,7 +118,7 @@ static inline bool ena_com_sq_have_enough_space(struct ena_com_io_sq *io_sq,
 	 * (one for the header line and one to compensate the devision
 	 * down calculation.
 	 */
-	temp = required_buffers / io_sq->llq_info.descs_per_entry + 2;
+	temp = required_buffers / io_sq->llq_runtime.common.descs_per_entry + 2;
 
 	return ena_com_free_q_entries(io_sq) > temp;
 }

@@ -137,13 +137,13 @@ static inline bool ena_com_meta_desc_changed(struct ena_com_io_sq *io_sq,
 static inline bool is_llq_max_tx_burst_exists(struct ena_com_io_sq *io_sq)
 {
 	return likely(ena_com_sq_is_llq(io_sq)) &&
-	       (io_sq->llq_info.max_entries_in_tx_burst > 0);
+	       (io_sq->llq_runtime.common.max_entries_in_tx_burst > 0);
 }
 
 static inline bool ena_com_is_doorbell_needed(struct ena_com_io_sq *io_sq,
 					      struct ena_com_tx_ctx *ena_tx_ctx)
 {
-	struct ena_com_llq_info *llq_info;
+	struct ena_com_llq_info_common *llq_info;
 	int descs_after_first_entry;
 	int num_entries_needed = 1;
 	u16 num_descs;

@@ -151,7 +151,7 @@ static inline bool ena_com_is_doorbell_needed(struct ena_com_io_sq *io_sq,
 	if (!is_llq_max_tx_burst_exists(io_sq))
 		return false;
 
-	llq_info = &io_sq->llq_info;
+	llq_info = &io_sq->llq_runtime.common;
 	num_descs = ena_tx_ctx->num_bufs;
 
 	if (llq_info->disable_meta_caching ||

@@ -168,7 +168,7 @@ static inline bool ena_com_is_doorbell_needed(struct ena_com_io_sq *io_sq,
 		    "Queue: %d num_descs: %d num_entries_needed: %d\n",
 		    io_sq->qid, num_descs, num_entries_needed);
 
-	return num_entries_needed > io_sq->entries_in_tx_burst_left;
+	return num_entries_needed > io_sq->llq_runtime.entries_in_tx_burst_left;
 }
 
 static inline void ena_com_write_rx_sq_doorbell(struct ena_com_io_sq *io_sq)

@@ -184,7 +184,7 @@ static inline void ena_com_write_rx_sq_doorbell(struct ena_com_io_sq *io_sq)
 
 static inline void ena_com_write_tx_sq_doorbell(struct ena_com_io_sq *io_sq)
 {
-	u16 max_entries_in_tx_burst = io_sq->llq_info.max_entries_in_tx_burst;
+	u16 max_entries_in_tx_burst = io_sq->llq_runtime.common.max_entries_in_tx_burst;
 	u16 tail = io_sq->tail;
 
 	ena_trc_dbg(ena_com_io_sq_to_ena_dev(io_sq),

@@ -197,7 +197,7 @@ static inline void ena_com_write_tx_sq_doorbell(struct ena_com_io_sq *io_sq)
 		ena_trc_dbg(ena_com_io_sq_to_ena_dev(io_sq),
 			    "Reset available entries in tx burst for queue %d to %d\n",
 			    io_sq->qid, max_entries_in_tx_burst);
-		io_sq->entries_in_tx_burst_left = max_entries_in_tx_burst;
+		io_sq->llq_runtime.entries_in_tx_burst_left = max_entries_in_tx_burst;
 	}
 }
 

index 1ed73c1890..161d5b5ebd 100644
@@ -3581,6 +3581,7 @@ int ena_com_config_dev_mode(struct ena_com_dev *ena_dev,
 			    struct ena_llq_configurations *llq_default_cfg)
 {
 	struct ena_com_llq_info *llq_info = &ena_dev->llq_info;
+	u32 llq_max_header_size;
 	int rc;
 
 	if (!llq_features->max_llq_num) {

@@ -3592,14 +3593,15 @@ int ena_com_config_dev_mode(struct ena_com_dev *ena_dev,
 	if (unlikely(rc))
 		return rc;
 
-	ena_dev->tx_max_header_size = llq_info->common.desc_list_entry_size -
+	llq_max_header_size = llq_info->common.desc_list_entry_size -
 		(llq_info->common.descs_num_before_header * sizeof(struct ena_eth_io_tx_desc));
 
-	if (unlikely(ena_dev->tx_max_header_size == 0)) {
+	if (unlikely(llq_max_header_size == 0)) {
 		ena_trc_err(ena_dev, "The size of the LLQ entry is smaller than needed\n");
 		return ENA_COM_INVAL;
 	}
 
+	ena_dev->tx_max_header_size = llq_max_header_size;
 	ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_DEV;
 
 	return 0;

312 lines not on this page · raw message

110 lines not on this page · raw message

119 lines not on this page · raw message

128 lines not on this page · raw message

index b204b07767..7a7d4a4036 100644
@@ -15,7 +15,7 @@
 #include <rte_timer.h>
 #include <rte_dev.h>
 #include <rte_net.h>
-
+#include "ena_flow.h"
 #include "ena_com.h"
 
 #define ENA_REGS_BAR	0

@@ -343,6 +343,7 @@ struct ena_adapter {
 	struct ena_stats_dev dev_stats;
 	struct ena_admin_basic_stats basic_stats;
 
+	uint16_t flow_rule_count;
 	u32 *indirect_table;
 	size_t indirect_table_size;
 

542 lines not on this page · raw message

new file mode 100644
index 0000000000..e83aa56e87
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2024 Amazon.com, Inc. or its affiliates.
+ */
+#ifndef _ENA_FLOW_H_
+#define _ENA_FLOW_H_
+
+#include <rte_flow.h>
+
+/* Flow operations */
+extern const struct rte_flow_ops ena_flow_ops;
+
+#endif /* _ENA_FLOW_H_ */

index ea0fb374db..5dbb432acc 100644
@@ -132,10 +132,27 @@ static int ena_com_write_bounce_buffer_to_dev(struct ena_com_io_sq *io_sq,
 			    io_sq->qid, io_sq->llq_runtime.entries_in_tx_burst_left);
 	}
 
-	/* Make sure everything was written into the bounce buffer before
-	 * writing the bounce buffer to the device
+	/* Ensure the previous LLQ entry reaches the device before this one.
+	 *
+	 * The original wmb() was intended to ensure the bounce buffer is
+	 * fully written before it is copied to the device. This is not
+	 * required: a CPU always observes its own prior stores, on every
+	 * architecture. Moreover, wmb() orders stores against later stores,
+	 * not against later loads, so it could not provide this guarantee.
+	 *
+	 * The copy below uses relaxed stores to the WC LLQ region: plain
+	 * stores with no barrier, which the CPU may send to the device in any
+	 * order. On ARM, this means an entry may reach the device partially
+	 * and out of order before the doorbell is written. In addition, the
+	 * barrier in the doorbell write cannot prevent this, as it only orders
+	 * the entries against the doorbell, not against each other.
+	 *
+	 * On x86, this barrier is not needed. Each entry is written as complete
+	 * 64-byte WC lines, all of which reach the device before the
+	 * doorbell: on Intel, the UC doorbell write flushes pending WC
+	 * buffers; on AMD, each line is flushed once it is fully written.
 	 */
-	wmb();
+	arm_wmb();
 
 	/* The line is completed. Copy it to dev */
 	ENA_MEMCPY_TO_DEVICE_64(io_sq->desc_addr.pbuf_dev_addr + dst_offset,

index 4a6d53d0a2..12f1643a85 100644
@@ -73,6 +73,14 @@ typedef RTE_ATOMIC(int32_t) ena_atomic32_t;
 #define rmb rte_rmb
 #define mb rte_mb
 #define mmiowb rte_io_wmb
+
+/* Write memory barrier for ARM architectures - covers 32 and 64 bit ARM */
+#if defined(RTE_ARCH_ARM)
+#define arm_wmb rte_wmb
+#else
+static inline void arm_wmb(void) { }
+#endif
+
 #define __iomem
 
 #ifndef READ_ONCE

index 3eee5e2545..435c2af477 100644
@@ -23,7 +23,7 @@
 #include <ena_eth_io_defs.h>
 
 #define DRV_MODULE_VER_MAJOR	2
-#define DRV_MODULE_VER_MINOR	14
+#define DRV_MODULE_VER_MINOR	15
 #define DRV_MODULE_VER_SUBMINOR	0
 
 #define __MERGE_64B_H_L(h, l) (((uint64_t)h << 32) | l)