Skip to contentSkip to conversation

Add ocode late binding support for CRI

RFC PATCH Superseded Badal Nilawar 4 participants 11 messages Mar 5 – May 29

Subsystems (2)

v1 of 7 Superseded Latest is v7 →

lore
Share & tools
lore
#0 lore raw

cover Add ocode late binding support for CRI

Ocode is the OOBMSM telemetry/manageability firmware, and on CRI it is loaded through late binding flow.

Dependancy: There is dependancy on mei side of patches to support ocode late binding,
which are not upstreamed yet. Hence, this patch is marked as RFC.

Badal Nilawar (4):
drm/xe/xe_late_bind_fw: Refactor pm flow drm/xe/xe_late_bind_fw: Add support to load Ocode firmware on CRI drm/xe/xe_late_bind_fw: Track firmware load status drm/xe/xe_late_bind_fw: Enable late binding support for CRI

 drivers/gpu/drm/xe/xe_late_bind_fw.c       | 143 ++++++++++++++++++---
 drivers/gpu/drm/xe/xe_late_bind_fw.h       |   7 +-
 drivers/gpu/drm/xe/xe_late_bind_fw_types.h |  25 ++++
 drivers/gpu/drm/xe/xe_pci.c                |   1 +
 drivers/gpu/drm/xe/xe_pm.c                 |   9 +-
 5 files changed, 161 insertions(+), 24 deletions(-)
Signature · 1 line
2.52.0
#1 lore raw

RFC PATCH 1/4 drm/xe/xe_late_bind_fw: Refactor pm flow

Refactor late binding flow across suspend/resume by adding dedicated pm functions.

---
 drivers/gpu/drm/xe/xe_late_bind_fw.c | 32 +++++++++++++++++++++-------
 drivers/gpu/drm/xe/xe_late_bind_fw.h |  4 ++--
 drivers/gpu/drm/xe/xe_pm.c           |  6 +++---
 3 files changed, 29 insertions(+), 13 deletions(-)
drivers/gpu/drm/xe/xe_late_bind_fw.c +24 -8
index 768442ca7da6d..5ca25186dc0cc 100644
@@ -193,7 +193,7 @@ static int xe_late_bind_fw_num_fans(struct xe_late_bind *late_bind, u32 *num_fan
 			     PCODE_MBOX(FAN_SPEED_CONTROL, FSC_READ_NUM_FANS, 0), num_fans, NULL);
 }
 
-void xe_late_bind_wait_for_worker_completion(struct xe_late_bind *late_bind)
+static void xe_late_bind_wait_for_worker_completion(struct xe_late_bind *late_bind)
 {
 	struct xe_device *xe = late_bind_to_xe(late_bind);
 	struct xe_late_bind_fw *lbfw;
@@ -269,18 +269,12 @@ static void xe_late_bind_work(struct work_struct *work)
 	xe_pm_runtime_put(xe);
 }
 
-int xe_late_bind_fw_load(struct xe_late_bind *late_bind)
+static int xe_late_bind_fw_load(struct xe_late_bind *late_bind)
 {
 	struct xe_device *xe = late_bind_to_xe(late_bind);
 	struct xe_late_bind_fw *lbfw;
 	int fw_id;
 
-	if (!late_bind->component_added)
-		return -ENODEV;
-
-	if (late_bind->disable)
-		return 0;
-
 	for (fw_id = 0; fw_id < XE_LB_FW_MAX_ID; fw_id++) {
 		lbfw = &late_bind->late_bind_fw[fw_id];
 		if (lbfw->payload) {
@@ -424,6 +418,28 @@ static void xe_late_bind_remove(void *arg)
 	}
 }
 
+void xe_late_bind_pm_suspend(struct xe_late_bind *late_bind)
+{
+	if (!late_bind->component_added)
+		return;
+
+	if (late_bind->disable)
+		return;
+
+	xe_late_bind_wait_for_worker_completion(late_bind);
+}
+
+void xe_late_bind_pm_resume(struct xe_late_bind *late_bind)
+{
+	if (!late_bind->component_added)
+		return;
+
+	if (late_bind->disable)
+		return;
+
+	xe_late_bind_fw_load(late_bind);
+}
+
 /**
  * xe_late_bind_init() - add xe mei late binding component
  * @late_bind: pointer to late bind structure.
drivers/gpu/drm/xe/xe_late_bind_fw.h +2 -2
index 07e4373905395..8896fea3068e5 100644
@@ -11,7 +11,7 @@
 struct xe_late_bind;
 
 int xe_late_bind_init(struct xe_late_bind *late_bind);
-int xe_late_bind_fw_load(struct xe_late_bind *late_bind);
-void xe_late_bind_wait_for_worker_completion(struct xe_late_bind *late_bind);
+void xe_late_bind_pm_suspend(struct xe_late_bind *late_bind);
+void xe_late_bind_pm_resume(struct xe_late_bind *late_bind);
 
 #endif
drivers/gpu/drm/xe/xe_pm.c +3 -3
index 01185f10a8835..f85b12c1a85d2 100644
@@ -182,7 +182,7 @@ int xe_pm_suspend(struct xe_device *xe)
 	if (err)
 		goto err;
 
-	xe_late_bind_wait_for_worker_completion(&xe->late_bind);
+	xe_late_bind_pm_suspend(&xe->late_bind);
 
 	for_each_gt(gt, xe, id)
 		xe_gt_suspend_prepare(gt);
@@ -284,7 +284,7 @@ int xe_pm_resume(struct xe_device *xe)
 	if (IS_VF_CCS_READY(xe))
 		xe_sriov_vf_ccs_register_context(xe);
 
-	xe_late_bind_fw_load(&xe->late_bind);
+	xe_late_bind_pm_resume(&xe->late_bind);
 
 	drm_dbg(&xe->drm, "Device resumed\n");
 	xe_pm_block_end_signalling();
@@ -698,7 +698,7 @@ int xe_pm_runtime_resume(struct xe_device *xe)
 		xe_sriov_vf_ccs_register_context(xe);
 
 	if (xe->d3cold.allowed)
-		xe_late_bind_fw_load(&xe->late_bind);
+		xe_late_bind_pm_resume(&xe->late_bind);
 
 out:
 	xe_rpm_lockmap_release(xe);
Signature · 1 line
2.52.0
maintainer Reviewed-by on v1 1/4
#2 lore raw

Re: [RFC PATCH 1/4] drm/xe/xe_late_bind_fw: Refactor pm flow

Quoted 15 lines · Badal Nilawar

Refactor late binding flow across suspend/resume by adding dedicated pm functions.

---
 drivers/gpu/drm/xe/xe_late_bind_fw.c | 32 +++++++++++++++++++++-------
 drivers/gpu/drm/xe/xe_late_bind_fw.h |  4 ++--
 drivers/gpu/drm/xe/xe_pm.c           |  6 +++---
 3 files changed, 29 insertions(+), 13 deletions(-)
drivers/gpu/drm/xe/xe_late_bind_fw.c +0 -0
index 768442ca7da6d..5ca25186dc0cc 100644

drivers/gpu/drm/xe/xe_late_bind_fw.c static int xe_late_bind_fw_num_fans(struct xe_late_bind *late_bind, u32 *num_fan · line 196

Quoted 5 lines
drivers/gpu/drm/xe/xe_late_bind_fw.c +1 -1
@@ -193,7 +193,7 @@ static int xe_late_bind_fw_num_fans(struct xe_late_bind *late_bind, u32 *num_fan
 			     PCODE_MBOX(FAN_SPEED_CONTROL, FSC_READ_NUM_FANS, 0), num_fans, NULL);
 }
 
-void xe_late_bind_wait_for_worker_completion(struct xe_late_bind *late_bind)
+static void xe_late_bind_wait_for_worker_completion(struct xe_late_bind *late_bind)

since it is now static, you can perhaps remove entirely the prefix 'xe_late_bind_' to make it shorter.

I prefer without it but I don't believe it hurts to keep it. Up to you:

Quoted 100 lines
drivers/gpu/drm/xe/xe_late_bind_fw.c +23 -7
 {
 	struct xe_device *xe = late_bind_to_xe(late_bind);
 	struct xe_late_bind_fw *lbfw;
@@ -269,18 +269,12 @@ static void xe_late_bind_work(struct work_struct *work)
 	xe_pm_runtime_put(xe);
 }
 
-int xe_late_bind_fw_load(struct xe_late_bind *late_bind)
+static int xe_late_bind_fw_load(struct xe_late_bind *late_bind)
 {
 	struct xe_device *xe = late_bind_to_xe(late_bind);
 	struct xe_late_bind_fw *lbfw;
 	int fw_id;
 
-	if (!late_bind->component_added)
-		return -ENODEV;
-
-	if (late_bind->disable)
-		return 0;
-
 	for (fw_id = 0; fw_id < XE_LB_FW_MAX_ID; fw_id++) {
 		lbfw = &late_bind->late_bind_fw[fw_id];
 		if (lbfw->payload) {
@@ -424,6 +418,28 @@ static void xe_late_bind_remove(void *arg)
 	}
 }
 
+void xe_late_bind_pm_suspend(struct xe_late_bind *late_bind)
+{
+	if (!late_bind->component_added)
+		return;
+
+	if (late_bind->disable)
+		return;
+
+	xe_late_bind_wait_for_worker_completion(late_bind);
+}
+
+void xe_late_bind_pm_resume(struct xe_late_bind *late_bind)
+{
+	if (!late_bind->component_added)
+		return;
+
+	if (late_bind->disable)
+		return;
+
+	xe_late_bind_fw_load(late_bind);
+}
+
 /**
  * xe_late_bind_init() - add xe mei late binding component
  * @late_bind: pointer to late bind structure.
drivers/gpu/drm/xe/xe_late_bind_fw.h +2 -2
index 07e4373905395..8896fea3068e5 100644
@@ -11,7 +11,7 @@
 struct xe_late_bind;
 
 int xe_late_bind_init(struct xe_late_bind *late_bind);
-int xe_late_bind_fw_load(struct xe_late_bind *late_bind);
-void xe_late_bind_wait_for_worker_completion(struct xe_late_bind *late_bind);
+void xe_late_bind_pm_suspend(struct xe_late_bind *late_bind);
+void xe_late_bind_pm_resume(struct xe_late_bind *late_bind);
 
 #endif
drivers/gpu/drm/xe/xe_pm.c +3 -3
index 01185f10a8835..f85b12c1a85d2 100644
@@ -182,7 +182,7 @@ int xe_pm_suspend(struct xe_device *xe)
 	if (err)
 		goto err;
 
-	xe_late_bind_wait_for_worker_completion(&xe->late_bind);
+	xe_late_bind_pm_suspend(&xe->late_bind);
 
 	for_each_gt(gt, xe, id)
 		xe_gt_suspend_prepare(gt);
@@ -284,7 +284,7 @@ int xe_pm_resume(struct xe_device *xe)
 	if (IS_VF_CCS_READY(xe))
 		xe_sriov_vf_ccs_register_context(xe);
 
-	xe_late_bind_fw_load(&xe->late_bind);
+	xe_late_bind_pm_resume(&xe->late_bind);
 
 	drm_dbg(&xe->drm, "Device resumed\n");
 	xe_pm_block_end_signalling();
@@ -698,7 +698,7 @@ int xe_pm_runtime_resume(struct xe_device *xe)
 		xe_sriov_vf_ccs_register_context(xe);
 
 	if (xe->d3cold.allowed)
-		xe_late_bind_fw_load(&xe->late_bind);
+		xe_late_bind_pm_resume(&xe->late_bind);
 
 out:
 	xe_rpm_lockmap_release(xe);
Signature · 1 line
2.52.0
#3 lore raw

RFC PATCH 2/4 drm/xe/xe_late_bind_fw: Add support to load Ocode firmware on CRI

In reply to Badal Nilawar

Ocode is the OOBMSM telemetry/manageability firmware, and on CRI it is loaded through late binding flow.

---
 drivers/gpu/drm/xe/xe_late_bind_fw.c       | 49 +++++++++++++++++-----
 drivers/gpu/drm/xe/xe_late_bind_fw_types.h |  2 +
 2 files changed, 40 insertions(+), 11 deletions(-)
drivers/gpu/drm/xe/xe_late_bind_fw.c +38 -11
index 5ca25186dc0cc..d3ef2926517bd 100644
@@ -33,10 +33,12 @@
 
 static const u32 fw_id_to_type[] = {
 		[XE_LB_FW_FAN_CONTROL] = INTEL_LB_TYPE_FAN_CONTROL,
+		[XE_LB_FW_OCODE] = INTEL_LB_TYPE_OCODE,
 	};
 
 static const char * const fw_id_to_name[] = {
 		[XE_LB_FW_FAN_CONTROL] = "fan_control",
+		[XE_LB_FW_OCODE] = "ocode",
 	};
 
 static struct xe_device *
@@ -179,6 +181,34 @@ static const char *xe_late_bind_parse_status(uint32_t status)
 		return "Invalid Payload";
 	case INTEL_LB_STATUS_TIMEOUT:
 		return "Timeout";
+	case INTEL_LB_STATUS_INTERNAL_ERROR:
+		return "Internal Error";
+	case INTEL_LB_STATUS_INVALID_FPT_TABLE:
+		return "Invalid FPT Table";
+	case INTEL_LB_STATUS_SIGNED_PAYLOAD_VERIFICATION_ERROR:
+		return "Signed Payload Verification Error";
+	case INTEL_LB_STATUS_SIGNED_PAYLOAD_INVALID_CPD:
+		return "Signed Payload Invalid CPD";
+	case INTEL_LB_STATUS_SIGNED_PAYLOAD_FW_VERSION_MISMATCH:
+		return "Signed Payload FW Version Mismatch";
+	case INTEL_LB_STATUS_SIGNED_PAYLOAD_INVALID_MANIFEST:
+		return "Signed Payload Invalid Manifest";
+	case INTEL_LB_STATUS_SIGNED_PAYLOAD_INVALID_HASH:
+		return "Signed Payload Invalid Hash";
+	case INTEL_LB_STATUS_SIGNED_PAYLOAD_BINDING_TYPE_MISMATCH:
+		return "Signed Payload Binding type Mismatch";
+	case INTEL_LB_STATUS_SIGNED_PAYLOAD_HANDLE_SVN_FAILED:
+		return "Signed Payload Handle SVN Failed";
+	case INTEL_LB_STATUS_DESTINATION_MBOX_FAILURE:
+		return "Destination MBOX Failure";
+	case INTEL_LB_STATUS_MISSING_LOADING_PATCH:
+		return "Missing Loading Patch";
+	case INTEL_LB_STATUS_INVALID_COMMAND:
+		return "Invalid Command";
+	case INTEL_LB_STATUS_INVALID_HECI_HEADER:
+		return "Invalid HECI Header";
+	case INTEL_LB_STATUS_IP_ERROR_START:
+		return "IP Error Start";
 	default:
 		return "Unknown error";
 	}
@@ -264,7 +294,6 @@ static void xe_late_bind_work(struct work_struct *work)
 	/* Do not re-attempt fw load */
 	drmm_kfree(&xe->drm, (void *)lbfw->payload);
 	lbfw->payload = NULL;
-
 out:
 	xe_pm_runtime_put(xe);
 }
@@ -314,9 +343,14 @@ static int __xe_late_bind_fw_init(struct xe_late_bind *late_bind, u32 fw_id)
 			return 0;
 	}
 
-	snprintf(lb_fw->blob_path, sizeof(lb_fw->blob_path), "xe/%s_8086_%04x_%04x_%04x.bin",
-		 fw_id_to_name[lb_fw->id], pdev->device,
-		 pdev->subsystem_vendor, pdev->subsystem_device);
+	if (lb_fw->type == INTEL_LB_TYPE_OCODE) {
+		snprintf(lb_fw->blob_path, sizeof(lb_fw->blob_path), "xe/%s_8086_%04x.bin",
+			 fw_id_to_name[lb_fw->id], pdev->device);
+	} else {
+		snprintf(lb_fw->blob_path, sizeof(lb_fw->blob_path), "xe/%s_8086_%04x_%04x_%04x.bin",
+			 fw_id_to_name[lb_fw->id], pdev->device,
+			 pdev->subsystem_vendor, pdev->subsystem_device);
+	}
 
 	drm_dbg(&xe->drm, "Request late binding firmware %s\n", lb_fw->blob_path);
 	ret = firmware_request_nowarn(&fw, lb_fw->blob_path, xe->drm.dev);
@@ -326,13 +360,6 @@ static int __xe_late_bind_fw_init(struct xe_late_bind *late_bind, u32 fw_id)
 		return 0;
 	}
 
-	if (fw->size > XE_LB_MAX_PAYLOAD_SIZE) {
-		drm_err(&xe->drm, "Firmware %s size %zu is larger than max pay load size %u\n",
-			lb_fw->blob_path, fw->size, XE_LB_MAX_PAYLOAD_SIZE);
-		release_firmware(fw);
-		return -ENODATA;
-	}
-
 	ret = parse_lb_layout(lb_fw, fw->data, fw->size, "LTES");
 	if (ret)
 		return ret;
drivers/gpu/drm/xe/xe_late_bind_fw_types.h +2 -0
index 2a8a985c37e71..a128e3f638a6c 100644
@@ -20,6 +20,8 @@
 enum xe_late_bind_fw_id {
 	/** @XE_LB_FW_FAN_CONTROL: Fan control */
 	XE_LB_FW_FAN_CONTROL = 0,
+	/** @XE_LB_FW_OCODE: Ocode firmware */
+	XE_LB_FW_OCODE,
 	/** @XE_LB_FW_MAX_ID: Number of IDs */
 	XE_LB_FW_MAX_ID
 };
Signature · 1 line
2.52.0
#4 lore raw

Re: [RFC PATCH 2/4] drm/xe/xe_late_bind_fw: Add support to load Ocode firmware on CRI

Quoted 62 lines · Badal Nilawar

Ocode is the OOBMSM telemetry/manageability firmware, and on CRI it is loaded through late binding flow.

---
  drivers/gpu/drm/xe/xe_late_bind_fw.c       | 49 +++++++++++++++++-----
  drivers/gpu/drm/xe/xe_late_bind_fw_types.h |  2 +
  2 files changed, 40 insertions(+), 11 deletions(-)
drivers/gpu/drm/xe/xe_late_bind_fw.c +30 -0
index 5ca25186dc0cc..d3ef2926517bd 100644
@@ -33,10 +33,12 @@
 
  static const u32 fw_id_to_type[] = {
  		[XE_LB_FW_FAN_CONTROL] = INTEL_LB_TYPE_FAN_CONTROL,
+		[XE_LB_FW_OCODE] = INTEL_LB_TYPE_OCODE,
  	};
 
  static const char * const fw_id_to_name[] = {
  		[XE_LB_FW_FAN_CONTROL] = "fan_control",
+		[XE_LB_FW_OCODE] = "ocode",
  	};
 
  static struct xe_device *
@@ -179,6 +181,34 @@ static const char *xe_late_bind_parse_status(uint32_t status)
  		return "Invalid Payload";
  	case INTEL_LB_STATUS_TIMEOUT:
  		return "Timeout";
+	case INTEL_LB_STATUS_INTERNAL_ERROR:
+		return "Internal Error";
+	case INTEL_LB_STATUS_INVALID_FPT_TABLE:
+		return "Invalid FPT Table";
+	case INTEL_LB_STATUS_SIGNED_PAYLOAD_VERIFICATION_ERROR:
+		return "Signed Payload Verification Error";
+	case INTEL_LB_STATUS_SIGNED_PAYLOAD_INVALID_CPD:
+		return "Signed Payload Invalid CPD";
+	case INTEL_LB_STATUS_SIGNED_PAYLOAD_FW_VERSION_MISMATCH:
+		return "Signed Payload FW Version Mismatch";
+	case INTEL_LB_STATUS_SIGNED_PAYLOAD_INVALID_MANIFEST:
+		return "Signed Payload Invalid Manifest";
+	case INTEL_LB_STATUS_SIGNED_PAYLOAD_INVALID_HASH:
+		return "Signed Payload Invalid Hash";
+	case INTEL_LB_STATUS_SIGNED_PAYLOAD_BINDING_TYPE_MISMATCH:
+		return "Signed Payload Binding type Mismatch";
+	case INTEL_LB_STATUS_SIGNED_PAYLOAD_HANDLE_SVN_FAILED:
+		return "Signed Payload Handle SVN Failed";
+	case INTEL_LB_STATUS_DESTINATION_MBOX_FAILURE:
+		return "Destination MBOX Failure";
+	case INTEL_LB_STATUS_MISSING_LOADING_PATCH:
+		return "Missing Loading Patch";
+	case INTEL_LB_STATUS_INVALID_COMMAND:
+		return "Invalid Command";
+	case INTEL_LB_STATUS_INVALID_HECI_HEADER:
+		return "Invalid HECI Header";
+	case INTEL_LB_STATUS_IP_ERROR_START:
+		return "IP Error Start";
  	default:
  		return "Unknown error";
  	}

drivers/gpu/drm/xe/xe_late_bind_fw.c static void xe_late_bind_work(struct work_struct *work) · line 267

Quoted 4 lines
drivers/gpu/drm/xe/xe_late_bind_fw.c +0 -1
@@ -264,7 +294,6 @@ static void xe_late_bind_work(struct work_struct *work)
  	/* Do not re-attempt fw load */
  	drmm_kfree(&xe->drm, (void *)lbfw->payload);
  	lbfw->payload = NULL;
-

I think it'd better to keep an empty line before the "out" label.

drivers/gpu/drm/xe/xe_late_bind_fw.c static int __xe_late_bind_fw_init(struct xe_late_bind *late_bind, u32 fw_id) · line 348

Quoted 13 lines
drivers/gpu/drm/xe/xe_late_bind_fw.c +3 -3
  out:
  	xe_pm_runtime_put(xe);
  }
@@ -314,9 +343,14 @@ static int __xe_late_bind_fw_init(struct xe_late_bind *late_bind, u32 fw_id)
  			return 0;
  	}
 
-	snprintf(lb_fw->blob_path, sizeof(lb_fw->blob_path), "xe/%s_8086_%04x_%04x_%04x.bin",
-		 fw_id_to_name[lb_fw->id], pdev->device,
-		 pdev->subsystem_vendor, pdev->subsystem_device);
+	if (lb_fw->type == INTEL_LB_TYPE_OCODE) {
+		snprintf(lb_fw->blob_path, sizeof(lb_fw->blob_path), "xe/%s_8086_%04x.bin",
+			 fw_id_to_name[lb_fw->id], pdev->device);

Are we expecting different ocodes for each PCI id? if not, we could name this like the other fw binaries as <platform>_ocode.bin (e.g., cri_ocode.bin)

Quoted 9 lines
drivers/gpu/drm/xe/xe_late_bind_fw.c +5 -0
+	} else {
+		snprintf(lb_fw->blob_path, sizeof(lb_fw->blob_path), "xe/%s_8086_%04x_%04x_%04x.bin",
+			 fw_id_to_name[lb_fw->id], pdev->device,
+			 pdev->subsystem_vendor, pdev->subsystem_device);
+	}
 
  	drm_dbg(&xe->drm, "Request late binding firmware %s\n", lb_fw->blob_path);
  	ret = firmware_request_nowarn(&fw, lb_fw->blob_path, xe->drm.dev);

drivers/gpu/drm/xe/xe_late_bind_fw.c static int __xe_late_bind_fw_init(struct xe_late_bind *late_bind, u32 fw_id) · line 334

Quoted 9 lines
drivers/gpu/drm/xe/xe_late_bind_fw.c +0 -6
@@ -326,13 +360,6 @@ static int __xe_late_bind_fw_init(struct xe_late_bind *late_bind, u32 fw_id)
  		return 0;
  	}
 
-	if (fw->size > XE_LB_MAX_PAYLOAD_SIZE) {
-		drm_err(&xe->drm, "Firmware %s size %zu is larger than max pay load size %u\n",
-			lb_fw->blob_path, fw->size, XE_LB_MAX_PAYLOAD_SIZE);
-		release_firmware(fw);
-		return -ENODATA;
-	}

Why is this check being removed? is MEI/CSC now going to accept bigger payloads?

Quoted 9 lines
drivers/gpu/drm/xe/xe_late_bind_fw.c +0 -1
-
  	ret = parse_lb_layout(lb_fw, fw->data, fw->size, "LTES");
  	if (ret)
  		return ret;
drivers/gpu/drm/xe/xe_late_bind_fw_types.h +0 -0
index 2a8a985c37e71..a128e3f638a6c 100644

drivers/gpu/drm/xe/xe_late_bind_fw_types.h line 24

Quoted 5 lines
drivers/gpu/drm/xe/xe_late_bind_fw_types.h +2 -0
@@ -20,6 +20,8 @@
  enum xe_late_bind_fw_id {
  	/** @XE_LB_FW_FAN_CONTROL: Fan control */
  	XE_LB_FW_FAN_CONTROL = 0,
+	/** @XE_LB_FW_OCODE: Ocode firmware */
+	XE_LB_FW_OCODE,

This will now automatically look for ocode for bmg as well. not an issue, but potentially noise in dmesg. Maybe we could change has_late_bind to be a mask of supported fw types? that way we can set it to BIT(XE_LB_FW_FAN_CONTROL) on BMG and BIT(XE_LB_FW_OCODE) on CRI and skip the FW type if not set in the mask. Not a blocker.

Daniele

Quoted 3 lines
drivers/gpu/drm/xe/xe_late_bind_fw_types.h +0 -0
  	/** @XE_LB_FW_MAX_ID: Number of IDs */
  	XE_LB_FW_MAX_ID
  };
#5 lore raw

Re: [RFC PATCH 2/4] drm/xe/xe_late_bind_fw: Add support to load Ocode firmware on CRI

Quoted 62 lines · Daniele Ceraolo Spurio
Quoted 59 lines · Badal Nilawar

Ocode is the OOBMSM telemetry/manageability firmware, and on CRI it is loaded through late binding flow.

---
  drivers/gpu/drm/xe/xe_late_bind_fw.c       | 49 +++++++++++++++++----- drivers/gpu/drm/xe/xe_late_bind_fw_types.h |  2 +

  2 files changed, 40 insertions(+), 11 deletions(-)
drivers/gpu/drm/xe/xe_late_bind_fw.c +2 -0
index 5ca25186dc0cc..d3ef2926517bd 100644
@@ -33,10 +33,12 @@
     static const u32 fw_id_to_type[] = {
           [XE_LB_FW_FAN_CONTROL] = INTEL_LB_TYPE_FAN_CONTROL,
+        [XE_LB_FW_OCODE] = INTEL_LB_TYPE_OCODE,
       };
     static const char * const fw_id_to_name[] = {
           [XE_LB_FW_FAN_CONTROL] = "fan_control",
+        [XE_LB_FW_OCODE] = "ocode",
       };

    static struct xe_device *

drivers/gpu/drm/xe/xe_late_bind_fw.c +0 -0
@@ -179,6 +181,34 @@ static const char 
 *xe_late_bind_parse_status(uint32_t status)
           return "Invalid Payload";

      case INTEL_LB_STATUS_TIMEOUT:
          return "Timeout";
+    case INTEL_LB_STATUS_INTERNAL_ERROR:
+        return "Internal Error";
+    case INTEL_LB_STATUS_INVALID_FPT_TABLE:
+        return "Invalid FPT Table";
+    case INTEL_LB_STATUS_SIGNED_PAYLOAD_VERIFICATION_ERROR:
+        return "Signed Payload Verification Error"; +    case INTEL_LB_STATUS_SIGNED_PAYLOAD_INVALID_CPD:
+        return "Signed Payload Invalid CPD";
+    case INTEL_LB_STATUS_SIGNED_PAYLOAD_FW_VERSION_MISMATCH:
+        return "Signed Payload FW Version Mismatch"; +    case INTEL_LB_STATUS_SIGNED_PAYLOAD_INVALID_MANIFEST:
+        return "Signed Payload Invalid Manifest";
+    case INTEL_LB_STATUS_SIGNED_PAYLOAD_INVALID_HASH:
+        return "Signed Payload Invalid Hash";
+    case INTEL_LB_STATUS_SIGNED_PAYLOAD_BINDING_TYPE_MISMATCH:
+        return "Signed Payload Binding type Mismatch"; +    case INTEL_LB_STATUS_SIGNED_PAYLOAD_HANDLE_SVN_FAILED:
+        return "Signed Payload Handle SVN Failed";
+    case INTEL_LB_STATUS_DESTINATION_MBOX_FAILURE:
+        return "Destination MBOX Failure";
+    case INTEL_LB_STATUS_MISSING_LOADING_PATCH:
+        return "Missing Loading Patch";
+    case INTEL_LB_STATUS_INVALID_COMMAND:
+        return "Invalid Command";
+    case INTEL_LB_STATUS_INVALID_HECI_HEADER:
+        return "Invalid HECI Header";
+    case INTEL_LB_STATUS_IP_ERROR_START:
+        return "IP Error Start";
      default:
          return "Unknown error";
      }

Quoted 8 lines

drivers/gpu/drm/xe/xe_late_bind_fw.c static void xe_late_bind_work(struct work_struct · line 268

Quoted 6 lines
drivers/gpu/drm/xe/xe_late_bind_fw.c +0 -1
@@ -264,7 +294,6 @@ static void xe_late_bind_work(struct work_struct 
 *work)
       /* Do not re-attempt fw load */
       drmm_kfree(&xe->drm, (void *)lbfw->payload);
       lbfw->payload = NULL;
-

I think it'd better to keep an empty line before the "out" label.

Sure.

Quoted 17 lines
Quoted 5 lines

  out:
      xe_pm_runtime_put(xe);
  }

drivers/gpu/drm/xe/xe_late_bind_fw.c static int __xe_late_bind_fw_init(struct · line 350

Quoted 10 lines
drivers/gpu/drm/xe/xe_late_bind_fw.c +3 -2
@@ -314,9 +343,14 @@ static int __xe_late_bind_fw_init(struct 
 xe_late_bind *late_bind, u32 fw_id)
               return 0;
       }
   -    snprintf(lb_fw->blob_path, sizeof(lb_fw->blob_path), "xe/%s_8086_%04x_%04x_%04x.bin",
-         fw_id_to_name[lb_fw->id], pdev->device,
-         pdev->subsystem_vendor, pdev->subsystem_device);
+    if (lb_fw->type == INTEL_LB_TYPE_OCODE) {
+        snprintf(lb_fw->blob_path, sizeof(lb_fw->blob_path), 
 "xe/%s_8086_%04x.bin",
+             fw_id_to_name[lb_fw->id], pdev->device);

Are we expecting different ocodes for each PCI id? if not, we could name this like the other fw binaries as <platform>_ocode.bin (e.g., cri_ocode.bin)

Yes, ocode firmware is 2-ID based.

Quoted 14 lines
Quoted 14 lines
drivers/gpu/drm/xe/xe_late_bind_fw.c +5 -0
+    } else {
+        snprintf(lb_fw->blob_path, sizeof(lb_fw->blob_path), 
 "xe/%s_8086_%04x_%04x_%04x.bin",
+             fw_id_to_name[lb_fw->id], pdev->device,
+             pdev->subsystem_vendor, pdev->subsystem_device);
+    }

        drm_dbg(&xe->drm, "Request late binding firmware %s\n", lb_fw->blob_path); ret = firmware_request_nowarn(&fw, lb_fw->blob_path, xe->drm.dev);

drivers/gpu/drm/xe/xe_late_bind_fw.c +0 -0
@@ -326,13 +360,6 @@ static int __xe_late_bind_fw_init(struct 
 xe_late_bind *late_bind, u32 fw_id)
           return 0;
       }
   -    if (fw->size > XE_LB_MAX_PAYLOAD_SIZE) {
Quoted 8 lines

drivers/gpu/drm/xe/xe_late_bind_fw.c static int __xe_late_bind_fw_init(struct xe_late_bind *late_bind, u32 fw_id) · line 334

Quoted 6 lines
drivers/gpu/drm/xe/xe_late_bind_fw.c +0 -5
-        drm_err(&xe->drm, "Firmware %s size %zu is larger than max 
 pay load size %u\n",
-            lb_fw->blob_path, fw->size, XE_LB_MAX_PAYLOAD_SIZE);
-        release_firmware(fw);
-        return -ENODATA;
-    }

Why is this check being removed? is MEI/CSC now going to accept bigger payloads?

Yes, ocode firmware size is 80k+. So dropped this check. MEI driver sends the payload in chunks of 4k.

Quoted 17 lines
Quoted 10 lines
drivers/gpu/drm/xe/xe_late_bind_fw.c +0 -1
-
       ret = parse_lb_layout(lb_fw, fw->data, fw->size, "LTES");

      if (ret)
          return ret;

drivers/gpu/drm/xe/xe_late_bind_fw_types.h +0 -0
index 2a8a985c37e71..a128e3f638a6c 100644

drivers/gpu/drm/xe/xe_late_bind_fw_types.h line 24

Quoted 5 lines
drivers/gpu/drm/xe/xe_late_bind_fw_types.h +2 -0
@@ -20,6 +20,8 @@
   enum xe_late_bind_fw_id {
       /** @XE_LB_FW_FAN_CONTROL: Fan control */
       XE_LB_FW_FAN_CONTROL = 0,
+    /** @XE_LB_FW_OCODE: Ocode firmware */
+    XE_LB_FW_OCODE,

This will now automatically look for ocode for bmg as well. not an issue, but potentially noise in dmesg. Maybe we could change has_late_bind to be a mask of supported fw types? that way we can set it to BIT(XE_LB_FW_FAN_CONTROL) on BMG and BIT(XE_LB_FW_OCODE) on CRI and skip the FW type if not set in the mask. Not a blocker.

Fine I will modify has_late_bind  to late_bind_mask.

Thanks,
Badal

Quoted 7 lines

Daniele

Quoted 3 lines
drivers/gpu/drm/xe/xe_late_bind_fw_types.h +0 -0
       /** @XE_LB_FW_MAX_ID: Number of IDs */

      XE_LB_FW_MAX_ID
  };

#6 lore raw

RFC PATCH 3/4 drm/xe/xe_late_bind_fw: Track firmware load status

In reply to Badal Nilawar

In CRI, Intel PMT depends on the ocode runtime status. Track the late-binding firmware load status and expose an API, for xe_vsec driver, to query it.

---
 drivers/gpu/drm/xe/xe_late_bind_fw.c       | 68 +++++++++++++++++++++-
 drivers/gpu/drm/xe/xe_late_bind_fw.h       |  5 +-
 drivers/gpu/drm/xe/xe_late_bind_fw_types.h | 23 ++++++++
 drivers/gpu/drm/xe/xe_pm.c                 |  5 +-
 4 files changed, 96 insertions(+), 5 deletions(-)
drivers/gpu/drm/xe/xe_late_bind_fw.c +66 -2
index d3ef2926517bd..87137d6e4aaec 100644
@@ -53,6 +53,32 @@ late_bind_fw_to_xe(struct xe_late_bind_fw *lb_fw)
 	return container_of(lb_fw, struct xe_device, late_bind.late_bind_fw[lb_fw->id]);
 }
 
+static const char *xe_late_bind_fw_status_to_str(enum xe_late_bind_fw_status status)
+{
+	switch (status) {
+	case XE_LB_FW_NOT_SUPPORTED:
+		return "Not Supported";
+	case XE_LB_FW_AVAILABLE:
+		return "Available";
+	case XE_LB_FW_RUNNING:
+		return "Running";
+	case XE_LB_FW_FAILED:
+		return "Failed";
+	default:
+		return "Unknown";
+	}
+}
+
+static void xe_late_bind_change_fw_status(struct xe_late_bind_fw *lb_fw,
+					  enum xe_late_bind_fw_status status)
+{
+	struct xe_device *xe = late_bind_fw_to_xe(lb_fw);
+
+	lb_fw->__status = status;
+	drm_dbg(&xe->drm, "%s firmware status changed to %s\n",
+		fw_id_to_name[lb_fw->id], xe_late_bind_fw_status_to_str(status));
+}
+
 /* Refer to the "Late Bind based Firmware Layout" documentation entry for details */
 static int parse_cpd_header(struct xe_late_bind_fw *lb_fw,
 			    const void *data, size_t size, const char *manifest_entry)
@@ -235,6 +261,13 @@ static void xe_late_bind_wait_for_worker_completion(struct xe_late_bind *late_bi
 			drm_dbg(&xe->drm, "Flush work: load %s firmware\n",
 				fw_id_to_name[lbfw->id]);
 			flush_work(&lbfw->work);
+
+			/*
+			 * flush is being called during component unbind, suspend etc.
+			 * So mark status XE_LB_FW_RUNNING -> XE_LB_FW_AVAILABLE
+			 */
+			if (lbfw->__status == XE_LB_FW_RUNNING)
+				xe_late_bind_change_fw_status(lbfw, XE_LB_FW_AVAILABLE);
 		}
 	}
 }
@@ -282,6 +315,7 @@ static void xe_late_bind_work(struct work_struct *work)
 	if (!ret) {
 		drm_dbg(&xe->drm, "Load %s firmware successful\n",
 			fw_id_to_name[lbfw->id]);
+		xe_late_bind_change_fw_status(lbfw, XE_LB_FW_RUNNING);
 		goto out;
 	}
 
@@ -294,10 +328,23 @@ static void xe_late_bind_work(struct work_struct *work)
 	/* Do not re-attempt fw load */
 	drmm_kfree(&xe->drm, (void *)lbfw->payload);
 	lbfw->payload = NULL;
+	xe_late_bind_change_fw_status(lbfw, XE_LB_FW_FAILED);
 out:
 	xe_pm_runtime_put(xe);
 }
 
+static void  xe_late_bind_fw_stop(struct xe_late_bind *late_bind)
+{
+	struct xe_late_bind_fw *lbfw;
+	int fw_id;
+
+	for (fw_id = 0; fw_id < XE_LB_FW_MAX_ID; fw_id++) {
+		lbfw = &late_bind->late_bind_fw[fw_id];
+		if (lbfw->payload && lbfw->status == XE_LB_FW_RUNNING)
+			xe_late_bind_change_fw_status(lbfw, XE_LB_FW_AVAILABLE);
+	}
+}
+
 static int xe_late_bind_fw_load(struct xe_late_bind *late_bind)
 {
 	struct xe_device *xe = late_bind_to_xe(late_bind);
@@ -331,6 +378,7 @@ static int __xe_late_bind_fw_init(struct xe_late_bind *late_bind, u32 fw_id)
 	lb_fw->id = fw_id;
 	lb_fw->type = fw_id_to_type[lb_fw->id];
 	lb_fw->flags &= ~INTEL_LB_FLAG_IS_PERSISTENT;
+	xe_late_bind_change_fw_status(lb_fw, XE_LB_FW_NOT_SUPPORTED);
 
 	if (lb_fw->type == INTEL_LB_TYPE_FAN_CONTROL) {
 		ret = xe_late_bind_fw_num_fans(late_bind, &num_fans);
@@ -379,6 +427,7 @@ static int __xe_late_bind_fw_init(struct xe_late_bind *late_bind, u32 fw_id)
 	memcpy((void *)lb_fw->payload, fw->data, lb_fw->payload_size);
 	release_firmware(fw);
 	INIT_WORK(&lb_fw->work, xe_late_bind_work);
+	xe_late_bind_change_fw_status(lb_fw, XE_LB_FW_AVAILABLE);
 
 	return 0;
 }
@@ -445,15 +494,30 @@ static void xe_late_bind_remove(void *arg)
 	}
 }
 
-void xe_late_bind_pm_suspend(struct xe_late_bind *late_bind)
+/**
+ * xe_late_bind_ocode_status() - get ocode late bind firmware status
+ * @xe: pointer to xe device.
+ *
+ * Return: 0 if unsupported, 1 if available, 2 if running, 3 if failed.
+ */
+int xe_late_bind_ocode_status(struct xe_device *xe)
+{
+	struct xe_late_bind_fw *lbfw = &xe->late_bind.late_bind_fw[XE_LB_FW_OCODE];
+
+	return lbfw->status;
+}
+
+void xe_late_bind_pm_suspend(struct xe_late_bind *late_bind, bool is_runtime)
 {
 	if (!late_bind->component_added)
 		return;
 
 	if (late_bind->disable)
 		return;
+	if (!is_runtime)
+		xe_late_bind_wait_for_worker_completion(late_bind);
 
-	xe_late_bind_wait_for_worker_completion(late_bind);
+	xe_late_bind_fw_stop(late_bind);
 }
 
 void xe_late_bind_pm_resume(struct xe_late_bind *late_bind)
drivers/gpu/drm/xe/xe_late_bind_fw.h +3 -2
index 8896fea3068e5..b441f5789f24b 100644
@@ -9,9 +9,10 @@
 #include <linux/types.h>
 
 struct xe_late_bind;
+struct xe_device;
 
 int xe_late_bind_init(struct xe_late_bind *late_bind);
-void xe_late_bind_pm_suspend(struct xe_late_bind *late_bind);
+void xe_late_bind_pm_suspend(struct xe_late_bind *late_bind, bool is_runtime);
 void xe_late_bind_pm_resume(struct xe_late_bind *late_bind);
-
+int xe_late_bind_ocode_status(struct xe_device *xe);
 #endif
drivers/gpu/drm/xe/xe_late_bind_fw_types.h +23 -0
index a128e3f638a6c..4cdbd775b103e 100644
@@ -26,6 +26,20 @@ enum xe_late_bind_fw_id {
 	XE_LB_FW_MAX_ID
 };
 
+/**
+ * enum xe_late_bind_fw_status - enum to determine late binding fw status
+ */
+enum xe_late_bind_fw_status {
+	/** @XE_LB_FW_NOT_SUPPORTED: firmware not available/supported */
+	XE_LB_FW_NOT_SUPPORTED,
+	/** @XE_LB_FW_AVAILABLE: firmware found and copied to memory */
+	XE_LB_FW_AVAILABLE,
+	/** @XE_LB_FW_RUNNING: firmware is running */
+	XE_LB_FW_RUNNING,
+	/** @XE_LB_FW_FAILED: firmware load is failed */
+	XE_LB_FW_FAILED,
+};
+
 /**
  * struct xe_late_bind_fw
  */
@@ -46,6 +60,15 @@ struct xe_late_bind_fw {
 	struct work_struct work;
 	/** @version: late binding blob manifest version */
 	struct gsc_version version;
+	union {
+		/** @status: firmware load status */
+		const enum xe_late_bind_fw_status status;
+		/**
+		 * @__status: private firmware load status - only to be used
+		 * by firmware loading code
+		 */
+		enum xe_late_bind_fw_status __status;
+	};
 };
 
 /**
drivers/gpu/drm/xe/xe_pm.c +4 -1
index f85b12c1a85d2..9aa873f57b5ad 100644
@@ -182,7 +182,7 @@ int xe_pm_suspend(struct xe_device *xe)
 	if (err)
 		goto err;
 
-	xe_late_bind_pm_suspend(&xe->late_bind);
+	xe_late_bind_pm_suspend(&xe->late_bind, false);
 
 	for_each_gt(gt, xe, id)
 		xe_gt_suspend_prepare(gt);
@@ -612,6 +612,9 @@ int xe_pm_runtime_suspend(struct xe_device *xe)
 			goto out_resume;
 	}
 
+	if (xe->d3cold.allowed)
+		xe_late_bind_pm_suspend(&xe->late_bind, true);
+
 	xe_irq_suspend(xe);
 
 	xe_display_pm_runtime_suspend_late(xe);
Signature · 1 line
2.52.0
#7 lore raw

Re: [RFC PATCH 3/4] drm/xe/xe_late_bind_fw: Track firmware load status

Quoted 3 lines · Badal Nilawar

In CRI, Intel PMT depends on the ocode runtime status. Track the late-binding firmware load status and expose an API, for xe_vsec driver, to query it.

I think it would be better to send this patch with the series that adds the xe_vsec changes, otherwise we have no idea from a review POV on how you plan to use the new exposed API.

Quoted 46 lines
---
  drivers/gpu/drm/xe/xe_late_bind_fw.c       | 68 +++++++++++++++++++++-
  drivers/gpu/drm/xe/xe_late_bind_fw.h       |  5 +-
  drivers/gpu/drm/xe/xe_late_bind_fw_types.h | 23 ++++++++
  drivers/gpu/drm/xe/xe_pm.c                 |  5 +-
  4 files changed, 96 insertions(+), 5 deletions(-)
drivers/gpu/drm/xe/xe_late_bind_fw.c +26 -0
index d3ef2926517bd..87137d6e4aaec 100644
@@ -53,6 +53,32 @@ late_bind_fw_to_xe(struct xe_late_bind_fw *lb_fw)
  	return container_of(lb_fw, struct xe_device, late_bind.late_bind_fw[lb_fw->id]);
  }
 
+static const char *xe_late_bind_fw_status_to_str(enum xe_late_bind_fw_status status)
+{
+	switch (status) {
+	case XE_LB_FW_NOT_SUPPORTED:
+		return "Not Supported";
+	case XE_LB_FW_AVAILABLE:
+		return "Available";
+	case XE_LB_FW_RUNNING:
+		return "Running";
+	case XE_LB_FW_FAILED:
+		return "Failed";
+	default:
+		return "Unknown";
+	}
+}
+
+static void xe_late_bind_change_fw_status(struct xe_late_bind_fw *lb_fw,
+					  enum xe_late_bind_fw_status status)
+{
+	struct xe_device *xe = late_bind_fw_to_xe(lb_fw);
+
+	lb_fw->__status = status;
+	drm_dbg(&xe->drm, "%s firmware status changed to %s\n",
+		fw_id_to_name[lb_fw->id], xe_late_bind_fw_status_to_str(status));
+}
+
  /* Refer to the "Late Bind based Firmware Layout" documentation entry for details */
  static int parse_cpd_header(struct xe_late_bind_fw *lb_fw,
  			    const void *data, size_t size, const char *manifest_entry)

drivers/gpu/drm/xe/xe_late_bind_fw.c static void xe_late_bind_wait_for_worker_completion(struct xe_late_bind *late_bi · line 270

Quoted 10 lines
drivers/gpu/drm/xe/xe_late_bind_fw.c +7 -0
@@ -235,6 +261,13 @@ static void xe_late_bind_wait_for_worker_completion(struct xe_late_bind *late_bi
  			drm_dbg(&xe->drm, "Flush work: load %s firmware\n",
  				fw_id_to_name[lbfw->id]);
  			flush_work(&lbfw->work);
+
+			/*
+			 * flush is being called during component unbind, suspend etc.
+			 * So mark status XE_LB_FW_RUNNING -> XE_LB_FW_AVAILABLE
+			 */
+			if (lbfw->__status == XE_LB_FW_RUNNING)
+				xe_late_bind_change_fw_status(lbfw, XE_LB_FW_AVAILABLE);

I am not convinced that this is the correct place for this. We might have cases in the future where we want to do a flush without clearing the state (e.g. before a reset). Also, in the suspend scenario you're already adding xe_late_bind_fw_stop.

Quoted 52 lines
drivers/gpu/drm/xe/xe_late_bind_fw.c +16 -0
  		}
  	}
  }
@@ -282,6 +315,7 @@ static void xe_late_bind_work(struct work_struct *work)
  	if (!ret) {
  		drm_dbg(&xe->drm, "Load %s firmware successful\n",
  			fw_id_to_name[lbfw->id]);
+		xe_late_bind_change_fw_status(lbfw, XE_LB_FW_RUNNING);
  		goto out;
  	}
 
@@ -294,10 +328,23 @@ static void xe_late_bind_work(struct work_struct *work)
  	/* Do not re-attempt fw load */
  	drmm_kfree(&xe->drm, (void *)lbfw->payload);
  	lbfw->payload = NULL;
+	xe_late_bind_change_fw_status(lbfw, XE_LB_FW_FAILED);
  out:
  	xe_pm_runtime_put(xe);
  }
 
+static void  xe_late_bind_fw_stop(struct xe_late_bind *late_bind)
+{
+	struct xe_late_bind_fw *lbfw;
+	int fw_id;
+
+	for (fw_id = 0; fw_id < XE_LB_FW_MAX_ID; fw_id++) {
+		lbfw = &late_bind->late_bind_fw[fw_id];
+		if (lbfw->payload && lbfw->status == XE_LB_FW_RUNNING)
+			xe_late_bind_change_fw_status(lbfw, XE_LB_FW_AVAILABLE);
+	}
+}
+
  static int xe_late_bind_fw_load(struct xe_late_bind *late_bind)
  {
  	struct xe_device *xe = late_bind_to_xe(late_bind);
@@ -331,6 +378,7 @@ static int __xe_late_bind_fw_init(struct xe_late_bind *late_bind, u32 fw_id)
  	lb_fw->id = fw_id;
  	lb_fw->type = fw_id_to_type[lb_fw->id];
  	lb_fw->flags &= ~INTEL_LB_FLAG_IS_PERSISTENT;
+	xe_late_bind_change_fw_status(lb_fw, XE_LB_FW_NOT_SUPPORTED);
 
  	if (lb_fw->type == INTEL_LB_TYPE_FAN_CONTROL) {
  		ret = xe_late_bind_fw_num_fans(late_bind, &num_fans);
@@ -379,6 +427,7 @@ static int __xe_late_bind_fw_init(struct xe_late_bind *late_bind, u32 fw_id)
  	memcpy((void *)lb_fw->payload, fw->data, lb_fw->payload_size);
  	release_firmware(fw);
  	INIT_WORK(&lb_fw->work, xe_late_bind_work);
+	xe_late_bind_change_fw_status(lb_fw, XE_LB_FW_AVAILABLE);
 
  	return 0;
  }

drivers/gpu/drm/xe/xe_late_bind_fw.c static void xe_late_bind_remove(void *arg) · line 501

Quoted 9 lines
drivers/gpu/drm/xe/xe_late_bind_fw.c +5 -1
@@ -445,15 +494,30 @@ static void xe_late_bind_remove(void *arg)
  	}
  }
 
-void xe_late_bind_pm_suspend(struct xe_late_bind *late_bind)
+/**
+ * xe_late_bind_ocode_status() - get ocode late bind firmware status
+ * @xe: pointer to xe device.
+ *
+ * Return: 0 if unsupported, 1 if available, 2 if running, 3 if failed.

This returns a value from xe_late_bind_fw_status, so might be worth just saying that. also, you haven't defined the enum as starting from 0.

Daniele

Quoted 104 lines
drivers/gpu/drm/xe/xe_late_bind_fw.c +12 -1
+ */
+int xe_late_bind_ocode_status(struct xe_device *xe)
+{
+	struct xe_late_bind_fw *lbfw = &xe->late_bind.late_bind_fw[XE_LB_FW_OCODE];
+
+	return lbfw->status;
+}
+
+void xe_late_bind_pm_suspend(struct xe_late_bind *late_bind, bool is_runtime)
  {
  	if (!late_bind->component_added)
  		return;
 
  	if (late_bind->disable)
  		return;
+	if (!is_runtime)
+		xe_late_bind_wait_for_worker_completion(late_bind);
 
-	xe_late_bind_wait_for_worker_completion(late_bind);
+	xe_late_bind_fw_stop(late_bind);
  }
 
  void xe_late_bind_pm_resume(struct xe_late_bind *late_bind)
drivers/gpu/drm/xe/xe_late_bind_fw.h +3 -2
index 8896fea3068e5..b441f5789f24b 100644
@@ -9,9 +9,10 @@
  #include <linux/types.h>
 
  struct xe_late_bind;
+struct xe_device;
 
  int xe_late_bind_init(struct xe_late_bind *late_bind);
-void xe_late_bind_pm_suspend(struct xe_late_bind *late_bind);
+void xe_late_bind_pm_suspend(struct xe_late_bind *late_bind, bool is_runtime);
  void xe_late_bind_pm_resume(struct xe_late_bind *late_bind);
-
+int xe_late_bind_ocode_status(struct xe_device *xe);
  #endif
drivers/gpu/drm/xe/xe_late_bind_fw_types.h +23 -0
index a128e3f638a6c..4cdbd775b103e 100644
@@ -26,6 +26,20 @@ enum xe_late_bind_fw_id {
  	XE_LB_FW_MAX_ID
  };
 
+/**
+ * enum xe_late_bind_fw_status - enum to determine late binding fw status
+ */
+enum xe_late_bind_fw_status {
+	/** @XE_LB_FW_NOT_SUPPORTED: firmware not available/supported */
+	XE_LB_FW_NOT_SUPPORTED,
+	/** @XE_LB_FW_AVAILABLE: firmware found and copied to memory */
+	XE_LB_FW_AVAILABLE,
+	/** @XE_LB_FW_RUNNING: firmware is running */
+	XE_LB_FW_RUNNING,
+	/** @XE_LB_FW_FAILED: firmware load is failed */
+	XE_LB_FW_FAILED,
+};
+
  /**
   * struct xe_late_bind_fw
   */
@@ -46,6 +60,15 @@ struct xe_late_bind_fw {
  	struct work_struct work;
  	/** @version: late binding blob manifest version */
  	struct gsc_version version;
+	union {
+		/** @status: firmware load status */
+		const enum xe_late_bind_fw_status status;
+		/**
+		 * @__status: private firmware load status - only to be used
+		 * by firmware loading code
+		 */
+		enum xe_late_bind_fw_status __status;
+	};
  };
 
  /**
drivers/gpu/drm/xe/xe_pm.c +4 -1
index f85b12c1a85d2..9aa873f57b5ad 100644
@@ -182,7 +182,7 @@ int xe_pm_suspend(struct xe_device *xe)
  	if (err)
  		goto err;
 
-	xe_late_bind_pm_suspend(&xe->late_bind);
+	xe_late_bind_pm_suspend(&xe->late_bind, false);
 
  	for_each_gt(gt, xe, id)
  		xe_gt_suspend_prepare(gt);
@@ -612,6 +612,9 @@ int xe_pm_runtime_suspend(struct xe_device *xe)
  			goto out_resume;
  	}
 
+	if (xe->d3cold.allowed)
+		xe_late_bind_pm_suspend(&xe->late_bind, true);
+
  	xe_irq_suspend(xe);
 
  	xe_display_pm_runtime_suspend_late(xe);
#8 lore raw

Re: [RFC PATCH 3/4] drm/xe/xe_late_bind_fw: Track firmware load status

Quoted 8 lines · Daniele Ceraolo Spurio
Quoted 3 lines · Badal Nilawar

In CRI, Intel PMT depends on the ocode runtime status. Track the late-binding firmware load status and expose an API, for xe_vsec driver, to query it.

I think it would be better to send this patch with the series that adds the xe_vsec changes, otherwise we have no idea from a review POV on how you plan to use the new exposed API.

PMT will be using System Controller mailbox to check ocode status, so I am dropping this patch.

Thanks,
Badal

Quoted 228 lines
Quoted 53 lines

---
  drivers/gpu/drm/xe/xe_late_bind_fw.c       | 68 +++++++++++++++++++++- drivers/gpu/drm/xe/xe_late_bind_fw.h       |  5 +- drivers/gpu/drm/xe/xe_late_bind_fw_types.h | 23 ++++++++ drivers/gpu/drm/xe/xe_pm.c                 |  5 +-

  4 files changed, 96 insertions(+), 5 deletions(-)
drivers/gpu/drm/xe/xe_late_bind_fw.c +25 -0
index d3ef2926517bd..87137d6e4aaec 100644
@@ -53,6 +53,32 @@ late_bind_fw_to_xe(struct xe_late_bind_fw *lb_fw)
       return container_of(lb_fw, struct xe_device, late_bind.late_bind_fw[lb_fw->id]);
   }
   +static const char *xe_late_bind_fw_status_to_str(enum xe_late_bind_fw_status status)
+{
+    switch (status) {
+    case XE_LB_FW_NOT_SUPPORTED:
+        return "Not Supported";
+    case XE_LB_FW_AVAILABLE:
+        return "Available";
+    case XE_LB_FW_RUNNING:
+        return "Running";
+    case XE_LB_FW_FAILED:
+        return "Failed";
+    default:
+        return "Unknown";
+    }
+}
+
+static void xe_late_bind_change_fw_status(struct xe_late_bind_fw 
 *lb_fw,
+                      enum xe_late_bind_fw_status status)
+{
+    struct xe_device *xe = late_bind_fw_to_xe(lb_fw);
+
+    lb_fw->__status = status;
+    drm_dbg(&xe->drm, "%s firmware status changed to %s\n",
+        fw_id_to_name[lb_fw->id], 
 xe_late_bind_fw_status_to_str(status));
+}
+
   /* Refer to the "Late Bind based Firmware Layout" documentation entry for details */

  static int parse_cpd_header(struct xe_late_bind_fw *lb_fw, const void *data, size_t size, const char *manifest_entry)

drivers/gpu/drm/xe/xe_late_bind_fw.c +0 -0
@@ -235,6 +261,13 @@ static void 

xe_late_bind_wait_for_worker_completion(struct xe_late_bind *late_bi drm_dbg(&xe->drm, "Flush work: load %s firmware\n", fw_id_to_name[lbfw->id]); flush_work(&lbfw->work);
+

Quoted 8 lines

+            /*
+             * flush is being called during component unbind, suspend etc.
+             * So mark status XE_LB_FW_RUNNING -> XE_LB_FW_AVAILABLE +             */
+            if (lbfw->__status == XE_LB_FW_RUNNING)
+                xe_late_bind_change_fw_status(lbfw, XE_LB_FW_AVAILABLE);

I am not convinced that this is the correct place for this. We might have cases in the future where we want to do a flush without clearing the state (e.g. before a reset). Also, in the suspend scenario you're already adding xe_late_bind_fw_stop.

Quoted 51 lines

          }
      }
  }

drivers/gpu/drm/xe/xe_late_bind_fw.c +1 -0
@@ -282,6 +315,7 @@ static void xe_late_bind_work(struct work_struct 
 *work)
       if (!ret) {
           drm_dbg(&xe->drm, "Load %s firmware successful\n",
               fw_id_to_name[lbfw->id]);
+        xe_late_bind_change_fw_status(lbfw, XE_LB_FW_RUNNING);
           goto out;
       }

  @@ -294,10 +328,23 @@ static void xe_late_bind_work(struct work_struct *work) /* Do not re-attempt fw load */
      drmm_kfree(&xe->drm, (void *)lbfw->payload);
      lbfw->payload = NULL;
+    xe_late_bind_change_fw_status(lbfw, XE_LB_FW_FAILED);
  out:
      xe_pm_runtime_put(xe);
  }
  +static void  xe_late_bind_fw_stop(struct xe_late_bind *late_bind)

drivers/gpu/drm/xe/xe_late_bind_fw.c +11 -0
+{
+    struct xe_late_bind_fw *lbfw;
+    int fw_id;
+
+    for (fw_id = 0; fw_id < XE_LB_FW_MAX_ID; fw_id++) {
+        lbfw = &late_bind->late_bind_fw[fw_id];
+        if (lbfw->payload && lbfw->status == XE_LB_FW_RUNNING)
+            xe_late_bind_change_fw_status(lbfw, XE_LB_FW_AVAILABLE);
+    }
+}
+

  static int xe_late_bind_fw_load(struct xe_late_bind *late_bind) {
      struct xe_device *xe = late_bind_to_xe(late_bind);

drivers/gpu/drm/xe/xe_late_bind_fw.c +2 -0
@@ -331,6 +378,7 @@ static int __xe_late_bind_fw_init(struct 
 xe_late_bind *late_bind, u32 fw_id)
       lb_fw->id = fw_id;
       lb_fw->type = fw_id_to_type[lb_fw->id];
       lb_fw->flags &= ~INTEL_LB_FLAG_IS_PERSISTENT;
+    xe_late_bind_change_fw_status(lb_fw, XE_LB_FW_NOT_SUPPORTED);
         if (lb_fw->type == INTEL_LB_TYPE_FAN_CONTROL) {
           ret = xe_late_bind_fw_num_fans(late_bind, &num_fans);
@@ -379,6 +427,7 @@ static int __xe_late_bind_fw_init(struct 
 xe_late_bind *late_bind, u32 fw_id)
       memcpy((void *)lb_fw->payload, fw->data, lb_fw->payload_size);
       release_firmware(fw);
       INIT_WORK(&lb_fw->work, xe_late_bind_work);
+    xe_late_bind_change_fw_status(lb_fw, XE_LB_FW_AVAILABLE);
         return 0;
   }

drivers/gpu/drm/xe/xe_late_bind_fw.c static void xe_late_bind_remove(void *arg) · line 501

Quoted 8 lines
drivers/gpu/drm/xe/xe_late_bind_fw.c +5 -0
@@ -445,15 +494,30 @@ static void xe_late_bind_remove(void *arg)
       }
   }
   -void xe_late_bind_pm_suspend(struct xe_late_bind *late_bind)
+/**
+ * xe_late_bind_ocode_status() - get ocode late bind firmware status
+ * @xe: pointer to xe device.
+ *
+ * Return: 0 if unsupported, 1 if available, 2 if running, 3 if failed.

This returns a value from xe_late_bind_fw_status, so might be worth just saying that. also, you haven't defined the enum as starting from 0.

Daniele

Quoted 97 lines
drivers/gpu/drm/xe/xe_late_bind_fw.c +12 -0
+ */
+int xe_late_bind_ocode_status(struct xe_device *xe)
+{
+    struct xe_late_bind_fw *lbfw = 
 &xe->late_bind.late_bind_fw[XE_LB_FW_OCODE];
+
+    return lbfw->status;
+}
+
+void xe_late_bind_pm_suspend(struct xe_late_bind *late_bind, bool 
 is_runtime)
   {
       if (!late_bind->component_added)
           return;
         if (late_bind->disable)
           return;
+    if (!is_runtime)
+        xe_late_bind_wait_for_worker_completion(late_bind);
   -    xe_late_bind_wait_for_worker_completion(late_bind);
+    xe_late_bind_fw_stop(late_bind);
   }
     void xe_late_bind_pm_resume(struct xe_late_bind *late_bind)
drivers/gpu/drm/xe/xe_late_bind_fw.h +3 -2
index 8896fea3068e5..b441f5789f24b 100644
@@ -9,9 +9,10 @@
   #include <linux/types.h>
     struct xe_late_bind;
+struct xe_device;
     int xe_late_bind_init(struct xe_late_bind *late_bind);
-void xe_late_bind_pm_suspend(struct xe_late_bind *late_bind);
+void xe_late_bind_pm_suspend(struct xe_late_bind *late_bind, bool 
 is_runtime);
   void xe_late_bind_pm_resume(struct xe_late_bind *late_bind);
-
+int xe_late_bind_ocode_status(struct xe_device *xe);
   #endif
drivers/gpu/drm/xe/xe_late_bind_fw_types.h +0 -0
index a128e3f638a6c..4cdbd775b103e 100644
@@ -26,6 +26,20 @@ enum xe_late_bind_fw_id {

      XE_LB_FW_MAX_ID
  };
  +/**
+ * enum xe_late_bind_fw_status - enum to determine late binding fw status
+ */

drivers/gpu/drm/xe/xe_late_bind_fw_types.h +11 -0
+enum xe_late_bind_fw_status {
+    /** @XE_LB_FW_NOT_SUPPORTED: firmware not available/supported */
+    XE_LB_FW_NOT_SUPPORTED,
+    /** @XE_LB_FW_AVAILABLE: firmware found and copied to memory */
+    XE_LB_FW_AVAILABLE,
+    /** @XE_LB_FW_RUNNING: firmware is running */
+    XE_LB_FW_RUNNING,
+    /** @XE_LB_FW_FAILED: firmware load is failed */
+    XE_LB_FW_FAILED,
+};
+

  /**
   * struct xe_late_bind_fw
   */

drivers/gpu/drm/xe/xe_late_bind_fw_types.h +9 -0
@@ -46,6 +60,15 @@ struct xe_late_bind_fw {
       struct work_struct work;
       /** @version: late binding blob manifest version */
       struct gsc_version version;
+    union {
+        /** @status: firmware load status */
+        const enum xe_late_bind_fw_status status;
+        /**
+         * @__status: private firmware load status - only to be used
+         * by firmware loading code
+         */
+        enum xe_late_bind_fw_status __status;
+    };
   };
     /**
drivers/gpu/drm/xe/xe_pm.c +3 -0
index f85b12c1a85d2..9aa873f57b5ad 100644
@@ -182,7 +182,7 @@ int xe_pm_suspend(struct xe_device *xe)
       if (err)
           goto err;
   -    xe_late_bind_pm_suspend(&xe->late_bind);
+    xe_late_bind_pm_suspend(&xe->late_bind, false);
         for_each_gt(gt, xe, id)
           xe_gt_suspend_prepare(gt);
@@ -612,6 +612,9 @@ int xe_pm_runtime_suspend(struct xe_device *xe)
               goto out_resume;
       }
   +    if (xe->d3cold.allowed)
+        xe_late_bind_pm_suspend(&xe->late_bind, true);
+
       xe_irq_suspend(xe);
         xe_display_pm_runtime_suspend_late(xe);
#9 lore raw

RFC PATCH 4/4 drm/xe/xe_late_bind_fw: Enable late binding support for CRI

In reply to Badal Nilawar

Enable late binding support for CRI

---
 drivers/gpu/drm/xe/xe_pci.c | 1 +
 1 file changed, 1 insertion(+)
drivers/gpu/drm/xe/xe_pci.c +1 -0
index 29f976e66848c..7564eebd6a654 100644
@@ -461,6 +461,7 @@ static const struct xe_device_desc cri_desc = {
 	.has_mbx_power_limits = true,
 	.has_mbx_thermal_info = true,
 	.has_mert = true,
+	.has_late_bind = true,
 	.has_pre_prod_wa = 1,
 	.has_soc_remapper_sysctrl = true,
 	.has_soc_remapper_telem = true,
Signature · 1 line
2.52.0
#10 lore raw

✗ CI.KUnit: failure for Add ocode late binding support for CRI

In reply to Badal Nilawar
== Series Details ==

== Series Details ==

Series: Add ocode late binding support for CRI
URL : https://patchwork.freedesktop.org/series/162657/
State : failure

== Summary ==

+ trap cleanup EXIT
+ /kernel/tools/testing/kunit/kunit.py run --kunitconfig /kernel/drivers/gpu/drm/xe/.kunitconfig ERROR:root:../drivers/gpu/drm/xe/xe_late_bind_fw.c:36:36: error: ‘INTEL_LB_TYPE_OCODE’ undeclared here (not in a function)
36 | [XE_LB_FW_OCODE] = INTEL_LB_TYPE_OCODE,

      |                                    ^~~~~~~~~~~~~~~~~~~
../drivers/gpu/drm/xe/xe_late_bind_fw.c: In function ‘xe_late_bind_parse_status’:
../drivers/gpu/drm/xe/xe_late_bind_fw.c:210:14: error: ‘INTEL_LB_STATUS_INTERNAL_ERROR’ undeclared (first use in this function); did you mean ‘INTEL_LB_STATUS_GENERAL_ERROR’?
  210 |         case INTEL_LB_STATUS_INTERNAL_ERROR:
      |              ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
      |              INTEL_LB_STATUS_GENERAL_ERROR
../drivers/gpu/drm/xe/xe_late_bind_fw.c:210:14: note: each undeclared identifier is reported only once for each function it appears in
../drivers/gpu/drm/xe/xe_late_bind_fw.c:212:14: error: ‘INTEL_LB_STATUS_INVALID_FPT_TABLE’ undeclared (first use in this function); did you mean ‘INTEL_LB_STATUS_INVALID_SIGNATURE’?
  212 |         case INTEL_LB_STATUS_INVALID_FPT_TABLE:
      |              ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
      |              INTEL_LB_STATUS_INVALID_SIGNATURE
../drivers/gpu/drm/xe/xe_late_bind_fw.c:214:14: error: ‘INTEL_LB_STATUS_SIGNED_PAYLOAD_VERIFICATION_ERROR’ undeclared (first use in this function)
  214 |         case INTEL_LB_STATUS_SIGNED_PAYLOAD_VERIFICATION_ERROR:
      |              ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
../drivers/gpu/drm/xe/xe_late_bind_fw.c:216:14: error: ‘INTEL_LB_STATUS_SIGNED_PAYLOAD_INVALID_CPD’ undeclared (first use in this function)
  216 |         case INTEL_LB_STATUS_SIGNED_PAYLOAD_INVALID_CPD:
      |              ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
../drivers/gpu/drm/xe/xe_late_bind_fw.c:218:14: error: ‘INTEL_LB_STATUS_SIGNED_PAYLOAD_FW_VERSION_MISMATCH’ undeclared (first use in this function)
  218 |         case INTEL_LB_STATUS_SIGNED_PAYLOAD_FW_VERSION_MISMATCH:
      |              ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
../drivers/gpu/drm/xe/xe_late_bind_fw.c:220:14: error: ‘INTEL_LB_STATUS_SIGNED_PAYLOAD_INVALID_MANIFEST’ undeclared (first use in this function)
  220 |         case INTEL_LB_STATUS_SIGNED_PAYLOAD_INVALID_MANIFEST:
      |              ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
../drivers/gpu/drm/xe/xe_late_bind_fw.c:222:14: error: ‘INTEL_LB_STATUS_SIGNED_PAYLOAD_INVALID_HASH’ undeclared (first use in this function)
  222 |         case INTEL_LB_STATUS_SIGNED_PAYLOAD_INVALID_HASH:
      |              ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
../drivers/gpu/drm/xe/xe_late_bind_fw.c:224:14: error: ‘INTEL_LB_STATUS_SIGNED_PAYLOAD_BINDING_TYPE_MISMATCH’ undeclared (first use in this function)
  224 |         case INTEL_LB_STATUS_SIGNED_PAYLOAD_BINDING_TYPE_MISMATCH:
      |              ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
../drivers/gpu/drm/xe/xe_late_bind_fw.c:226:14: error: ‘INTEL_LB_STATUS_SIGNED_PAYLOAD_HANDLE_SVN_FAILED’ undeclared (first use in this function)
  226 |         case INTEL_LB_STATUS_SIGNED_PAYLOAD_HANDLE_SVN_FAILED:
      |              ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
../drivers/gpu/drm/xe/xe_late_bind_fw.c:228:14: error: ‘INTEL_LB_STATUS_DESTINATION_MBOX_FAILURE’ undeclared (first use in this function); did you mean ‘INTEL_LB_STATUS_ARB_FAILURE’?
  228 |         case INTEL_LB_STATUS_DESTINATION_MBOX_FAILURE:
      |              ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
      |              INTEL_LB_STATUS_ARB_FAILURE
../drivers/gpu/drm/xe/xe_late_bind_fw.c:230:14: error: ‘INTEL_LB_STATUS_MISSING_LOADING_PATCH’ undeclared (first use in this function)
  230 |         case INTEL_LB_STATUS_MISSING_LOADING_PATCH:
      |              ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
../drivers/gpu/drm/xe/xe_late_bind_fw.c:232:14: error: ‘INTEL_LB_STATUS_INVALID_COMMAND’ undeclared (first use in this function); did you mean ‘INTEL_LB_STATUS_INVALID_PAYLOAD’?
  232 |         case INTEL_LB_STATUS_INVALID_COMMAND:
      |              ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
      |              INTEL_LB_STATUS_INVALID_PAYLOAD
../drivers/gpu/drm/xe/xe_late_bind_fw.c:234:14: error: ‘INTEL_LB_STATUS_INVALID_HECI_HEADER’ undeclared (first use in this function); did you mean ‘INTEL_LB_STATUS_INVALID_PAYLOAD’?
  234 |         case INTEL_LB_STATUS_INVALID_HECI_HEADER:
      |              ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
      |              INTEL_LB_STATUS_INVALID_PAYLOAD
../drivers/gpu/drm/xe/xe_late_bind_fw.c:236:14: error: ‘INTEL_LB_STATUS_IP_ERROR_START’ undeclared (first use in this function)
  236 |         case INTEL_LB_STATUS_IP_ERROR_START:
      |              ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

make[7]: *** [../scripts/Makefile.build:289: drivers/gpu/drm/xe/xe_late_bind_fw.o] Error 1 make[7]: *** Waiting for unfinished jobs....
make[6]: *** [../scripts/Makefile.build:546: drivers/gpu/drm/xe] Error 2 make[5]: *** [../scripts/Makefile.build:546: drivers/gpu/drm] Error 2 make[4]: *** [../scripts/Makefile.build:546: drivers/gpu] Error 2
make[3]: *** [../scripts/Makefile.build:546: drivers] Error 2
make[2]: *** [/kernel/Makefile:2101: .] Error 2
make[1]: *** [/kernel/Makefile:248: __sub-make] Error 2
make: *** [Makefile:248: __sub-make] Error 2

[11:26:10] Configuring KUnit Kernel ...
Generating .config ...
Populating config with:

$ make ARCH=um O=.kunit olddefconfig
[11:26:15] Building KUnit Kernel ...
Populating config with:
$ make ARCH=um O=.kunit olddefconfig
Building with:
$ make all compile_commands.json scripts_gdb ARCH=um O=.kunit --jobs=48
+ cleanup
++ stat -c %u:%g /kernel
+ chown -R 1003:1003 /kernel