Skip to contentSkip to conversation

habanalabs: implement dma-fence mechanism

PATCH In review Oded Gabbay 6 participants 11 messages Jul 13, 2020 – Jul 15, 2020 Char and misc drivers

lore
Share & tools
lore
#0 lore raw

PATCH 1/3 habanalabs: implement dma-fence mechanism

From: Ofir Bitton <[email protected]>

Instead of using standard dma-fence mechanism designed for GPU's, we introduce our own implementation based on the former one. This implementation is much more sparse than the original, contains only mandatory functionality required by the driver.

---
 drivers/misc/habanalabs/Makefile             |   2 +-
 drivers/misc/habanalabs/command_submission.c | 120 ++-----
 drivers/misc/habanalabs/context.c            |  10 +-
 drivers/misc/habanalabs/habanalabs.h         |  12 +-
 drivers/misc/habanalabs/hl_dma_fence.c       | 338 +++++++++++++++++++
 drivers/misc/habanalabs/hl_dma_fence.h       | 148 ++++++++
 drivers/misc/habanalabs/hw_queue.c           |   2 +-
 7 files changed, 519 insertions(+), 113 deletions(-)
 create mode 100644 drivers/misc/habanalabs/hl_dma_fence.c
 create mode 100644 drivers/misc/habanalabs/hl_dma_fence.h
drivers/misc/habanalabs/Makefile +1 -1
index 421ebd903069..83b0e9ac35d6 100644
@@ -7,7 +7,7 @@ obj-m	:= habanalabs.o
 
 habanalabs-y := habanalabs_drv.o device.o context.o asid.o habanalabs_ioctl.o \
 		command_buffer.o hw_queue.o irq.o sysfs.o hwmon.o memory.o \
-		command_submission.o mmu.o firmware_if.o pci.o
+		command_submission.o mmu.o firmware_if.o pci.o hl_dma_fence.o
 
 habanalabs-$(CONFIG_DEBUG_FS) += debugfs.o
 
drivers/misc/habanalabs/command_submission.c +20 -100

211 lines not on this page · raw message

drivers/misc/habanalabs/context.c +5 -5
index 1e3e5b19ecd9..fd1d4ea2854d 100644
@@ -23,7 +23,7 @@ static void hl_ctx_fini(struct hl_ctx *ctx)
 	 */
 
 	for (i = 0 ; i < hdev->asic_prop.max_pending_cs ; i++)
-		dma_fence_put(ctx->cs_pending[i]);
+		hl_dma_fence_put(ctx->cs_pending[i]);
 
 	kfree(ctx->cs_pending);
 
@@ -128,7 +128,7 @@ int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx)
 	atomic_set(&ctx->thread_ctx_switch_token, 1);
 	ctx->thread_ctx_switch_wait_token = 0;
 	ctx->cs_pending = kcalloc(hdev->asic_prop.max_pending_cs,
-				sizeof(struct dma_fence *),
+				sizeof(struct hl_dma_fence *),
 				GFP_KERNEL);
 	if (!ctx->cs_pending)
 		return -ENOMEM;
@@ -174,10 +174,10 @@ int hl_ctx_put(struct hl_ctx *ctx)
 	return kref_put(&ctx->refcount, hl_ctx_do_release);
 }
 
-struct dma_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq)
+struct hl_dma_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq)
 {
 	struct asic_fixed_properties *asic_prop = &ctx->hdev->asic_prop;
-	struct dma_fence *fence;
+	struct hl_dma_fence *fence;
 
 	spin_lock(&ctx->cs_lock);
 
@@ -191,7 +191,7 @@ struct dma_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq)
 		return NULL;
 	}
 
-	fence = dma_fence_get(
+	fence = hl_dma_fence_get(
 			ctx->cs_pending[seq & (asic_prop->max_pending_cs - 1)]);
 	spin_unlock(&ctx->cs_lock);
 
drivers/misc/habanalabs/habanalabs.h +6 -6
index 14def0d26d2d..3f9270ace869 100644
@@ -15,10 +15,10 @@
 #include <linux/cdev.h>
 #include <linux/iopoll.h>
 #include <linux/irqreturn.h>
-#include <linux/dma-fence.h>
 #include <linux/dma-direction.h>
 #include <linux/scatterlist.h>
 #include <linux/hashtable.h>
+#include "hl_dma_fence.h"
 
 #define HL_NAME				"habanalabs"
 
@@ -353,7 +353,7 @@ struct asic_fixed_properties {
  * @sob_val: the SOB value that is used in this signal/wait CS.
  */
 struct hl_cs_compl {
-	struct dma_fence	base_fence;
+	struct hl_dma_fence	base_fence;
 	spinlock_t		lock;
 	struct hl_device	*hdev;
 	struct hl_hw_sob	*hw_sob;
@@ -823,7 +823,7 @@ struct hl_ctx {
 	struct hl_fpriv		*hpriv;
 	struct hl_device	*hdev;
 	struct kref		refcount;
-	struct dma_fence	**cs_pending;
+	struct hl_dma_fence	**cs_pending;
 	struct hl_va_range	*host_va_range;
 	struct hl_va_range	*host_huge_va_range;
 	struct hl_va_range	*dram_va_range;
@@ -910,8 +910,8 @@ struct hl_cs {
 	struct list_head	job_list;
 	spinlock_t		job_lock;
 	struct kref		refcount;
-	struct dma_fence	*fence;
-	struct dma_fence	*signal_fence;
+	struct hl_dma_fence	*fence;
+	struct hl_dma_fence	*signal_fence;
 	struct work_struct	finish_work;
 	struct delayed_work	work_tdr;
 	struct list_head	mirror_node;
@@ -1716,7 +1716,7 @@ int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx);
 void hl_ctx_do_release(struct kref *ref);
 void hl_ctx_get(struct hl_device *hdev,	struct hl_ctx *ctx);
 int hl_ctx_put(struct hl_ctx *ctx);
-struct dma_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq);
+struct hl_dma_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq);
 void hl_ctx_mgr_init(struct hl_ctx_mgr *mgr);
 void hl_ctx_mgr_fini(struct hl_device *hdev, struct hl_ctx_mgr *mgr);
 
drivers/misc/habanalabs/hl_dma_fence.c new file +338 -0

338 lines not on this page · raw message

drivers/misc/habanalabs/hl_dma_fence.h new file +148 -0

148 lines not on this page · raw message

drivers/misc/habanalabs/hw_queue.c +1 -1
index 287681646071..c32796c761ba 100644
@@ -474,7 +474,7 @@ static void init_signal_wait_cs(struct hl_cs *cs)
 		 * wait CS was submitted.
 		 */
 		mb();
-		dma_fence_put(cs->signal_fence);
+		hl_dma_fence_put(cs->signal_fence);
 		cs->signal_fence = NULL;
 	}
 }
Signature · 1 line
2.17.1
#1 lore raw

PATCH 2/3 habanalabs: create common folder

For internal needs of our CI we need to move all the common code into a common folder instead of putting them in the root folder of the driver.

Same applies to the common header files under include/

---
 drivers/misc/habanalabs/Makefile                      | 11 +++++------
 drivers/misc/habanalabs/common/Makefile               |  9 +++++++++
 drivers/misc/habanalabs/{ => common}/asid.c           |  0
 drivers/misc/habanalabs/{ => common}/command_buffer.c |  0
 .../misc/habanalabs/{ => common}/command_submission.c |  0
 drivers/misc/habanalabs/{ => common}/context.c        |  0
 drivers/misc/habanalabs/{ => common}/debugfs.c        |  0
 drivers/misc/habanalabs/{ => common}/device.c         |  0
 drivers/misc/habanalabs/{ => common}/firmware_if.c    |  2 +-
 drivers/misc/habanalabs/{ => common}/habanalabs.h     |  4 ++--
 drivers/misc/habanalabs/{ => common}/habanalabs_drv.c |  0
 .../misc/habanalabs/{ => common}/habanalabs_ioctl.c   |  0
 drivers/misc/habanalabs/{ => common}/hl_dma_fence.c   |  0
 drivers/misc/habanalabs/{ => common}/hl_dma_fence.h   |  0
 drivers/misc/habanalabs/{ => common}/hw_queue.c       |  0
 drivers/misc/habanalabs/{ => common}/hwmon.c          |  0
 drivers/misc/habanalabs/{ => common}/irq.c            |  0
 drivers/misc/habanalabs/{ => common}/memory.c         |  0
 drivers/misc/habanalabs/{ => common}/mmu.c            |  0
 drivers/misc/habanalabs/{ => common}/pci.c            |  0
 drivers/misc/habanalabs/{ => common}/sysfs.c          |  0
 drivers/misc/habanalabs/gaudi/Makefile                |  2 +-
 drivers/misc/habanalabs/gaudi/gaudiP.h                |  2 +-
 drivers/misc/habanalabs/goya/goyaP.h                  |  2 +-
 .../misc/habanalabs/include/{ => common}/armcp_if.h   |  0
 .../misc/habanalabs/include/{ => common}/hl_boot_if.h |  0
 .../misc/habanalabs/include/{ => common}/qman_if.h    |  0
 27 files changed, 20 insertions(+), 12 deletions(-)
 create mode 100644 drivers/misc/habanalabs/common/Makefile
 rename drivers/misc/habanalabs/{ => common}/asid.c (100%)
 rename drivers/misc/habanalabs/{ => common}/command_buffer.c (100%)
 rename drivers/misc/habanalabs/{ => common}/command_submission.c (100%)
 rename drivers/misc/habanalabs/{ => common}/context.c (100%)
 rename drivers/misc/habanalabs/{ => common}/debugfs.c (100%)
 rename drivers/misc/habanalabs/{ => common}/device.c (100%)
 rename drivers/misc/habanalabs/{ => common}/firmware_if.c (99%)
 rename drivers/misc/habanalabs/{ => common}/habanalabs.h (99%)
 rename drivers/misc/habanalabs/{ => common}/habanalabs_drv.c (100%)
 rename drivers/misc/habanalabs/{ => common}/habanalabs_ioctl.c (100%)
 rename drivers/misc/habanalabs/{ => common}/hl_dma_fence.c (100%)
 rename drivers/misc/habanalabs/{ => common}/hl_dma_fence.h (100%)
 rename drivers/misc/habanalabs/{ => common}/hw_queue.c (100%)
 rename drivers/misc/habanalabs/{ => common}/hwmon.c (100%)
 rename drivers/misc/habanalabs/{ => common}/irq.c (100%)
 rename drivers/misc/habanalabs/{ => common}/memory.c (100%)
 rename drivers/misc/habanalabs/{ => common}/mmu.c (100%)
 rename drivers/misc/habanalabs/{ => common}/pci.c (100%)
 rename drivers/misc/habanalabs/{ => common}/sysfs.c (100%)
 rename drivers/misc/habanalabs/include/{ => common}/armcp_if.h (100%)
 rename drivers/misc/habanalabs/include/{ => common}/hl_boot_if.h (100%)
 rename drivers/misc/habanalabs/include/{ => common}/qman_if.h (100%)
drivers/misc/habanalabs/Makefile +5 -6
index 83b0e9ac35d6..a786c0a7de9a 100644
@@ -3,16 +3,15 @@
 # Makefile for HabanaLabs AI accelerators driver
 #
 
-obj-m	:= habanalabs.o
+obj-$(CONFIG_HABANA_AI) := habanalabs.o
 
-habanalabs-y := habanalabs_drv.o device.o context.o asid.o habanalabs_ioctl.o \
-		command_buffer.o hw_queue.o irq.o sysfs.o hwmon.o memory.o \
-		command_submission.o mmu.o firmware_if.o pci.o hl_dma_fence.o
-
-habanalabs-$(CONFIG_DEBUG_FS) += debugfs.o
+include $(src)/common/Makefile
+habanalabs-y += $(HL_COMMON_FILES)
 
 include $(src)/goya/Makefile
 habanalabs-y += $(HL_GOYA_FILES)
 
 include $(src)/gaudi/Makefile
 habanalabs-y += $(HL_GAUDI_FILES)
+
+habanalabs-$(CONFIG_DEBUG_FS) += common/debugfs.o
drivers/misc/habanalabs/common/Makefile new file +9 -0
new file mode 100644
index 000000000000..1e149f5852c6
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+subdir-ccflags-y += -I$(src)/common
+
+HL_COMMON_FILES := common/habanalabs_drv.o common/device.o common/context.o \
+		common/asid.o common/habanalabs_ioctl.o \
+		common/command_buffer.o common/hw_queue.o common/irq.o \
+		common/sysfs.o common/hwmon.o common/memory.o \
+		common/command_submission.o common/mmu.o common/firmware_if.o \
+		common/pci.o common/hl_dma_fence.o
drivers/misc/habanalabs/common/asid.c renamed from drivers/misc/habanalabs/asid.c +0 -0
similarity index 100%
rename from drivers/misc/habanalabs/asid.c
rename to drivers/misc/habanalabs/common/asid.c
drivers/misc/habanalabs/common/command_buffer.c renamed from drivers/misc/habanalabs/command_buffer.c +0 -0
similarity index 100%
rename from drivers/misc/habanalabs/command_buffer.c
rename to drivers/misc/habanalabs/common/command_buffer.c
drivers/misc/habanalabs/common/command_submission.c renamed from drivers/misc/habanalabs/command_submission.c +0 -0
similarity index 100%
rename from drivers/misc/habanalabs/command_submission.c
rename to drivers/misc/habanalabs/common/command_submission.c
drivers/misc/habanalabs/common/context.c renamed from drivers/misc/habanalabs/context.c +0 -0
similarity index 100%
rename from drivers/misc/habanalabs/context.c
rename to drivers/misc/habanalabs/common/context.c
drivers/misc/habanalabs/common/debugfs.c renamed from drivers/misc/habanalabs/debugfs.c +0 -0
similarity index 100%
rename from drivers/misc/habanalabs/debugfs.c
rename to drivers/misc/habanalabs/common/debugfs.c
drivers/misc/habanalabs/common/device.c renamed from drivers/misc/habanalabs/device.c +0 -0
similarity index 100%
rename from drivers/misc/habanalabs/device.c
rename to drivers/misc/habanalabs/common/device.c
drivers/misc/habanalabs/common/firmware_if.c renamed from drivers/misc/habanalabs/firmware_if.c +1 -1
similarity index 99%
rename from drivers/misc/habanalabs/firmware_if.c
rename to drivers/misc/habanalabs/common/firmware_if.c
index 3be1549cd137..b2b84510b932 100644
@@ -6,7 +6,7 @@
  */
 
 #include "habanalabs.h"
-#include "include/hl_boot_if.h"
+#include "include/common/hl_boot_if.h"
 
 #include <linux/firmware.h>
 #include <linux/genalloc.h>
drivers/misc/habanalabs/common/habanalabs.h renamed from drivers/misc/habanalabs/habanalabs.h +2 -2
similarity index 99%
rename from drivers/misc/habanalabs/habanalabs.h
rename to drivers/misc/habanalabs/common/habanalabs.h
index 3f9270ace869..c362df319d17 100644
@@ -8,8 +8,8 @@
 #ifndef HABANALABSP_H_
 #define HABANALABSP_H_
 
-#include "include/armcp_if.h"
-#include "include/qman_if.h"
+#include "include/common/armcp_if.h"
+#include "include/common/qman_if.h"
 #include <uapi/misc/habanalabs.h>
 
 #include <linux/cdev.h>
drivers/misc/habanalabs/common/habanalabs_drv.c renamed from drivers/misc/habanalabs/habanalabs_drv.c +0 -0
similarity index 100%
rename from drivers/misc/habanalabs/habanalabs_drv.c
rename to drivers/misc/habanalabs/common/habanalabs_drv.c
drivers/misc/habanalabs/common/habanalabs_ioctl.c renamed from drivers/misc/habanalabs/habanalabs_ioctl.c +0 -0
similarity index 100%
rename from drivers/misc/habanalabs/habanalabs_ioctl.c
rename to drivers/misc/habanalabs/common/habanalabs_ioctl.c
drivers/misc/habanalabs/common/hl_dma_fence.c renamed from drivers/misc/habanalabs/hl_dma_fence.c +0 -0
similarity index 100%
rename from drivers/misc/habanalabs/hl_dma_fence.c
rename to drivers/misc/habanalabs/common/hl_dma_fence.c
drivers/misc/habanalabs/common/hl_dma_fence.h renamed from drivers/misc/habanalabs/hl_dma_fence.h +0 -0
similarity index 100%
rename from drivers/misc/habanalabs/hl_dma_fence.h
rename to drivers/misc/habanalabs/common/hl_dma_fence.h
drivers/misc/habanalabs/common/hw_queue.c renamed from drivers/misc/habanalabs/hw_queue.c +0 -0
similarity index 100%
rename from drivers/misc/habanalabs/hw_queue.c
rename to drivers/misc/habanalabs/common/hw_queue.c
drivers/misc/habanalabs/common/hwmon.c renamed from drivers/misc/habanalabs/hwmon.c +0 -0
similarity index 100%
rename from drivers/misc/habanalabs/hwmon.c
rename to drivers/misc/habanalabs/common/hwmon.c
drivers/misc/habanalabs/common/irq.c renamed from drivers/misc/habanalabs/irq.c +0 -0
similarity index 100%
rename from drivers/misc/habanalabs/irq.c
rename to drivers/misc/habanalabs/common/irq.c
drivers/misc/habanalabs/common/memory.c renamed from drivers/misc/habanalabs/memory.c +0 -0
similarity index 100%
rename from drivers/misc/habanalabs/memory.c
rename to drivers/misc/habanalabs/common/memory.c
drivers/misc/habanalabs/common/mmu.c renamed from drivers/misc/habanalabs/mmu.c +0 -0
similarity index 100%
rename from drivers/misc/habanalabs/mmu.c
rename to drivers/misc/habanalabs/common/mmu.c
drivers/misc/habanalabs/common/pci.c renamed from drivers/misc/habanalabs/pci.c +0 -0
similarity index 100%
rename from drivers/misc/habanalabs/pci.c
rename to drivers/misc/habanalabs/common/pci.c
drivers/misc/habanalabs/common/sysfs.c renamed from drivers/misc/habanalabs/sysfs.c +0 -0
similarity index 100%
rename from drivers/misc/habanalabs/sysfs.c
rename to drivers/misc/habanalabs/common/sysfs.c
drivers/misc/habanalabs/gaudi/Makefile +1 -1
index f802cdc980ca..75104ae74e2b 100644
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0-only
-subdir-ccflags-y += -I$(src)
+subdir-ccflags-y += -I$(src)/common
 
 HL_GAUDI_FILES := gaudi/gaudi.o gaudi/gaudi_hwmgr.o gaudi/gaudi_security.o \
 	gaudi/gaudi_coresight.o
drivers/misc/habanalabs/gaudi/gaudiP.h +1 -1
index bdc5f96085a7..a94ab6a180f0 100644
@@ -10,7 +10,7 @@
 
 #include <uapi/misc/habanalabs.h>
 #include "habanalabs.h"
-#include "include/hl_boot_if.h"
+#include "include/common/hl_boot_if.h"
 #include "include/gaudi/gaudi_packets.h"
 #include "include/gaudi/gaudi.h"
 #include "include/gaudi/gaudi_async_events.h"
drivers/misc/habanalabs/goya/goyaP.h +1 -1
index 8265cc21b45a..9e674cf39fd9 100644
@@ -10,7 +10,7 @@
 
 #include <uapi/misc/habanalabs.h>
 #include "habanalabs.h"
-#include "include/hl_boot_if.h"
+#include "include/common/hl_boot_if.h"
 #include "include/goya/goya_packets.h"
 #include "include/goya/goya.h"
 #include "include/goya/goya_async_events.h"
drivers/misc/habanalabs/include/common/armcp_if.h renamed from drivers/misc/habanalabs/include/armcp_if.h +0 -0
similarity index 100%
rename from drivers/misc/habanalabs/include/armcp_if.h
rename to drivers/misc/habanalabs/include/common/armcp_if.h
drivers/misc/habanalabs/include/common/hl_boot_if.h renamed from drivers/misc/habanalabs/include/hl_boot_if.h +0 -0
similarity index 100%
rename from drivers/misc/habanalabs/include/hl_boot_if.h
rename to drivers/misc/habanalabs/include/common/hl_boot_if.h
drivers/misc/habanalabs/include/common/qman_if.h renamed from drivers/misc/habanalabs/include/qman_if.h +0 -0
similarity index 100%
rename from drivers/misc/habanalabs/include/qman_if.h
rename to drivers/misc/habanalabs/include/common/qman_if.h
Signature · 1 line
2.17.1
#3 lore raw

PATCH 3/3 habanalabs: update hl_boot_if.h from firmware

In reply to Oded Gabbay

Update the boot interface file from the latest version from firmware. Defines for secure boot were added.

---
 .../misc/habanalabs/include/common/hl_boot_if.h    | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
drivers/misc/habanalabs/include/common/hl_boot_if.h +14 -0
index c22d134e73af..bb67cafc6e00 100644
@@ -44,6 +44,15 @@
  *					The NIC FW loading and initialization
  *					failed. This means NICs are not usable.
  *
+ * CPU_BOOT_ERR0_SECURITY_NOT_RDY	Chip security initialization has been
+ *					started, but is not ready yet - chip
+ *					cannot be accessed.
+ *
+ * CPU_BOOT_ERR0_SECURITY_FAIL		Security related tasks have failed.
+ *					The tasks are security init (root of
+ *					trust), boot authentication (chain of
+ *					trust), data packets authentication.
+ *
  * CPU_BOOT_ERR0_ENABLED		Error registers enabled.
  *					This is a main indication that the
  *					running FW populates the error
@@ -57,6 +66,8 @@
 #define CPU_BOOT_ERR0_BMC_WAIT_SKIPPED		(1 << 4)
 #define CPU_BOOT_ERR0_NIC_DATA_NOT_RDY		(1 << 5)
 #define CPU_BOOT_ERR0_NIC_FW_FAIL		(1 << 6)
+#define CPU_BOOT_ERR0_SECURITY_NOT_RDY		(1 << 7)
+#define CPU_BOOT_ERR0_SECURITY_FAIL		(1 << 8)
 #define CPU_BOOT_ERR0_ENABLED			(1 << 31)
 
 enum cpu_boot_status {
@@ -79,7 +90,10 @@ enum cpu_boot_status {
 	CPU_BOOT_STATUS_BMC_WAITING_SKIPPED, /* deprecated - will be removed */
 	/* Last boot loader progress status, ready to receive commands */
 	CPU_BOOT_STATUS_READY_TO_BOOT = 15,
+	/* Internal Boot finished, ready for boot-fit */
 	CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT = 16,
+	/* Internal Security has been initialized, device can be accessed */
+	CPU_BOOT_STATUS_SECURITY_READY = 17,
 };
 
 enum kmd_msg {
Signature · 1 line
2.17.1
#5 lore raw
In reply to Oded Gabbay
Quoted 6 lines · Oded Gabbay

From: Ofir Bitton <[email protected]>

Instead of using standard dma-fence mechanism designed for GPU's, we introduce our own implementation based on the former one. This implementation is much more sparse than the original, contains only mandatory functionality required by the driver.

Sad you can't use the in-kernel code for this, I really don't understand what's wrong with using it as-is.

Daniel, why do we need/want duplicate code floating around in the tree like this?

Copying code leads to errors, here's some documentation ones:

drivers/misc/habanalabs/hl_dma_fence.c line 3

Quoted 6 lines
drivers/misc/habanalabs/hl_dma_fence.c new file +3 -0
@@ -0,0 +1,338 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Fence mechanism for dma-buf and to allow for asynchronous dma access

Is that what this still does?

drivers/misc/habanalabs/hl_dma_fence.c line 12

Quoted 9 lines
drivers/misc/habanalabs/hl_dma_fence.c new file +9 -0
+ *
+ * Copyright (C) 2012 Canonical Ltd
+ * Copyright (C) 2012 Texas Instruments
+ *
+ * Authors:
+ * Rob Clark <[email protected]>
+ * Maarten Lankhorst <[email protected]>
+ *
+ * The dma_fence module is a copy of dma-fence at drivers/dma-buf.

"The hl_dma_fence" module...

And is it a stand-alone module? Or just a single file?

drivers/misc/habanalabs/hl_dma_fence.c line 14

Quoted 2 lines
drivers/misc/habanalabs/hl_dma_fence.c new file +2 -0
+ * This was done due to an explicit request by GPU developers who asked not
+ * to use the dma-buf module because we aren't part of DRM subsystem.

Why is dma-buf only for use for DRM?

If it is, should the symbol namespace be set to that to catch users that want to use it for their own code?

Quoted 9 lines
drivers/misc/habanalabs/hl_dma_fence.c new file +9 -0
+ * This copy was stripped from all extra features that habanalabs driver
+ * doesn't use, including the uapi interface dma-buf exposes.
+ * In addition, we removed the callbacks because the only usage is from inside
+ * habanalabs driver
+ */
+
+#include "hl_dma_fence.h"
+#include "habanalabs.h"
+#include <linux/slab.h>

drivers/misc/habanalabs/hl_dma_fence.c line 33

Quoted 10 lines
drivers/misc/habanalabs/hl_dma_fence.c new file +10 -0
+#include <linux/export.h>
+#include <linux/atomic.h>
+#include <linux/sched/signal.h>
+
+/**
+ * DOC: DMA fences overview
+ *
+ * DMA fences, represented by &struct hl_dma_fence, are the kernel internal
+ * synchronization primitive for DMA operations like GPU rendering, video
+ * encoding/decoding, or displaying buffers on a screen.

I don't think this is correct anymore, right? :(

Quoted 5 lines
drivers/misc/habanalabs/hl_dma_fence.h new file +2 -0
@@ -0,0 +1,148 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*

drivers/misc/habanalabs/hl_dma_fence.h line 12

Quoted 10 lines
drivers/misc/habanalabs/hl_dma_fence.h new file +10 -0
+ * Fence mechanism for dma-buf to allow for asynchronous dma access
+ *
+ * Copyright (C) 2012 Canonical Ltd
+ * Copyright (C) 2012 Texas Instruments
+ *
+ * Authors:
+ * Rob Clark <[email protected]>
+ * Maarten Lankhorst <[email protected]>
+ *
+ * The dma_fence module is a copy of dma-fence at drivers/dma-buf.

Same comments here for the .h file.

thanks,

greg k-h

#6 lore raw
Quoted 9 lines · Greg Kroah-Hartman
Quoted 6 lines · Oded Gabbay

From: Ofir Bitton <[email protected]>

Instead of using standard dma-fence mechanism designed for GPU's, we introduce our own implementation based on the former one. This implementation is much more sparse than the original, contains only mandatory functionality required by the driver.

Quoted 5 lines

Sad you can't use the in-kernel code for this, I really don't understand what's wrong with using it as-is.

Daniel, why do we need/want duplicate code floating around in the tree like this?

The rules around dma-fence are ridiculously strict, and it only makes sense to inflict that upon you if you actually want to participate in the cross driver uapi built up around dma-buf and dma-fence.

I've recently started some lockdep annotations to better enforce these rules (and document them), and it's finding tons of subtle bugs even in drivers/gpu (and I only just started with annotating drivers:

https://lore.kernel.org/dri-devel/[email protected]/

You really don't want to deal with this if you don't have to. If drivers/gpu folks (who created this) aren't good enough to understand it, maybe it's not a good idea to sprinkle this all over the tree. And fundamentally all this is is a slightly fancier struct completion. Use that one instead, or a wait_queue.

I discussed this a bit with Oded, and he thinks it's easier to copypaste and simplify, but given that all other drivers seem to get by perfectly well with completion or wait_queue, I'm not sure that's a solid case.

Also adding Jason Gunthorpe, who very much suggested this should be limited to dma-buf/gpu related usage only.

Quoted 1 line

Copying code leads to errors, here's some documentation ones:

Yeah except here reusing code without understanding what it does and how it should be used leads to error :-) At least given by the drivers/gpu track record, I'm pretty sure sprinkling my new dma_fence lockdep annotations would lead to lots of splats.

Cheers, Daniel

Quoted 75 lines

drivers/misc/habanalabs/hl_dma_fence.c line 3

Quoted 7 lines
drivers/misc/habanalabs/hl_dma_fence.c new file +3 -0
@@ -0,0 +1,338 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Fence mechanism for dma-buf and to allow for asynchronous dma access

Is that what this still does?

drivers/misc/habanalabs/hl_dma_fence.c line 12

Quoted 9 lines
drivers/misc/habanalabs/hl_dma_fence.c new file +9 -0
+ *
+ * Copyright (C) 2012 Canonical Ltd
+ * Copyright (C) 2012 Texas Instruments
+ *
+ * Authors:
+ * Rob Clark <[email protected]>
+ * Maarten Lankhorst <[email protected]>
+ *
+ * The dma_fence module is a copy of dma-fence at drivers/dma-buf.

"The hl_dma_fence" module...

And is it a stand-alone module? Or just a single file?

drivers/misc/habanalabs/hl_dma_fence.c line 14

Quoted 2 lines
drivers/misc/habanalabs/hl_dma_fence.c new file +2 -0
+ * This was done due to an explicit request by GPU developers who asked not
+ * to use the dma-buf module because we aren't part of DRM subsystem.

Why is dma-buf only for use for DRM?

If it is, should the symbol namespace be set to that to catch users that want to use it for their own code?

Quoted 9 lines
drivers/misc/habanalabs/hl_dma_fence.c new file +9 -0
+ * This copy was stripped from all extra features that habanalabs driver
+ * doesn't use, including the uapi interface dma-buf exposes.
+ * In addition, we removed the callbacks because the only usage is from inside
+ * habanalabs driver
+ */
+
+#include "hl_dma_fence.h"
+#include "habanalabs.h"
+#include <linux/slab.h>

drivers/misc/habanalabs/hl_dma_fence.c line 33

Quoted 10 lines
drivers/misc/habanalabs/hl_dma_fence.c new file +10 -0
+#include <linux/export.h>
+#include <linux/atomic.h>
+#include <linux/sched/signal.h>
+
+/**
+ * DOC: DMA fences overview
+ *
+ * DMA fences, represented by &struct hl_dma_fence, are the kernel internal
+ * synchronization primitive for DMA operations like GPU rendering, video
+ * encoding/decoding, or displaying buffers on a screen.

I don't think this is correct anymore, right? :(

Quoted 5 lines
drivers/misc/habanalabs/hl_dma_fence.h new file +2 -0
@@ -0,0 +1,148 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*

drivers/misc/habanalabs/hl_dma_fence.h line 12

Quoted 10 lines
drivers/misc/habanalabs/hl_dma_fence.h new file +10 -0
+ * Fence mechanism for dma-buf to allow for asynchronous dma access
+ *
+ * Copyright (C) 2012 Canonical Ltd
+ * Copyright (C) 2012 Texas Instruments
+ *
+ * Authors:
+ * Rob Clark <[email protected]>
+ * Maarten Lankhorst <[email protected]>
+ *
+ * The dma_fence module is a copy of dma-fence at drivers/dma-buf.

Same comments here for the .h file.

thanks,

greg k-h

Signature · 3 lines
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch
#7 lore raw
Quoted 33 lines · Daniel Vetter
Quoted 9 lines · Greg Kroah-Hartman
Quoted 6 lines · Oded Gabbay

From: Ofir Bitton <[email protected]>

Instead of using standard dma-fence mechanism designed for GPU's, we introduce our own implementation based on the former one. This implementation is much more sparse than the original, contains only mandatory functionality required by the driver.

Quoted 5 lines

Sad you can't use the in-kernel code for this, I really don't understand what's wrong with using it as-is.

Daniel, why do we need/want duplicate code floating around in the tree like this?

The rules around dma-fence are ridiculously strict, and it only makes sense to inflict that upon you if you actually want to participate in the cross driver uapi built up around dma-buf and dma-fence.

I've recently started some lockdep annotations to better enforce these rules (and document them), and it's finding tons of subtle bugs even in drivers/gpu (and I only just started with annotating drivers:

https://lore.kernel.org/dri-devel/[email protected]/

You really don't want to deal with this if you don't have to. If drivers/gpu folks (who created this) aren't good enough to understand it, maybe it's not a good idea to sprinkle this all over the tree. And fundamentally all this is is a slightly fancier struct completion. Use that one instead, or a wait_queue.

Quoted 7 lines

I discussed this a bit with Oded, and he thinks it's easier to copypaste and simplify, but given that all other drivers seem to get by perfectly well with completion or wait_queue, I'm not sure that's a solid case.

Also adding Jason Gunthorpe, who very much suggested this should be limited to dma-buf/gpu related usage only.

Without all the cross-driver stuff dma_fence is just a completion. Using dma_fence to get a completion is big abuse of what it is intended for.

I think the only problem with this patch is that it keeps too much of the dma_fence stuff around. From what I could tell it really just wants to add a kref and completion to struct hl_cs_compl and delete everything to do with dma_fence.

Not even sure the kref is needed ;)

Jason

#8 lore raw
Quoted 43 lines · Jason Gunthorpe
Quoted 33 lines · Daniel Vetter
Quoted 9 lines · Greg Kroah-Hartman
Quoted 6 lines · Oded Gabbay

From: Ofir Bitton <[email protected]>

Instead of using standard dma-fence mechanism designed for GPU's, we introduce our own implementation based on the former one. This implementation is much more sparse than the original, contains only mandatory functionality required by the driver.

Quoted 5 lines

Sad you can't use the in-kernel code for this, I really don't understand what's wrong with using it as-is.

Daniel, why do we need/want duplicate code floating around in the tree like this?

The rules around dma-fence are ridiculously strict, and it only makes sense to inflict that upon you if you actually want to participate in the cross driver uapi built up around dma-buf and dma-fence.

I've recently started some lockdep annotations to better enforce these rules (and document them), and it's finding tons of subtle bugs even in drivers/gpu (and I only just started with annotating drivers:

https://lore.kernel.org/dri-devel/[email protected]/

You really don't want to deal with this if you don't have to. If drivers/gpu folks (who created this) aren't good enough to understand it, maybe it's not a good idea to sprinkle this all over the tree. And fundamentally all this is is a slightly fancier struct completion. Use that one instead, or a wait_queue.

Quoted 7 lines

I discussed this a bit with Oded, and he thinks it's easier to copypaste and simplify, but given that all other drivers seem to get by perfectly well with completion or wait_queue, I'm not sure that's a solid case.

Also adding Jason Gunthorpe, who very much suggested this should be limited to dma-buf/gpu related usage only.

Quoted 8 lines

Without all the cross-driver stuff dma_fence is just a completion. Using dma_fence to get a completion is big abuse of what it is intended for.

I think the only problem with this patch is that it keeps too much of the dma_fence stuff around. From what I could tell it really just wants to add a kref and completion to struct hl_cs_compl and delete everything to do with dma_fence.

Yeah, that's what I recommended doing too. error flag might be needed too I think, but that's it.
-Daniel

Quoted 3 lines

Not even sure the kref is needed ;)

Jason

Signature · 3 lines
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch
#9 lore raw
Quoted 48 lines · Daniel Vetter
Quoted 47 lines · Jason Gunthorpe
Quoted 33 lines · Daniel Vetter
Quoted 9 lines · Greg Kroah-Hartman
Quoted 6 lines · Oded Gabbay

From: Ofir Bitton <[email protected]>

Instead of using standard dma-fence mechanism designed for GPU's, we introduce our own implementation based on the former one. This implementation is much more sparse than the original, contains only mandatory functionality required by the driver.

Quoted 5 lines

Sad you can't use the in-kernel code for this, I really don't understand what's wrong with using it as-is.

Daniel, why do we need/want duplicate code floating around in the tree like this?

The rules around dma-fence are ridiculously strict, and it only makes sense to inflict that upon you if you actually want to participate in the cross driver uapi built up around dma-buf and dma-fence.

I've recently started some lockdep annotations to better enforce these rules (and document them), and it's finding tons of subtle bugs even in drivers/gpu (and I only just started with annotating drivers:

https://lore.kernel.org/dri-devel/[email protected]/

You really don't want to deal with this if you don't have to. If drivers/gpu folks (who created this) aren't good enough to understand it, maybe it's not a good idea to sprinkle this all over the tree. And fundamentally all this is is a slightly fancier struct completion. Use that one instead, or a wait_queue.

Quoted 7 lines

I discussed this a bit with Oded, and he thinks it's easier to copypaste and simplify, but given that all other drivers seem to get by perfectly well with completion or wait_queue, I'm not sure that's a solid case.

Also adding Jason Gunthorpe, who very much suggested this should be limited to dma-buf/gpu related usage only.

Without all the cross-driver stuff dma_fence is just a completion. Using dma_fence to get a completion is big abuse of what it is intended for.

Quoted 7 lines
Quoted 4 lines

I think the only problem with this patch is that it keeps too much of the dma_fence stuff around. From what I could tell it really just wants to add a kref and completion to struct hl_cs_compl and delete everything to do with dma_fence.

Yeah, that's what I recommended doing too. error flag might be needed too I think, but that's it.

Ok, so this should be made much simpler and not use this copy/paste code at all. I can accept that :)

Ofir, care to redo this?

thanks,

greg k-h

#10 lore raw

Sure,
I will send a new patch using completion instead of dma-fence

Thanks,
Ofir

-----Original Message-----
From: Greg Kroah-Hartman <[email protected]>
Sent: Tuesday, July 14, 2020 09:37
To: Daniel Vetter <[email protected]>
Cc: Jason Gunthorpe <[email protected]>; Oded Gabbay <[email protected]>; Linux Kernel Mailing List <[email protected]>; SW_Drivers <[email protected]>; Ofir Bitton <[email protected]>
Subject: Re: [PATCH 1/3] habanalabs: implement dma-fence mechanism

Quoted 49 lines · Daniel Vetter
Quoted 48 lines · Jason Gunthorpe
Quoted 34 lines · Daniel Vetter
Quoted 9 lines · Greg Kroah-Hartman
Quoted 6 lines · Oded Gabbay

From: Ofir Bitton <[email protected]>

Instead of using standard dma-fence mechanism designed for GPU's, we introduce our own implementation based on the former one. This implementation is much more sparse than the original, contains only mandatory functionality required by the driver.

Quoted 5 lines

Sad you can't use the in-kernel code for this, I really don't understand what's wrong with using it as-is.

Daniel, why do we need/want duplicate code floating around in the tree like this?

The rules around dma-fence are ridiculously strict, and it only makes sense to inflict that upon you if you actually want to participate in the cross driver uapi built up around dma-buf and dma-fence.

I've recently started some lockdep annotations to better enforce these rules (and document them), and it's finding tons of subtle bugs even in drivers/gpu (and I only just started with annotating drivers:

https://lore.kernel.org/dri-devel/20200707201229.472834-1-daniel.v
[email protected]/

You really don't want to deal with this if you don't have to. If drivers/gpu folks (who created this) aren't good enough to understand it, maybe it's not a good idea to sprinkle this all over the tree. And fundamentally all this is is a slightly fancier struct completion. Use that one instead, or a wait_queue.

Quoted 7 lines

I discussed this a bit with Oded, and he thinks it's easier to copypaste and simplify, but given that all other drivers seem to get by perfectly well with completion or wait_queue, I'm not sure that's a solid case.

Also adding Jason Gunthorpe, who very much suggested this should be limited to dma-buf/gpu related usage only.

Without all the cross-driver stuff dma_fence is just a completion. Using dma_fence to get a completion is big abuse of what it is intended for.

Quoted 7 lines
Quoted 4 lines

I think the only problem with this patch is that it keeps too much of the dma_fence stuff around. From what I could tell it really just wants to add a kref and completion to struct hl_cs_compl and delete everything to do with dma_fence.

Yeah, that's what I recommended doing too. error flag might be needed too I think, but that's it.

Ok, so this should be made much simpler and not use this copy/paste code at all. I can accept that :)

Ofir, care to redo this?

thanks,

greg k-h