habanalabs: sync stream collective infrastructure
PATCH 1/3 habanalabs: sync stream collective infrastructure
From: Ofir Bitton <[email protected]>
Define new API for collective wait support and modify sync stream common flow. In addition add kernel CB allocation support for internal queues.
--- .../habanalabs/common/command_submission.c | 38 ++++++++++--- drivers/misc/habanalabs/common/habanalabs.h | 55 +++++++++++++++++-- drivers/misc/habanalabs/common/hw_queue.c | 44 ++++++++++++++- drivers/misc/habanalabs/gaudi/gaudi.c | 22 +++++++- drivers/misc/habanalabs/goya/goya.c | 22 +++++++- include/uapi/misc/habanalabs.h | 17 ++++-- 6 files changed, 176 insertions(+), 22 deletions(-)
drivers/misc/habanalabs/common/command_submission.c +31 -7
@@ -85,7 +85,8 @@ static void hl_fence_release(struct kref *kref) goto free;if ((hl_cs_cmpl->type ==CS_TYPE_SIGNAL ) || - (hl_cs_cmpl->type ==CS_TYPE_WAIT )) { + (hl_cs_cmpl->type ==CS_TYPE_WAIT ) || + (hl_cs_cmpl->type ==CS_TYPE_COLLECTIVE_WAIT )) { dev_dbg(hdev->dev, "CS 0x%llx type %d finished, sob_id: %d, sob_val: 0x%x\n",
@@ -112,6 +113,10 @@ static void hl_fence_release(struct kref *kref)* hence the above scenario is avoided. */ kref_put (&hl_cs_cmpl->hw_sob->kref, hl_sob_reset); + +if (hl_cs_cmpl->type ==CS_TYPE_COLLECTIVE_WAIT ) + hdev->asic_funcs->reset_sob_group (hdev, + hl_cs_cmpl->sob_group); } free:
@@ -247,9 +252,11 @@ static void free_job(struct hl_device *hdev, struct hl_cs_job *job)/* For H/W queue jobs, if a user CB was allocated by driver and MMU is * enabled, the user CB isn't released in cs_parser() and thus should be * released here. +* This is also true for INT queues jobs which were allocated by driver */ -if (job->queue_type ==QUEUE_TYPE_HW && - job->is_kernel_allocated_cb && hdev->mmu_enable) { +if (job->is_kernel_allocated_cb && + ((job->queue_type ==QUEUE_TYPE_HW && hdev->mmu_enable) || + job->queue_type ==QUEUE_TYPE_INT )) {spin_lock (&job->user_cb->lock); job->user_cb->cs_cnt--;spin_unlock (&job->user_cb->lock);
@@ -932,7 +939,7 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type,struct hl_cs_compl *sig_waitcs_cmpl;struct hl_cs *cs;enum hl_queue_type q_type; -u32 size_to_copy, q_idx; +u32 size_to_copy, q_idx, collective_engine_id;u64 signal_seq;int rc;
@@ -983,7 +990,18 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, goto free_cs_chunk_array; } -if (cs_type ==CS_TYPE_WAIT ) { +if (cs_type ==CS_TYPE_COLLECTIVE_WAIT ) { +if (!(hw_queue_prop->collective_mode ==HL_COLLECTIVE_MASTER )) { +dev_err (hdev->dev, +"Queue index %d is invalid\n" , q_idx); + rc = -EINVAL ; + goto free_cs_chunk_array; + } + + collective_engine_id = chunk->collective_engine_id; + } + +if (cs_type ==CS_TYPE_WAIT || cs_type ==CS_TYPE_COLLECTIVE_WAIT ) { rc =cs_ioctl_extract_signal_seq (hdev, chunk, &signal_seq);if (rc) goto free_cs_chunk_array;
@@ -1028,7 +1046,8 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, rc =allocate_cs (hdev, ctx, cs_type, &cs);if (rc) { -if (cs_type ==CS_TYPE_WAIT ) +if (cs_type ==CS_TYPE_WAIT || + cs_type ==CS_TYPE_COLLECTIVE_WAIT )hl_fence_put (sig_fence);hl_ctx_put (ctx); goto free_cs_chunk_array;
@@ -1038,7 +1057,7 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type,* Save the signal CS fence for later initialization right before * hanging the wait CS on the queue. */ -if (cs_type ==CS_TYPE_WAIT ) +if (cs_type ==CS_TYPE_WAIT || cs_type ==CS_TYPE_COLLECTIVE_WAIT ) cs->signal_fence = sig_fence;hl_debugfs_add_cs (cs);
@@ -1048,6 +1067,9 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type,if (cs_type ==CS_TYPE_WAIT || cs_type ==CS_TYPE_SIGNAL ) rc =cs_ioctl_signal_wait_create_jobs (hdev, ctx, cs, q_type, q_idx); +else + rc = hdev->asic_funcs->collective_wait_create_jobs (hdev, ctx, + cs, q_idx, collective_engine_id);if (rc) goto put_cs;
@@ -1122,6 +1144,8 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data) cs_type =CS_TYPE_SIGNAL ; else if (args->in.cs_flags & HL_CS_FLAGS_WAIT) cs_type =CS_TYPE_WAIT ; + else if (args->in.cs_flags & HL_CS_FLAGS_COLLECTIVE_WAIT) + cs_type =CS_TYPE_COLLECTIVE_WAIT ; else cs_type =CS_TYPE_DEFAULT ;
drivers/misc/habanalabs/common/habanalabs.h +49 -6
166 lines not on this page · raw message
drivers/misc/habanalabs/common/hw_queue.c +41 -3
72 lines not on this page · raw message
drivers/misc/habanalabs/gaudi/gaudi.c +21 -1
@@ -794,6 +794,23 @@ static int gaudi_init_tpc_mem(struct hl_device *hdev)return rc; } +static void gaudi_reset_sob_group (struct hl_device *hdev,u16 sob_groupt) +{ + +} + +static void gaudi_collective_wait_init_cs (struct hl_cs *cs) +{ + +} + +static int gaudi_collective_wait_create_jobs (struct hl_device *hdev, +struct hl_ctx *ctx,struct hl_cs *cs,u32 wait_queue_id, +u32 collective_engine_id) +{ +return -EINVAL ; +} +static int gaudi_late_init (struct hl_device *hdev) {struct gaudi_device *gaudi = hdev->asic_specific;
@@ -7375,8 +7392,11 @@ static const struct hl_asic_funcs gaudi_funcs = { .gen_signal_cb = gaudi_gen_signal_cb, .gen_wait_cb = gaudi_gen_wait_cb, .reset_sob = gaudi_reset_sob, + .reset_sob_group = gaudi_reset_sob_group, .set_dma_mask_from_fw = gaudi_set_dma_mask_from_fw, - .get_device_time = gaudi_get_device_time + .get_device_time = gaudi_get_device_time, + .collective_wait_init_cs = gaudi_collective_wait_init_cs, + .collective_wait_create_jobs = gaudi_collective_wait_create_jobs };/**
drivers/misc/habanalabs/goya/goya.c +21 -1
@@ -5309,6 +5309,11 @@ static void goya_reset_sob(struct hl_device *hdev, void *data) } +void goya_reset_sob_group (struct hl_device *hdev,u16 sob_group) +{ + +} +static void goya_set_dma_mask_from_fw (struct hl_device *hdev) { if (RREG32(mmPSOC_GLOBAL_CONF_NON_RST_FLOPS_0) ==
@@ -5330,6 +5335,18 @@ u64 goya_get_device_time(struct hl_device *hdev)return device_time |RREG32 (mmPSOC_TIMESTAMP_CNTCVL); } +void goya_collective_wait_init_cs (struct hl_cs *cs) +{ + +} + +int goya_collective_wait_create_jobs (struct hl_device *hdev, +struct hl_ctx *ctx,struct hl_cs *cs,u32 wait_queue_id, +u32 collective_engine_id) +{ +return -EINVAL ; +} +static const struct hl_asic_funcs goya_funcs = { .early_init = goya_early_init, .early_fini = goya_early_fini,
@@ -5401,8 +5418,11 @@ static const struct hl_asic_funcs goya_funcs = { .gen_signal_cb = goya_gen_signal_cb, .gen_wait_cb = goya_gen_wait_cb, .reset_sob = goya_reset_sob, + .reset_sob_group = goya_reset_sob_group, .set_dma_mask_from_fw = goya_set_dma_mask_from_fw, - .get_device_time = goya_get_device_time + .get_device_time = goya_get_device_time, + .collective_wait_init_cs = goya_collective_wait_init_cs, + .collective_wait_create_jobs = goya_collective_wait_create_jobs };/*
include/uapi/misc/habanalabs.h +13 -4
@@ -523,7 +523,8 @@ struct hl_cs_chunk {*/ __u64 cb_handle; -/* Relevant only when HL_CS_FLAGS_WAIT is set. +/* Relevant only when HL_CS_FLAGS_WAIT or +* HL_CS_FLAGS_COLLECTIVE_WAIT is set. * This holds address of array of u64 values that contain * signal CS sequence numbers. The wait described by this job * will listen on all those signals (wait event per signal)
@@ -541,7 +542,8 @@ struct hl_cs_chunk {*/ __u32 cb_size; -/* Relevant only when HL_CS_FLAGS_WAIT is set. +/* Relevant only when HL_CS_FLAGS_WAIT or +* HL_CS_FLAGS_COLLECTIVE_WAIT is set. * Number of entries in signal_seq_arr */ __u32 num_signal_seq_arr;
@@ -550,14 +552,21 @@ struct hl_cs_chunk {/* HL_CS_CHUNK_FLAGS_* */ __u32 cs_chunk_flags; +/* Relevant only when HL_CS_FLAGS_COLLECTIVE_WAIT is set. +* This holds the collective engine ID. The wait described by this job +* will sync with this engine and with all NICs before completion. +*/ +__u32 collective_engine_id; +/* Align structure to 64 bytes */ -__u32 pad[11 ]; +__u32 pad[10 ]; }; -/* SIGNAL and WAIT flags are mutually exclusive */ +/* SIGNAL and WAIT/COLLECTIVE_WAIT flags are mutually exclusive */ #define HL_CS_FLAGS_FORCE_RESTORE 0x1#define HL_CS_FLAGS_SIGNAL 0x2#define HL_CS_FLAGS_WAIT 0x4 +#define HL_CS_FLAGS_COLLECTIVE_WAIT 0x8#define HL_CS_STATUS_SUCCESS 0
Signature · 1 line
2.17.1
PATCH 1/1 habanalabs: use enum for CB allocation options
From: Tal Cohen <[email protected]>
In the future there will be situations where queues can accept either kernel allocated CBs or user allocated CBs, depending on different states.
Therefore, instead of using a boolean variable of kernel/user allocated CB, we need to use a bitmask to indicate that, which will allow to combine the two options.
Add a flag to the uapi so the user will be able to indicate whether the CB was allocated by kernel or by user. Of course the driver validates that.
--- .../habanalabs/common/command_submission.c | 31 +++++++++++++++++-- drivers/misc/habanalabs/common/habanalabs.h | 19 ++++++++++-- drivers/misc/habanalabs/gaudi/gaudi.c | 13 +++++--- drivers/misc/habanalabs/goya/goya.c | 6 ++-- include/uapi/misc/habanalabs.h | 16 ++++++++++ 5 files changed, 73 insertions(+), 12 deletions(-)
drivers/misc/habanalabs/common/command_submission.c +29 -2
@@ -568,9 +568,36 @@ static int validate_queue_index(struct hl_device *hdev,return -EINVAL ; } - *queue_type = hw_queue_prop->type; - *is_kernel_allocated_cb = !!hw_queue_prop->requires_kernel_cb; +/* When hw queue type isn't QUEUE_TYPE_HW, +* USER_ALLOC_CB flag shall be referred as "don't care". +*/ +if (hw_queue_prop->type ==QUEUE_TYPE_HW ) { +if (chunk->cs_chunk_flags &HL_CS_CHUNK_FLAGS_USER_ALLOC_CB ) { +if (!(hw_queue_prop->cb_alloc_flags &CB_ALLOC_USER )) { +dev_err (hdev->dev, +"Queue index %d doesn't support user CB\n" , + chunk->queue_index); +return -EINVAL ; + } + *is_kernel_allocated_cb = false; + }else { +if (!(hw_queue_prop->cb_alloc_flags & +CB_ALLOC_KERNEL )) { +dev_err (hdev->dev, +"Queue index %d doesn't support kernel CB\n" , + chunk->queue_index); +return -EINVAL ; + } + + *is_kernel_allocated_cb = true; + } + }else { + *is_kernel_allocated_cb = !!(hw_queue_prop->cb_alloc_flags + &CB_ALLOC_KERNEL ); + } + + *queue_type = hw_queue_prop->type;return 0 ; }
drivers/misc/habanalabs/common/habanalabs.h +16 -3
@@ -206,6 +206,17 @@ struct hl_outbound_pci_region {u64 size; }; +/* +* enum queue_cb_alloc_flags - Indicates queue support for CBs that +* allocated by Kernel or by User +* @CB_ALLOC_KERNEL: support only CBs that allocated by Kernel +* @CB_ALLOC_USER: support only CBs that allocated by User +*/ +enum queue_cb_alloc_flags { +CB_ALLOC_KERNEL =0x1 , +CB_ALLOC_USER =0x2 +}; +/* * struct hl_hw_sob - H/W SOB info. * @hdev: habanalabs device structure.
@@ -223,16 +234,18 @@ struct hl_hw_sob {/** * struct hw_queue_properties - queue information. * @type: queue type. +* @queue_cb_alloc_flags: bitmap which indicates if the hw queue supports CB +* that allocated by the Kernel driver and therefore, +* a CB handle can be provided for jobs on this queue. +* Otherwise, a CB address must be provided. * @driver_only: true if only the driver is allowed to send a job to this queue, * false otherwise. -* @requires_kernel_cb: true if a CB handle must be provided for jobs on this -* queue, false otherwise (a CB address must be provided). * @supports_sync_stream: True if queue supports sync stream */ struct hw_queue_properties {enum hl_queue_type type; +enum queue_cb_alloc_flags cb_alloc_flags;u8 driver_only; -u8 requires_kernel_cb;u8 supports_sync_stream; };
drivers/misc/habanalabs/gaudi/gaudi.c +9 -4
@@ -381,23 +381,28 @@ static int gaudi_get_fixed_properties(struct hl_device *hdev)if (gaudi_queue_type[i] ==QUEUE_TYPE_EXT ) { prop->hw_queues_props[i].type =QUEUE_TYPE_EXT ; prop->hw_queues_props[i].driver_only =0 ; - prop->hw_queues_props[i].requires_kernel_cb =1 ; prop->hw_queues_props[i].supports_sync_stream =1 ; + prop->hw_queues_props[i].cb_alloc_flags = +CB_ALLOC_KERNEL ; num_sync_stream_queues++; }else if (gaudi_queue_type[i] ==QUEUE_TYPE_CPU ) { prop->hw_queues_props[i].type =QUEUE_TYPE_CPU ; prop->hw_queues_props[i].driver_only =1 ; - prop->hw_queues_props[i].requires_kernel_cb =0 ; prop->hw_queues_props[i].supports_sync_stream =0 ; + prop->hw_queues_props[i].cb_alloc_flags = +CB_ALLOC_KERNEL ; }else if (gaudi_queue_type[i] ==QUEUE_TYPE_INT ) { prop->hw_queues_props[i].type =QUEUE_TYPE_INT ; prop->hw_queues_props[i].driver_only =0 ; - prop->hw_queues_props[i].requires_kernel_cb =0 ; + prop->hw_queues_props[i].supports_sync_stream =0 ; + prop->hw_queues_props[i].cb_alloc_flags = +CB_ALLOC_USER ; }else if (gaudi_queue_type[i] ==QUEUE_TYPE_NA ) { prop->hw_queues_props[i].type =QUEUE_TYPE_NA ; prop->hw_queues_props[i].driver_only =0 ; - prop->hw_queues_props[i].requires_kernel_cb =0 ; prop->hw_queues_props[i].supports_sync_stream =0 ; + prop->hw_queues_props[i].cb_alloc_flags = +CB_ALLOC_USER ; } }
drivers/misc/habanalabs/goya/goya.c +3 -3
@@ -373,20 +373,20 @@ int goya_get_fixed_properties(struct hl_device *hdev)for (i =0 ; i <NUMBER_OF_EXT_HW_QUEUES ; i++) { prop->hw_queues_props[i].type =QUEUE_TYPE_EXT ; prop->hw_queues_props[i].driver_only =0 ; - prop->hw_queues_props[i].requires_kernel_cb =1 ; + prop->hw_queues_props[i].cb_alloc_flags =CB_ALLOC_KERNEL ; }for (; i <NUMBER_OF_EXT_HW_QUEUES +NUMBER_OF_CPU_HW_QUEUES ; i++) { prop->hw_queues_props[i].type =QUEUE_TYPE_CPU ; prop->hw_queues_props[i].driver_only =1 ; - prop->hw_queues_props[i].requires_kernel_cb =0 ; + prop->hw_queues_props[i].cb_alloc_flags =CB_ALLOC_KERNEL ; }for (; i <NUMBER_OF_EXT_HW_QUEUES +NUMBER_OF_CPU_HW_QUEUES +NUMBER_OF_INT_HW_QUEUES ; i++) { prop->hw_queues_props[i].type =QUEUE_TYPE_INT ; prop->hw_queues_props[i].driver_only =0 ; - prop->hw_queues_props[i].requires_kernel_cb =0 ; + prop->hw_queues_props[i].cb_alloc_flags =CB_ALLOC_USER ; } prop->completion_queues_count =NUMBER_OF_CMPLT_QUEUES ;
include/uapi/misc/habanalabs.h +16 -0
@@ -490,6 +490,22 @@ union hl_cb_args {struct hl_cb_out out; }; +/* HL_CS_CHUNK_FLAGS_ values +* +* HL_CS_CHUNK_FLAGS_USER_ALLOC_CB: +* Indicates if the CB was allocated and mapped by userspace. +* User allocated CB is a command buffer allocated by the user, via malloc +* (or similar). After allocating the CB, the user invokes “memory ioctl” +* to map the user memory into a device virtual address. The user provides +* this address via the cb_handle field. The interface provides the +* ability to create a large CBs, Which aren’t limited to +* “HL_MAX_CB_SIZE”. Therefore, it increases the PCI-DMA queues +* throughput. This CB allocation method also reduces the use of Linux +* DMA-able memory pool. Which are limited and used by other Linux +* sub-systems. +*/ +#define HL_CS_CHUNK_FLAGS_USER_ALLOC_CB 0x1 +/* * This structure size must always be fixed to 64-bytes for backward * compatibility
Signature · 1 line
2.17.1
PATCH 3/3 habanalabs: sync stream collective support
In reply to Oded GabbayFrom: Ofir Bitton <[email protected]>
Implement sync stream collective for GAUDI. Need to allocate additional resources for that and add ctx_fini() to clean up those resources.
--- .../habanalabs/common/command_submission.c | 11 +- drivers/misc/habanalabs/common/context.c | 1 + drivers/misc/habanalabs/common/habanalabs.h | 7 +- drivers/misc/habanalabs/gaudi/gaudi.c | 853 ++++++++++++++++-- drivers/misc/habanalabs/gaudi/gaudiP.h | 40 +- drivers/misc/habanalabs/goya/goya.c | 6 + include/uapi/misc/habanalabs.h | 14 +- 7 files changed, 867 insertions(+), 65 deletions(-)
drivers/misc/habanalabs/common/command_submission.c +5 -6
@@ -142,7 +142,7 @@ static void hl_fence_init(struct hl_fence *fence)init_completion (&fence->completion); } -static void cs_get (struct hl_cs *cs) +void cs_get (struct hl_cs *cs) {kref_get (&cs->refcount); }
@@ -917,6 +917,9 @@ static int cs_ioctl_signal_wait_create_jobs(struct hl_device *hdev, job->job_cb_size = job->user_cb_size;hl_cb_destroy (hdev, &hdev->kernel_cb_mgr, cb->id <<PAGE_SHIFT ); +/* increment refcount as for external queues we get completion */ +cs_get (cs); + cs->jobs_in_queue_cnt[job->hw_queue_id]++;list_add_tail (&job->cs_node, &cs->job_list);
@@ -1072,11 +1075,7 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, cs, q_idx, collective_engine_id);if (rc) - goto put_cs; - - -/* increment refcount as for external queues we get completion */ -cs_get (cs); + goto free_cs_object; rc =hl_hw_queue_schedule_cs (cs);if (rc) {
drivers/misc/habanalabs/common/context.c +1 -0
@@ -40,6 +40,7 @@ static void hl_ctx_fini(struct hl_ctx *ctx)if ((hdev->in_debug) && (hdev->compute_ctx == ctx))hl_device_set_debug_mode (hdev, false); + hdev->asic_funcs->ctx_fini (ctx);hl_cb_va_pool_fini (ctx);hl_vm_ctx_fini (ctx);hl_asid_free (hdev, ctx->asid);
drivers/misc/habanalabs/common/habanalabs.h +5 -2
@@ -65,8 +65,8 @@* HL_RSVD_SOBS 'sync stream' reserved sync objects per QMAN stream * HL_RSVD_MONS 'sync stream' reserved monitors per QMAN stream */ -#define HL_RSVD_SOBS 4 -#define HL_RSVD_MONS 2 +#define HL_RSVD_SOBS 2 +#define HL_RSVD_MONS 1/* * HL_COLLECTIVE_RSVD_MSTR_MONS 'collective' reserved monitors per QMAN stream
@@ -785,6 +785,7 @@ enum div_select_defs {* @wreg: Write a register. Needed for simulator support. * @halt_coresight: stop the ETF and ETR traces. * @ctx_init: context dependent initialization. +* @ctx_fini: context dependent cleanup. * @get_clk_rate: Retrieve the ASIC current and maximum clock rate in MHz * @get_queue_id_for_cq: Get the H/W queue id related to the given CQ index. * @read_device_fw_version: read the device's firmware versions that are
@@ -891,6 +892,7 @@ struct hl_asic_funcs {void (*wreg)(struct hl_device *hdev,u32 reg,u32 val);void (*halt_coresight)(struct hl_device *hdev);int (*ctx_init)(struct hl_ctx *ctx); +void (*ctx_fini)(struct hl_ctx *ctx);int (*get_clk_rate)(struct hl_device *hdev,u32 *cur_clk,u32 *max_clk);u32 (*get_queue_id_for_cq)(struct hl_device *hdev,u32 cq_idx); void (*read_device_fw_version)(struct hl_device *hdev,
@@ -1992,6 +1994,7 @@ void hl_sob_reset_error(struct kref *ref);int hl_gen_sob_mask (u16 sob_base,u8 sob_mask,u8 *mask);void hl_fence_put (struct hl_fence *fence);void hl_fence_get (struct hl_fence *fence); +void cs_get (struct hl_cs *cs);void goya_set_asic_funcs (struct hl_device *hdev);void gaudi_set_asic_funcs (struct hl_device *hdev);
drivers/misc/habanalabs/gaudi/gaudi.c +799 -54
1,000 lines not on this page · raw message
drivers/misc/habanalabs/gaudi/gaudiP.h +39 -1
@@ -23,6 +23,10 @@ NUMBER_OF_CPU_HW_QUEUES + \ NUMBER_OF_INT_HW_QUEUES) +/* 10 NIC QMANs, DMA5 QMAN, TPC7 QMAN */ +#define NUMBER_OF_COLLECTIVE_QUEUES 12 +#define NUMBER_OF_SOBS_IN_GRP 11 +/* * Number of MSI interrupts IDS: * Each completion queue has 1 ID
@@ -149,10 +153,14 @@/* Virtual address space */ #define VA_HOST_SPACE_START 0x1000000000000ull/* 256TB */ -#define VA_HOST_SPACE_END 0x3FF8000000000ull/* 1PB - 1TB */ +#define VA_HOST_SPACE_END 0x3FF7FFFE00000ull/* 1PB - 1TB */ #define VA_HOST_SPACE_SIZE (VA_HOST_SPACE_END - \ VA_HOST_SPACE_START)/* 767TB */ +#define VA_HOST_SPACE_INTERNAL_CB_START 0x3FF7FFFE00000ull/* 1PB - 1TB - 2MB */ +#define VA_HOST_SPACE_INTERNAL_CB_END 0x3FF8000000000ull/* 1PB - 1TB */ +#define HOST_SPACE_INTERNAL_CB_SZ SZ_2M +#define HW_CAP_PLL BIT(0)#define HW_CAP_HBM BIT(1)#define HW_CAP_MMU BIT(2)
@@ -240,6 +248,34 @@ enum gaudi_nic_mask { GAUDI_NIC_MASK_ALL = 0x3FF }; +/* +* struct gaudi_hw_sob_group - H/W SOB group info. +* @hdev: habanalabs device structure. +* @kref: refcount of this SOB group. group will reset once refcount is zero. +* @base_sob_id: base sob id of this SOB group. +*/ +struct gaudi_hw_sob_group { +struct hl_device *hdev; +struct kref kref; +u32 base_sob_id; +}; + +#define NUM_SOB_GROUPS (HL_RSVD_SOBS * QMAN_STREAMS) +/** +* struct gaudi_collective_properties - +* holds all SOB groups and queues info reserved for the collective +* @hw_sob_group: H/W SOB groups. +* @next_sob_group_val: the next value to use for the currently used SOB group. +* @curr_sob_group_idx: the index of the currently used SOB group. +* @mstr_sob_mask: pre-defined masks for collective master monitors +*/ +struct gaudi_collective_properties { +struct gaudi_hw_sob_group hw_sob_group[NUM_SOB_GROUPS ]; +u16 next_sob_group_val[QMAN_STREAMS ]; +u8 curr_sob_group_idx[QMAN_STREAMS ]; +u8 mstr_sob_mask[HL_COLLECTIVE_RSVD_MSTR_MONS ]; +}; +/** * struct gaudi_internal_qman_info - Internal QMAN information. * @pq_kernel_addr: Kernel address of the PQ memory area in the host.
@@ -285,6 +321,8 @@ struct gaudi_device {struct gaudi_internal_qman_info internal_qmans[GAUDI_QUEUE_ID_SIZE ]; +struct gaudi_collective_properties collective_props; +u64 hbm_bar_cur_addr;u64 max_freq_value;
drivers/misc/habanalabs/goya/goya.c +6 -0
@@ -5347,6 +5347,11 @@ int goya_collective_wait_create_jobs(struct hl_device *hdev,return -EINVAL ; } +static void goya_ctx_fini (struct hl_ctx *ctx) +{ + +} +static const struct hl_asic_funcs goya_funcs = { .early_init = goya_early_init, .early_fini = goya_early_fini,
@@ -5408,6 +5413,7 @@ static const struct hl_asic_funcs goya_funcs = { .wreg = hl_wreg, .halt_coresight = goya_halt_coresight, .ctx_init = goya_ctx_init, + .ctx_fini = goya_ctx_fini, .get_clk_rate = goya_get_clk_rate, .get_queue_id_for_cq = goya_get_queue_id_for_cq, .read_device_fw_version = goya_read_device_fw_version,
include/uapi/misc/habanalabs.h +12 -2
@@ -18,8 +18,18 @@#define GOYA_KMD_SRAM_RESERVED_SIZE_FROM_START 0x8000/* 32KB */ #define GAUDI_DRIVER_SRAM_RESERVED_SIZE_FROM_START 0x80/* 128 bytes */ -#define GAUDI_FIRST_AVAILABLE_W_S_SYNC_OBJECT 32 -#define GAUDI_FIRST_AVAILABLE_W_S_MONITOR 16 +/* +* 128 SOBs reserved for collective wait +* 16 SOBs reserved for sync stream +*/ +#define GAUDI_FIRST_AVAILABLE_W_S_SYNC_OBJECT 144 + +/* +* 64 monitors reserved for collective wait +* 8 monitors reserved for sync stream +*/ +#define GAUDI_FIRST_AVAILABLE_W_S_MONITOR 72 +/* * Goya queue Numbering *
Signature · 1 line
2.17.1
PATCH 2/3 habanalabs/gaudi: Set DMA5 QMAN internal
In reply to Oded GabbayFrom: Ofir Bitton <[email protected]>
DMA5 QMAN is designated to be used for reduction process, hence it will be no longer configured as external queue.
--- drivers/misc/habanalabs/gaudi/gaudi.c | 31 +++++++++++--------------- drivers/misc/habanalabs/gaudi/gaudiP.h | 8 +++---- include/uapi/misc/habanalabs.h | 12 +++++----- 3 files changed, 23 insertions(+), 28 deletions(-)
drivers/misc/habanalabs/gaudi/gaudi.c +13 -18
@@ -38,7 +38,7 @@* * MMU is always enabled. * -* QMAN DMA channels 0,1,5 (PCI DMAN): +* QMAN DMA channels 0,1 (PCI DMAN): * - DMA is not secured. * - PQ and CQ are secured. * - CP is secured: The driver needs to parse CB but WREG should be allowed
@@ -55,7 +55,7 @@* idle) * - MMU page tables area clear (happens on init) * -* QMAN DMA 2-4,6,7, TPC, MME, NIC: +* QMAN DMA 2-7, TPC, MME, NIC: * PQ is secured and is located on the Host (HBM CON TPC3 bug) * CQ, CP and the engine are not secured *
@@ -113,12 +113,12 @@ static const char gaudi_irq_name[GAUDI_MSI_ENTRIES][GAUDI_MAX_STRING_LEN] = {static const u8 gaudi_dma_assignment[GAUDI_DMA_MAX ] = { [GAUDI_PCI_DMA_1 ] =GAUDI_ENGINE_ID_DMA_0 , [GAUDI_PCI_DMA_2 ] =GAUDI_ENGINE_ID_DMA_1 , - [GAUDI_PCI_DMA_3 ] =GAUDI_ENGINE_ID_DMA_5 , [GAUDI_HBM_DMA_1 ] =GAUDI_ENGINE_ID_DMA_2 , [GAUDI_HBM_DMA_2 ] =GAUDI_ENGINE_ID_DMA_3 , [GAUDI_HBM_DMA_3 ] =GAUDI_ENGINE_ID_DMA_4 , - [GAUDI_HBM_DMA_4 ] =GAUDI_ENGINE_ID_DMA_6 , - [GAUDI_HBM_DMA_5 ] =GAUDI_ENGINE_ID_DMA_7 + [GAUDI_HBM_DMA_4 ] =GAUDI_ENGINE_ID_DMA_5 , + [GAUDI_HBM_DMA_5 ] =GAUDI_ENGINE_ID_DMA_6 , + [GAUDI_HBM_DMA_6 ] =GAUDI_ENGINE_ID_DMA_7 };static const u8 gaudi_cq_assignment[NUMBER_OF_CMPLT_QUEUES ] = {
@@ -130,10 +130,6 @@ static const u8 gaudi_cq_assignment[NUMBER_OF_CMPLT_QUEUES] = { [5] = GAUDI_QUEUE_ID_DMA_1_1, [6] = GAUDI_QUEUE_ID_DMA_1_2, [7] = GAUDI_QUEUE_ID_DMA_1_3, - [8] = GAUDI_QUEUE_ID_DMA_5_0, - [9] = GAUDI_QUEUE_ID_DMA_5_1, - [10] = GAUDI_QUEUE_ID_DMA_5_2, - [11 ] =GAUDI_QUEUE_ID_DMA_5_3 };static const u16 gaudi_packet_sizes[MAX_PACKET_ID ] = {
@@ -249,10 +245,10 @@ static enum hl_queue_type gaudi_queue_type[GAUDI_QUEUE_ID_SIZE] = {QUEUE_TYPE_INT ,/* GAUDI_QUEUE_ID_DMA_4_1 */ QUEUE_TYPE_INT ,/* GAUDI_QUEUE_ID_DMA_4_2 */ QUEUE_TYPE_INT ,/* GAUDI_QUEUE_ID_DMA_4_3 */ -QUEUE_TYPE_EXT ,/* GAUDI_QUEUE_ID_DMA_5_0 */ -QUEUE_TYPE_EXT ,/* GAUDI_QUEUE_ID_DMA_5_1 */ -QUEUE_TYPE_EXT ,/* GAUDI_QUEUE_ID_DMA_5_2 */ -QUEUE_TYPE_EXT ,/* GAUDI_QUEUE_ID_DMA_5_3 */ +QUEUE_TYPE_INT ,/* GAUDI_QUEUE_ID_DMA_5_0 */ +QUEUE_TYPE_INT ,/* GAUDI_QUEUE_ID_DMA_5_1 */ +QUEUE_TYPE_INT ,/* GAUDI_QUEUE_ID_DMA_5_2 */ +QUEUE_TYPE_INT ,/* GAUDI_QUEUE_ID_DMA_5_3 */ QUEUE_TYPE_INT ,/* GAUDI_QUEUE_ID_DMA_6_0 */ QUEUE_TYPE_INT ,/* GAUDI_QUEUE_ID_DMA_6_1 */ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_6_2 */
@@ -979,8 +975,7 @@ static int gaudi_alloc_internal_qmans_pq_mem(struct hl_device *hdev) q = &gaudi->internal_qmans[i];switch (i) { - case GAUDI_QUEUE_ID_DMA_2_0 ... GAUDI_QUEUE_ID_DMA_4_3: - case GAUDI_QUEUE_ID_DMA_6_0 ... GAUDI_QUEUE_ID_DMA_7_3: + case GAUDI_QUEUE_ID_DMA_2_0 ... GAUDI_QUEUE_ID_DMA_7_3: q->pq_size =HBM_DMA_QMAN_SIZE_IN_BYTES ;break ; case GAUDI_QUEUE_ID_MME_0_0 ... GAUDI_QUEUE_ID_MME_1_3:
@@ -3433,21 +3428,21 @@ static void gaudi_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi)break ; case GAUDI_QUEUE_ID_DMA_5_0...GAUDI_QUEUE_ID_DMA_5_3: - dma_id = gaudi_dma_assignment[GAUDI_PCI_DMA_3 ]; + dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_4 ]; dma_qm_offset = dma_id *DMA_QMAN_OFFSET ; q_off = dma_qm_offset + ((hw_queue_id -1 ) &0x3 ) *4 ; db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;break ; case GAUDI_QUEUE_ID_DMA_6_0...GAUDI_QUEUE_ID_DMA_6_3: - dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_4 ]; + dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_5 ]; dma_qm_offset = dma_id *DMA_QMAN_OFFSET ; q_off = dma_qm_offset + ((hw_queue_id -1 ) &0x3 ) *4 ; db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;break ; case GAUDI_QUEUE_ID_DMA_7_0...GAUDI_QUEUE_ID_DMA_7_3: - dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_5 ]; + dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_6 ]; dma_qm_offset = dma_id *DMA_QMAN_OFFSET ; q_off = dma_qm_offset + ((hw_queue_id -1 ) &0x3 ) *4 ; db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;
drivers/misc/habanalabs/gaudi/gaudiP.h +4 -4
@@ -15,7 +15,7 @@#include "../include/gaudi/gaudi.h" #include "../include/gaudi/gaudi_async_events.h" -#define NUMBER_OF_EXT_HW_QUEUES 12 +#define NUMBER_OF_EXT_HW_QUEUES 8#define NUMBER_OF_CMPLT_QUEUES NUMBER_OF_EXT_HW_QUEUES#define NUMBER_OF_CPU_HW_QUEUES 1#define NUMBER_OF_INT_HW_QUEUES 100
@@ -62,8 +62,8 @@#error "GAUDI_MAX_PENDING_CS must be power of 2 and greater than 1"#endif -#define PCI_DMA_NUMBER_OF_CHNLS 3 -#define HBM_DMA_NUMBER_OF_CHNLS 5 +#define PCI_DMA_NUMBER_OF_CHNLS 2 +#define HBM_DMA_NUMBER_OF_CHNLS 6#define DMA_NUMBER_OF_CHNLS (PCI_DMA_NUMBER_OF_CHNLS + \ HBM_DMA_NUMBER_OF_CHNLS)
@@ -205,12 +205,12 @@enum gaudi_dma_channels {GAUDI_PCI_DMA_1 ,GAUDI_PCI_DMA_2 , -GAUDI_PCI_DMA_3 ,GAUDI_HBM_DMA_1 ,GAUDI_HBM_DMA_2 ,GAUDI_HBM_DMA_3 ,GAUDI_HBM_DMA_4 ,GAUDI_HBM_DMA_5 , +GAUDI_HBM_DMA_6 ,GAUDI_DMA_MAX };
include/uapi/misc/habanalabs.h +6 -6
@@ -18,8 +18,8 @@#define GOYA_KMD_SRAM_RESERVED_SIZE_FROM_START 0x8000/* 32KB */ #define GAUDI_DRIVER_SRAM_RESERVED_SIZE_FROM_START 0x80/* 128 bytes */ -#define GAUDI_FIRST_AVAILABLE_W_S_SYNC_OBJECT 48 -#define GAUDI_FIRST_AVAILABLE_W_S_MONITOR 24 +#define GAUDI_FIRST_AVAILABLE_W_S_SYNC_OBJECT 32 +#define GAUDI_FIRST_AVAILABLE_W_S_MONITOR 16/* * Goya queue Numbering *
@@ -76,10 +76,10 @@ enum gaudi_queue_id {GAUDI_QUEUE_ID_DMA_4_1 =18 ,/* internal */ GAUDI_QUEUE_ID_DMA_4_2 =19 ,/* internal */ GAUDI_QUEUE_ID_DMA_4_3 =20 ,/* internal */ -GAUDI_QUEUE_ID_DMA_5_0 =21 ,/* external */ -GAUDI_QUEUE_ID_DMA_5_1 =22 ,/* external */ -GAUDI_QUEUE_ID_DMA_5_2 =23 ,/* external */ -GAUDI_QUEUE_ID_DMA_5_3 =24 ,/* external */ +GAUDI_QUEUE_ID_DMA_5_0 =21 ,/* internal */ +GAUDI_QUEUE_ID_DMA_5_1 =22 ,/* internal */ +GAUDI_QUEUE_ID_DMA_5_2 =23 ,/* internal */ +GAUDI_QUEUE_ID_DMA_5_3 =24 ,/* internal */ GAUDI_QUEUE_ID_DMA_6_0 =25 ,/* internal */ GAUDI_QUEUE_ID_DMA_6_1 =26 ,/* internal */ GAUDI_QUEUE_ID_DMA_6_2 = 27, /* internal */
Signature · 1 line
2.17.1