dmaengine: dw-edma: Enable HDMA 64R/W Channels
Subsystems (2)
PATCH RESEND v4 1/1 dmaengine: dw-edma: Enable HDMA 64R/W Channels
As per 'Designware Cores PCI Express Controller Databook', Section 7.1 - Overview, HDMA supports 64 Read and 64 Write channels. Current controller driver supports up to 8 read and write channels only. In order to utilize all the channels the controller driver need to have the channel related structs and variables as per the number of channels supported by IP. Following changes are made to enable 64 Read / 64 Write channel support:
o Defined HDMA specific macros to reflect the channel count.
o The count of ll_regions and dt_regions in dw_edma_chip and dw_edma_pcie_data shall be in accordance to number of read and write channels.
o In dw_edma_probe() configure the channels as per the channels of the IP used.
o Changed mask types to u64 for higher channel counts.
--- drivers/dma/dw-edma/dw-edma-core.c | 19 ++++++++++----- drivers/dma/dw-edma/dw-edma-core.h | 4 ++-- drivers/dma/dw-edma/dw-edma-pcie.c | 8 +++---- drivers/dma/dw-edma/dw-hdma-v0-core.c | 33 ++++++++++++++++++++------- drivers/dma/dw-edma/dw-hdma-v0-regs.h | 2 +- include/linux/dma/edma.h | 10 ++++---- 6 files changed, 51 insertions(+), 25 deletions(-)
drivers/dma/dw-edma/dw-edma-core.c +13 -6
@@ -925,9 +925,9 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) irq = &dw->irq[pos];if (chan->dir ==EDMA_DIR_WRITE ) - irq->wr_mask |=BIT (chan->id); + irq->wr_mask |=BIT_ULL (chan->id);else - irq->rd_mask |=BIT (chan->id); + irq->rd_mask |=BIT_ULL (chan->id); irq->dw = dw;memcpy (&chan->msi, &irq->msi,sizeof (chan->msi));
@@ -1079,6 +1079,8 @@ int dw_edma_probe(struct dw_edma_chip *chip)struct dw_edma *dw;u32 wr_alloc =0 ;u32 rd_alloc =0 ; +u16 max_wr_cnt; +u16 max_rd_cnt;int i, err;if (!chip)
@@ -1094,20 +1096,25 @@ int dw_edma_probe(struct dw_edma_chip *chip) dw->chip = chip; -if (dw->chip->mf ==EDMA_MF_HDMA_NATIVE ) +if (dw->chip->mf ==EDMA_MF_HDMA_NATIVE ) {dw_hdma_v0_core_register (dw); -else + max_wr_cnt =HDMA_MAX_WR_CH ; + max_rd_cnt =HDMA_MAX_RD_CH ; + }else {dw_edma_v0_core_register (dw); + max_wr_cnt =EDMA_MAX_WR_CH ; + max_rd_cnt =EDMA_MAX_RD_CH ; + }raw_spin_lock_init (&dw->lock); dw->wr_ch_cnt =min_t (u16, chip->ll_wr_cnt,dw_edma_core_ch_count (dw,EDMA_DIR_WRITE )); - dw->wr_ch_cnt =min_t (u16, dw->wr_ch_cnt,EDMA_MAX_WR_CH ); + dw->wr_ch_cnt =min_t (u16, dw->wr_ch_cnt, max_wr_cnt); dw->rd_ch_cnt =min_t (u16, chip->ll_rd_cnt,dw_edma_core_ch_count (dw,EDMA_DIR_READ )); - dw->rd_ch_cnt =min_t (u16, dw->rd_ch_cnt,EDMA_MAX_RD_CH ); + dw->rd_ch_cnt =min_t (u16, dw->rd_ch_cnt, max_rd_cnt);if (!dw->wr_ch_cnt && !dw->rd_ch_cnt)return -EINVAL ;
drivers/dma/dw-edma/dw-edma-core.h +2 -2
@@ -91,8 +91,8 @@ struct dw_edma_chan {struct dw_edma_irq {struct msi_msg msi; -u32 wr_mask; -u32 rd_mask; +u64 wr_mask; +u64 rd_mask;struct dw_edma *dw; };
drivers/dma/dw-edma/dw-edma-pcie.c +4 -4
@@ -61,11 +61,11 @@ struct dw_edma_pcie_data {/* eDMA registers location */ struct dw_edma_block rg;/* eDMA memory linked list location */ -struct dw_edma_block ll_wr[EDMA_MAX_WR_CH ]; -struct dw_edma_block ll_rd[EDMA_MAX_RD_CH ]; +struct dw_edma_block ll_wr[HDMA_MAX_WR_CH ]; +struct dw_edma_block ll_rd[HDMA_MAX_RD_CH ];/* eDMA memory data location */ -struct dw_edma_block dt_wr[EDMA_MAX_WR_CH ]; -struct dw_edma_block dt_rd[EDMA_MAX_RD_CH ]; +struct dw_edma_block dt_wr[HDMA_MAX_WR_CH ]; +struct dw_edma_block dt_rd[HDMA_MAX_RD_CH ];/* Other */ enum dw_edma_map_format mf;u8 irqs;
drivers/dma/dw-edma/dw-hdma-v0-core.c +25 -8
@@ -53,13 +53,24 @@ __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch)static void dw_hdma_v0_core_off (struct dw_edma *dw) {int id; +enum dw_edma_dir dir; + + dir =EDMA_DIR_WRITE ; +for (id =0 ; id < dw->wr_ch_cnt; id++) { +SET_CH_32 (dw, dir, id, int_setup, +HDMA_V0_STOP_INT_MASK |HDMA_V0_ABORT_INT_MASK ); +SET_CH_32 (dw, dir, id, int_clear, +HDMA_V0_STOP_INT_MASK |HDMA_V0_ABORT_INT_MASK ); +SET_CH_32 (dw, dir, id, ch_en,0 ); + } -for (id =0 ; id <HDMA_V0_MAX_NR_CH ; id++) { -SET_BOTH_CH_32 (dw, id, int_setup, -HDMA_V0_STOP_INT_MASK |HDMA_V0_ABORT_INT_MASK ); -SET_BOTH_CH_32 (dw, id, int_clear, -HDMA_V0_STOP_INT_MASK |HDMA_V0_ABORT_INT_MASK ); -SET_BOTH_CH_32 (dw, id, ch_en,0 ); + dir =EDMA_DIR_READ ; +for (id =0 ; id < dw->rd_ch_cnt; id++) { +SET_CH_32 (dw, dir, id, int_setup, +HDMA_V0_STOP_INT_MASK |HDMA_V0_ABORT_INT_MASK ); +SET_CH_32 (dw, dir, id, int_clear, +HDMA_V0_STOP_INT_MASK |HDMA_V0_ABORT_INT_MASK ); +SET_CH_32 (dw, dir, id, ch_en,0 ); } }
@@ -118,7 +129,8 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,unsigned long total, pos, val;irqreturn_t ret =IRQ_NONE ;struct dw_edma_chan *chan; -unsigned long off, mask; +unsigned long off; +u64 mask;if (dir ==EDMA_DIR_WRITE ) { total = dw->wr_ch_cnt;
@@ -130,7 +142,11 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, mask = dw_irq->rd_mask; } -for_each_set_bit (pos, &mask, total) { +while (mask) { + pos =__ffs64 (mask); +if (pos >= total) +break ; + chan = &dw->chan[pos + off]; val =dw_hdma_v0_core_status_int (chan);
@@ -147,6 +163,7 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, ret =IRQ_HANDLED ; } + mask &= mask -1 ; }return ret;
drivers/dma/dw-edma/dw-hdma-v0-regs.h +1 -1
@@ -11,7 +11,7 @@#include <linux/dmaengine.h> -#define HDMA_V0_MAX_NR_CH 8 +#define HDMA_V0_MAX_NR_CH 64#define HDMA_V0_CH_EN BIT(0)#define HDMA_V0_LOCAL_ABORT_INT_EN BIT(6)#define HDMA_V0_REMOTE_ABORT_INT_EN BIT(5)
include/linux/dma/edma.h +6 -4
@@ -14,6 +14,8 @@#define EDMA_MAX_WR_CH 8#define EDMA_MAX_RD_CH 8 +#define HDMA_MAX_WR_CH 64 +#define HDMA_MAX_RD_CH 64struct dw_edma ;
@@ -89,12 +91,12 @@ struct dw_edma_chip {u16 ll_wr_cnt;u16 ll_rd_cnt;/* link list address */ -struct dw_edma_region ll_region_wr[EDMA_MAX_WR_CH ]; -struct dw_edma_region ll_region_rd[EDMA_MAX_RD_CH ]; +struct dw_edma_region ll_region_wr[HDMA_MAX_WR_CH ]; +struct dw_edma_region ll_region_rd[HDMA_MAX_RD_CH ];/* data region */ -struct dw_edma_region dt_region_wr[EDMA_MAX_WR_CH ]; -struct dw_edma_region dt_region_rd[EDMA_MAX_RD_CH ]; +struct dw_edma_region dt_region_wr[HDMA_MAX_WR_CH ]; +struct dw_edma_region dt_region_rd[HDMA_MAX_RD_CH ];/* interrupt emulation */ int db_irq;
Signature · 1 line
2.43.0
Please ignore this resend. Details related to version change got missed.
regards,
Devendra
Quoted 226 lines · Devendra K Verma
As per 'Designware Cores PCI Express Controller Databook', Section 7.1 - Overview, HDMA supports 64 Read and 64 Write channels. Current controller driver supports up to 8 read and write channels only. In order to utilize all the channels the controller driver need to have the channel related structs and variables as per the number of channels supported by IP. Following changes are made to enable 64 Read / 64 Write channel support:
o Defined HDMA specific macros to reflect the channel count.
o The count ofll_regionsanddt_regionsindw_edma_chipanddw_edma_pcie_datashall be in accordance to number of read and write channels.
o Indw_edma_probe()configure the channels as per the channels of the IP used.
o Changed mask types to u64 for higher channel counts.Signed-off-by Devendra K Verma <[email protected]>--- drivers/dma/dw-edma/dw-edma-core.c | 19 ++++++++++----- drivers/dma/dw-edma/dw-edma-core.h | 4 ++-- drivers/dma/dw-edma/dw-edma-pcie.c | 8 +++---- drivers/dma/dw-edma/dw-hdma-v0-core.c | 33 ++++++++++++++++++++------- drivers/dma/dw-edma/dw-hdma-v0-regs.h | 2 +- include/linux/dma/edma.h | 10 ++++---- 6 files changed, 51 insertions(+), 25 deletions(-)drivers/dma/dw-edma/dw-edma-core.c +13
-6@@ -925,9 +925,9 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) irq = &dw->irq[pos];if (chan->dir ==EDMA_DIR_WRITE ) - irq->wr_mask |=BIT (chan->id); + irq->wr_mask |=BIT_ULL (chan->id);else - irq->rd_mask |=BIT (chan->id); + irq->rd_mask |=BIT_ULL (chan->id); irq->dw = dw;memcpy (&chan->msi, &irq->msi,sizeof (chan->msi));@@ -1079,6 +1079,8 @@ int dw_edma_probe(struct dw_edma_chip *chip)struct dw_edma *dw;u32 wr_alloc =0 ;u32 rd_alloc =0 ; +u16 max_wr_cnt; +u16 max_rd_cnt;int i, err;if (!chip)@@ -1094,20 +1096,25 @@ int dw_edma_probe(struct dw_edma_chip *chip) dw->chip = chip; -if (dw->chip->mf ==EDMA_MF_HDMA_NATIVE ) +if (dw->chip->mf ==EDMA_MF_HDMA_NATIVE ) {dw_hdma_v0_core_register (dw); -else + max_wr_cnt =HDMA_MAX_WR_CH ; + max_rd_cnt =HDMA_MAX_RD_CH ; + }else {dw_edma_v0_core_register (dw); + max_wr_cnt =EDMA_MAX_WR_CH ; + max_rd_cnt =EDMA_MAX_RD_CH ; + }raw_spin_lock_init (&dw->lock); dw->wr_ch_cnt =min_t (u16, chip->ll_wr_cnt,dw_edma_core_ch_count (dw,EDMA_DIR_WRITE )); - dw->wr_ch_cnt =min_t (u16, dw->wr_ch_cnt,EDMA_MAX_WR_CH ); + dw->wr_ch_cnt =min_t (u16, dw->wr_ch_cnt, max_wr_cnt); dw->rd_ch_cnt =min_t (u16, chip->ll_rd_cnt,dw_edma_core_ch_count (dw,EDMA_DIR_READ )); - dw->rd_ch_cnt =min_t (u16, dw->rd_ch_cnt,EDMA_MAX_RD_CH ); + dw->rd_ch_cnt =min_t (u16, dw->rd_ch_cnt, max_rd_cnt);if (!dw->wr_ch_cnt && !dw->rd_ch_cnt)return -EINVAL ;drivers/dma/dw-edma/dw-edma-core.h +2
-2@@ -91,8 +91,8 @@ struct dw_edma_chan {struct dw_edma_irq {struct msi_msg msi; -u32 wr_mask; -u32 rd_mask; +u64 wr_mask; +u64 rd_mask;struct dw_edma *dw; };drivers/dma/dw-edma/dw-edma-pcie.c +4
-4@@ -61,11 +61,11 @@ struct dw_edma_pcie_data {/* eDMA registers location */ struct dw_edma_block rg;/* eDMA memory linked list location */ -struct dw_edma_block ll_wr[EDMA_MAX_WR_CH ]; -struct dw_edma_block ll_rd[EDMA_MAX_RD_CH ]; +struct dw_edma_block ll_wr[HDMA_MAX_WR_CH ]; +struct dw_edma_block ll_rd[HDMA_MAX_RD_CH ];/* eDMA memory data location */ -struct dw_edma_block dt_wr[EDMA_MAX_WR_CH ]; -struct dw_edma_block dt_rd[EDMA_MAX_RD_CH ]; +struct dw_edma_block dt_wr[HDMA_MAX_WR_CH ]; +struct dw_edma_block dt_rd[HDMA_MAX_RD_CH ];/* Other */ enum dw_edma_map_format mf;u8 irqs;drivers/dma/dw-edma/dw-hdma-v0-core.c +25
-8@@ -53,13 +53,24 @@ __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch)static void dw_hdma_v0_core_off (struct dw_edma *dw) {int id; +enum dw_edma_dir dir; + + dir =EDMA_DIR_WRITE ; +for (id =0 ; id < dw->wr_ch_cnt; id++) { +SET_CH_32 (dw, dir, id, int_setup, +HDMA_V0_STOP_INT_MASK |HDMA_V0_ABORT_INT_MASK ); +SET_CH_32 (dw, dir, id, int_clear, +HDMA_V0_STOP_INT_MASK |HDMA_V0_ABORT_INT_MASK ); +SET_CH_32 (dw, dir, id, ch_en,0 ); + } -for (id =0 ; id <HDMA_V0_MAX_NR_CH ; id++) { -SET_BOTH_CH_32 (dw, id, int_setup, -HDMA_V0_STOP_INT_MASK |HDMA_V0_ABORT_INT_MASK ); -SET_BOTH_CH_32 (dw, id, int_clear, -HDMA_V0_STOP_INT_MASK |HDMA_V0_ABORT_INT_MASK ); -SET_BOTH_CH_32 (dw, id, ch_en,0 ); + dir =EDMA_DIR_READ ; +for (id =0 ; id < dw->rd_ch_cnt; id++) { +SET_CH_32 (dw, dir, id, int_setup, +HDMA_V0_STOP_INT_MASK |HDMA_V0_ABORT_INT_MASK ); +SET_CH_32 (dw, dir, id, int_clear, +HDMA_V0_STOP_INT_MASK |HDMA_V0_ABORT_INT_MASK ); +SET_CH_32 (dw, dir, id, ch_en,0 ); } }@@ -118,7 +129,8 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,unsigned long total, pos, val;irqreturn_t ret =IRQ_NONE ;struct dw_edma_chan *chan; -unsigned long off, mask; +unsigned long off; +u64 mask;if (dir ==EDMA_DIR_WRITE ) { total = dw->wr_ch_cnt;@@ -130,7 +142,11 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, mask = dw_irq->rd_mask; } -for_each_set_bit (pos, &mask, total) { +while (mask) { + pos =__ffs64 (mask); +if (pos >= total) +break ; + chan = &dw->chan[pos + off]; val =dw_hdma_v0_core_status_int (chan);@@ -147,6 +163,7 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, ret =IRQ_HANDLED ; } + mask &= mask -1 ; }return ret;drivers/dma/dw-edma/dw-hdma-v0-regs.h +1
-1@@ -11,7 +11,7 @@#include <linux/dmaengine.h> -#define HDMA_V0_MAX_NR_CH 8 +#define HDMA_V0_MAX_NR_CH 64#define HDMA_V0_CH_EN BIT(0)#define HDMA_V0_LOCAL_ABORT_INT_EN BIT(6)#define HDMA_V0_REMOTE_ABORT_INT_EN BIT(5)include/linux/dma/edma.h +6
-4@@ -14,6 +14,8 @@#define EDMA_MAX_WR_CH 8#define EDMA_MAX_RD_CH 8 +#define HDMA_MAX_WR_CH 64 +#define HDMA_MAX_RD_CH 64struct dw_edma ;@@ -89,12 +91,12 @@ struct dw_edma_chip {u16 ll_wr_cnt;u16 ll_rd_cnt;/* link list address */ -struct dw_edma_region ll_region_wr[EDMA_MAX_WR_CH ]; -struct dw_edma_region ll_region_rd[EDMA_MAX_RD_CH ]; +struct dw_edma_region ll_region_wr[HDMA_MAX_WR_CH ]; +struct dw_edma_region ll_region_rd[HDMA_MAX_RD_CH ];/* data region */ -struct dw_edma_region dt_region_wr[EDMA_MAX_WR_CH ]; -struct dw_edma_region dt_region_rd[EDMA_MAX_RD_CH ]; +struct dw_edma_region dt_region_wr[HDMA_MAX_WR_CH ]; +struct dw_edma_region dt_region_rd[HDMA_MAX_RD_CH ];/* interrupt emulation */ int db_irq;
[email protected]
AI reviewThank you for your contribution! Sashiko AI review found 3 potential issue(s) to consider:
Thank you for your contribution! Sashiko AI review found 3 potential issue(s) to consider:
New issues:
- [High] Incomplete hardware reset during initialization leaves unallocated active channels unmasked and uncleared, leading to an IRQ storm.
- [High] The patch claims to enable 64 channels but fails to update integration layers, leaving hardcoded 8-channel limits that prevent utilization and break probing.
Pre-existing issues:
- [High] The GET_CH_32 macro is called with swapped dir and ch arguments in dw_hdma_v0_core_ch_status(), reading the wrong register. --
commit 973288f2abf5c733055723f7bd6942953badd2e0
Author: Devendra K Verma <[email protected]>
dmaengine: dw-edma: Enable HDMA 64R/W Channels
This patch updates the dw-edma driver to support 64 read and write channels for HDMA. It defines new macros for the maximum channel counts, scales up internal data structures, and changes channel masks to 64-bit types to accommodate the increased capacity.
Quoted 10 lines
drivers/dma/dw-edma/dw-edma-pcie.c +0
-2@@ -61,11 +61,11 @@ struct dw_edma_pcie_data {/* eDMA registers location */ struct dw_edma_block rg;/* eDMA memory linked list location */ -struct dw_edma_block ll_wr[EDMA_MAX_WR_CH ]; -struct dw_edma_block ll_rd[EDMA_MAX_RD_CH ];
drivers/dma/dw-edma/dw-edma-pcie.c struct dw_edma_pcie_data { · line 71
Quoted 10 lines
drivers/dma/dw-edma/dw-edma-pcie.c +4
-2+struct dw_edma_block ll_wr[HDMA_MAX_WR_CH ]; +struct dw_edma_block ll_rd[HDMA_MAX_RD_CH ];/* eDMA memory data location */ -struct dw_edma_block dt_wr[EDMA_MAX_WR_CH ]; -struct dw_edma_block dt_rd[EDMA_MAX_RD_CH ]; +struct dw_edma_block dt_wr[HDMA_MAX_WR_CH ]; +struct dw_edma_block dt_rd[HDMA_MAX_RD_CH ];/* Other */ enum dw_edma_map_format mf;u8 irqs;
[Severity: High]
Does this successfully enable 64 channels if the integration layers still enforce an 8-channel limit?
For example, in drivers/pci/controller/dwc/pcie-designware.c dw_pcie_edma_find_channels() checks against EDMA_MAX_WR_CH:
if (!pci->edma.ll_wr_cnt || pci->edma.ll_wr_cnt > EDMA_MAX_WR_CH ||
!pci->edma.ll_rd_cnt || pci->edma.ll_rd_cnt > EDMA_MAX_RD_CH)
return -EINVAL;And in drivers/dma/dw-edma/dw-edma-pcie.c dw_edma_pcie_get_synopsys_dma_data() clamps capabilities to defaults:
pdata->wr_ch_cnt = min_t(u16, pdata->wr_ch_cnt,
FIELD_GET(DW_PCIE_SYNOPSYS_VSEC_DMA_WR_CH, val));If a device actually reports 64 channels, won't probe fail immediately because the PCIe wrapper bounds checks against EDMA_MAX_WR_CH (8) instead of HDMA_MAX_WR_CH (64)?
Quoted 8 lines
drivers/dma/dw-edma/dw-hdma-v0-core.c +0
-0@@ -53,13 +53,24 @@ __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch)static void dw_hdma_v0_core_off (struct dw_edma *dw) {int id;
drivers/dma/dw-edma/dw-hdma-v0-core.c __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch) · line 65
Quoted 10 lines
drivers/dma/dw-edma/dw-hdma-v0-core.c +10
-0+enum dw_edma_dir dir; + + dir =EDMA_DIR_WRITE ; +for (id =0 ; id < dw->wr_ch_cnt; id++) { +SET_CH_32 (dw, dir, id, int_setup, +HDMA_V0_STOP_INT_MASK |HDMA_V0_ABORT_INT_MASK ); +SET_CH_32 (dw, dir, id, int_clear, +HDMA_V0_STOP_INT_MASK |HDMA_V0_ABORT_INT_MASK ); +SET_CH_32 (dw, dir, id, ch_en,0 ); + }
[Severity: High]
Can changing this loop in dw_hdma_v0_core_off() from a fixed hardware maximum to the software-allocated count dw->wr_ch_cnt lead to an IRQ storm?
If a bootloader or previous kernel instance leaves channels greater than dw->wr_ch_cnt active, won't they remain unmasked and unhandled?
Because dw_hdma_v0_core_handle_int() bounds the interrupt checking mask:
while (mask) {
pos = __ffs64(mask);
if (pos >= total)
break;Any unallocated active channel might repeatedly raise hardware interrupts without ever being cleared.
[Severity: High]
This is a pre-existing issue, but in dw_hdma_v0_core_ch_status() in drivers/dma/dw-edma/dw-hdma-v0-core.c, does the GET_CH_32 macro have the arguments swapped?
tmp = FIELD_GET(HDMA_V0_CH_STATUS_MASK,
GET_CH_32(dw, chan->id, chan->dir, ch_stat));
The GET_CH_32 macro expects (dw, dir, ch, name). If chan->id is passed as dir, won't __dw_ch_regs() evaluate dir as non-write (0) for any channel ID greater than 0, causing it to always read from channel 0 or 1's read registers?
[ ... ]
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=1