Skip to contentSkip to conversation

fix dma_alloc_coherent on m68knommu / coldfire

PATCH In review Christoph Hellwig 3 participants 8 messages Nov 21, 2022 – Dec 5, 2022

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

Hi all,

this series ensures dma_alloc_coherent returns NULL for m68knommu and coldfire given that these platforms can't actually allocate dma coherent memory after fixing what is according to an old mail from Greg the only driver that actually uses memory returned by dma_alloc_coherent on those platforms.

#1 lore raw

PATCH 1/2 net: fec: use dma_alloc_noncoherent for m532x

The m532x coldfire platforms can't properly implement dma_alloc_coherent and currently just return noncoherent memory from it. The fec driver than works around this with a flush of all caches in the receive path. Make this hack a little less bad by using the explicit dma_alloc_noncoherent API and documenting the hacky cache flushes.

---
 drivers/net/ethernet/freescale/fec_main.c | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
drivers/net/ethernet/freescale/fec_main.c +19 -0
index 28ef4d3c18789..5230698310b5e 100644
@@ -1580,6 +1580,10 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id)
 	struct page *page;
 
 #ifdef CONFIG_M532x
+	/*
+	 * Hacky flush of all caches instead of using the DMA API for the TSO
+	 * headers.
+	 */
 	flush_cache_all();
 #endif
 	rxq = fep->rx_queue[queue_id];
@@ -3123,10 +3127,17 @@ static void fec_enet_free_queue(struct net_device *ndev)
 	for (i = 0; i < fep->num_tx_queues; i++)
 		if (fep->tx_queue[i] && fep->tx_queue[i]->tso_hdrs) {
 			txq = fep->tx_queue[i];
+#ifdef CONFIG_M532x
 			dma_free_coherent(&fep->pdev->dev,
 					  txq->bd.ring_size * TSO_HEADER_SIZE,
 					  txq->tso_hdrs,
 					  txq->tso_hdrs_dma);
+#else
+			dma_free_noncoherent(&fep->pdev->dev,
+					  txq->bd.ring_size * TSO_HEADER_SIZE,
+					  txq->tso_hdrs, txq->tso_hdrs_dma,
+					  DMA_BIDIRECTIONAL);
+#endif
 		}
 
 	for (i = 0; i < fep->num_rx_queues; i++)
@@ -3157,10 +3168,18 @@ static int fec_enet_alloc_queue(struct net_device *ndev)
 		txq->tx_wake_threshold =
 			(txq->bd.ring_size - txq->tx_stop_threshold) / 2;
 
+#ifdef CONFIG_M532x
 		txq->tso_hdrs = dma_alloc_coherent(&fep->pdev->dev,
 					txq->bd.ring_size * TSO_HEADER_SIZE,
 					&txq->tso_hdrs_dma,
 					GFP_KERNEL);
+#else
+		/* m68knommu manually flushes all caches in fec_enet_rx_queue */
+		txq->tso_hdrs = dma_alloc_noncoherent(&fep->pdev->dev,
+					txq->bd.ring_size * TSO_HEADER_SIZE,
+					&txq->tso_hdrs_dma, DMA_BIDIRECTIONAL,
+					GFP_KERNEL);
+#endif
 		if (!txq->tso_hdrs) {
 			ret = -ENOMEM;
 			goto alloc_failed;
Signature · 1 line
2.30.2
#2 lore raw

Re: [PATCH 1/2] net: fec: use dma_alloc_noncoherent for m532x

Hi Christoph,

Quoted 7 lines · Christoph Hellwig

The m532x coldfire platforms can't properly implement dma_alloc_coherent and currently just return noncoherent memory from it. The fec driver than works around this with a flush of all caches in the receive path. Make this hack a little less bad by using the explicit dma_alloc_noncoherent API and documenting the hacky cache flushes.

Thanks for your patch!

Quoted 14 lines
drivers/net/ethernet/freescale/fec_main.c +4 -0
@@ -1580,6 +1580,10 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id)
        struct page *page;
 
 #ifdef CONFIG_M532x
+       /*
+        * Hacky flush of all caches instead of using the DMA API for the TSO
+        * headers.
+        */
        flush_cache_all();
 #endif
        rxq = fep->rx_queue[queue_id];

drivers/net/ethernet/freescale/fec_main.c static void fec_enet_free_queue(struct net_device *ndev) · line 3130

Quoted 4 lines
drivers/net/ethernet/freescale/fec_main.c +1 -0
@@ -3123,10 +3127,17 @@ static void fec_enet_free_queue(struct net_device *ndev)
        for (i = 0; i < fep->num_tx_queues; i++)
                if (fep->tx_queue[i] && fep->tx_queue[i]->tso_hdrs) {
                        txq = fep->tx_queue[i];
+#ifdef CONFIG_M532x

Shouldn't this be the !CONFIG_M532x path?

Quoted 14 lines
drivers/net/ethernet/freescale/fec_main.c +6 -0
                        dma_free_coherent(&fep->pdev->dev,
                                          txq->bd.ring_size * TSO_HEADER_SIZE,
                                          txq->tso_hdrs,
                                          txq->tso_hdrs_dma);
+#else
+                       dma_free_noncoherent(&fep->pdev->dev,
+                                         txq->bd.ring_size * TSO_HEADER_SIZE,
+                                         txq->tso_hdrs, txq->tso_hdrs_dma,
+                                         DMA_BIDIRECTIONAL);
+#endif
                }
 
        for (i = 0; i < fep->num_rx_queues; i++)

drivers/net/ethernet/freescale/fec_main.c static int fec_enet_alloc_queue(struct net_device *ndev) · line 3171

Quoted 4 lines
drivers/net/ethernet/freescale/fec_main.c +1 -0
@@ -3157,10 +3168,18 @@ static int fec_enet_alloc_queue(struct net_device *ndev)
                txq->tx_wake_threshold =
                        (txq->bd.ring_size - txq->tx_stop_threshold) / 2;
 
+#ifdef CONFIG_M532x

Likewise

Quoted 4 lines
drivers/net/ethernet/freescale/fec_main.c +0 -0
                txq->tso_hdrs = dma_alloc_coherent(&fep->pdev->dev,
                                        txq->bd.ring_size * TSO_HEADER_SIZE,
                                        &txq->tso_hdrs_dma,
                                        GFP_KERNEL);

drivers/net/ethernet/freescale/fec_main.c static int fec_enet_alloc_queue(struct net_device *ndev) · line 3185

Quoted 10 lines
drivers/net/ethernet/freescale/fec_main.c +7 -0
+#else
+               /* m68knommu manually flushes all caches in fec_enet_rx_queue */
+               txq->tso_hdrs = dma_alloc_noncoherent(&fep->pdev->dev,
+                                       txq->bd.ring_size * TSO_HEADER_SIZE,
+                                       &txq->tso_hdrs_dma, DMA_BIDIRECTIONAL,
+                                       GFP_KERNEL);
+#endif
                if (!txq->tso_hdrs) {
                        ret = -ENOMEM;
                        goto alloc_failed;

Gr{oetje,eeting}s,

Geert

--
Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- [email protected]

In personal conversations with technical people, I call myself a hacker. But when I'm talking to journalists I just say "programmer" or something like that.
-- Linus Torvalds

#3 lore raw

Re: [PATCH 1/2] net: fec: use dma_alloc_noncoherent for m532x

Quoted 3 lines · Geert Uytterhoeven
Quoted 1 line

+#ifdef CONFIG_M532x

Shouldn't this be the !CONFIG_M532x path?

Yes.

#4 lore raw

Re: [PATCH 1/2] net: fec: use dma_alloc_noncoherent for m532x

In reply to Christoph Hellwig

Hi Christoph,

Quoted 45 lines · Christoph Hellwig

The m532x coldfire platforms can't properly implement dma_alloc_coherent and currently just return noncoherent memory from it. The fec driver than works around this with a flush of all caches in the receive path. Make this hack a little less bad by using the explicit dma_alloc_noncoherent API and documenting the hacky cache flushes.

---
  drivers/net/ethernet/freescale/fec_main.c | 19 +++++++++++++++++++
  1 file changed, 19 insertions(+)
drivers/net/ethernet/freescale/fec_main.c +11 -0
index 28ef4d3c18789..5230698310b5e 100644
@@ -1580,6 +1580,10 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id)
  	struct page *page;
 
  #ifdef CONFIG_M532x
+	/*
+	 * Hacky flush of all caches instead of using the DMA API for the TSO
+	 * headers.
+	 */
  	flush_cache_all();
  #endif
  	rxq = fep->rx_queue[queue_id];
@@ -3123,10 +3127,17 @@ static void fec_enet_free_queue(struct net_device *ndev)
  	for (i = 0; i < fep->num_tx_queues; i++)
  		if (fep->tx_queue[i] && fep->tx_queue[i]->tso_hdrs) {
  			txq = fep->tx_queue[i];
+#ifdef CONFIG_M532x
  			dma_free_coherent(&fep->pdev->dev,
  					  txq->bd.ring_size * TSO_HEADER_SIZE,
  					  txq->tso_hdrs,
  					  txq->tso_hdrs_dma);
+#else
+			dma_free_noncoherent(&fep->pdev->dev,
+					  txq->bd.ring_size * TSO_HEADER_SIZE,
+					  txq->tso_hdrs, txq->tso_hdrs_dma,
+					  DMA_BIDIRECTIONAL);
+#endif
  		}
 
  	for (i = 0; i < fep->num_rx_queues; i++)

drivers/net/ethernet/freescale/fec_main.c static int fec_enet_alloc_queue(struct net_device *ndev) · line 3175

Quoted 8 lines
drivers/net/ethernet/freescale/fec_main.c +1 -0
@@ -3157,10 +3168,18 @@ static int fec_enet_alloc_queue(struct net_device *ndev)
  		txq->tx_wake_threshold =
  			(txq->bd.ring_size - txq->tx_stop_threshold) / 2;
 
+#ifdef CONFIG_M532x
  		txq->tso_hdrs = dma_alloc_coherent(&fep->pdev->dev,
  					txq->bd.ring_size * TSO_HEADER_SIZE,
  					&txq->tso_hdrs_dma,
  					GFP_KERNEL);

Even with this corrected this will now end up failing on all other ColdFire types with the FEC hardware module (all the non-M532x types) once the arch_dma_alloc() returns NULL.

Did you mean "ifndef CONFIG_COLDFIRE" here?

drivers/net/ethernet/freescale/fec_main.c static int fec_enet_alloc_queue(struct net_device *ndev) · line 3185

Quoted 10 lines
drivers/net/ethernet/freescale/fec_main.c +7 -0
+#else
+		/* m68knommu manually flushes all caches in fec_enet_rx_queue */
+		txq->tso_hdrs = dma_alloc_noncoherent(&fep->pdev->dev,
+					txq->bd.ring_size * TSO_HEADER_SIZE,
+					&txq->tso_hdrs_dma, DMA_BIDIRECTIONAL,
+					GFP_KERNEL);
+#endif
  		if (!txq->tso_hdrs) {
  			ret = -ENOMEM;
  			goto alloc_failed;

And what about the dmam_alloc_coherent() call in fec_enet_init()? Does that need changing too?

Regards
Greg

#5 lore raw

Re: [PATCH 1/2] net: fec: use dma_alloc_noncoherent for m532x

Quoted 11 lines · Greg Ungerer

drivers/net/ethernet/freescale/fec_main.c fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id) · line 1587

Quoted 6 lines
    #ifdef CONFIG_M532x
drivers/net/ethernet/freescale/fec_main.c +4 -0
+	/*
+	 * Hacky flush of all caches instead of using the DMA API for the TSO
+	 * headers.
+	 */
  	flush_cache_all();

Even with this corrected this will now end up failing on all other ColdFire types with the FEC hardware module (all the non-M532x types) once the arch_dma_alloc() returns NULL.

Quoted 1 line

Did you mean "ifndef CONFIG_COLDFIRE" here?

How did these work before given that the cache flush is conditional on CONFIG_M532x?

Quoted 5 lines
Quoted 5 lines
drivers/net/ethernet/freescale/fec_main.c +5 -0
+#else
+		/* m68knommu manually flushes all caches in fec_enet_rx_queue */
+		txq->tso_hdrs = dma_alloc_noncoherent(&fep->pdev->dev,
+					txq->bd.ring_size * TSO_HEADER_SIZE,
+					&txq->tso_hdrs_dma, DMA_BIDIRECTIONAL,
Quoted 8 lines

drivers/net/ethernet/freescale/fec_main.c static int fec_enet_alloc_queue(struct net_device *ndev) · line 3185

Quoted 5 lines
drivers/net/ethernet/freescale/fec_main.c +2 -0
+					GFP_KERNEL);
+#endif
  		if (!txq->tso_hdrs) {
  			ret = -ENOMEM;
  			goto alloc_failed;

And what about the dmam_alloc_coherent() call in fec_enet_init()? Does that need changing too?

If that's actually use by the FEC implementations on coldire: yes. But maybe I need even more help on how the cache flushing is suppoѕed to actually work here.

#6 lore raw

Re: [PATCH 1/2] net: fec: use dma_alloc_noncoherent for m532x

Quoted 12 lines · Christoph Hellwig
Quoted 11 lines · Greg Ungerer

drivers/net/ethernet/freescale/fec_main.c fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id) · line 1587

Quoted 6 lines
     #ifdef CONFIG_M532x
drivers/net/ethernet/freescale/fec_main.c +4 -0
+	/*
+	 * Hacky flush of all caches instead of using the DMA API for the TSO
+	 * headers.
+	 */
   	flush_cache_all();

Even with this corrected this will now end up failing on all other ColdFire types with the FEC hardware module (all the non-M532x types) once the arch_dma_alloc() returns NULL.

Quoted 4 lines
Quoted 1 line

Did you mean "ifndef CONFIG_COLDFIRE" here?

How did these work before given that the cache flush is conditional on CONFIG_M532x?

The case for the 5272 is that it only has instruction cache, no data cache. That was the first supported part, and the one I have used the most over the years (though not so much recently). So it should be ok.

I am not convinced about the other version 2 cores either. They all have both instruction and data cache - they can be flexibily configured to be all instruction, or a mix of instruction and data - but the default is both.

I don't have a 532x platform, so I have never done any work on that. The flush_cache_all() for that seems dubious to me.

Quoted 14 lines
Quoted 5 lines
Quoted 5 lines
drivers/net/ethernet/freescale/fec_main.c +5 -0
+#else
+		/* m68knommu manually flushes all caches in fec_enet_rx_queue */
+		txq->tso_hdrs = dma_alloc_noncoherent(&fep->pdev->dev,
+					txq->bd.ring_size * TSO_HEADER_SIZE,
+					&txq->tso_hdrs_dma, DMA_BIDIRECTIONAL,
Quoted 8 lines

drivers/net/ethernet/freescale/fec_main.c static int fec_enet_alloc_queue(struct net_device *ndev) · line 3185

Quoted 5 lines
drivers/net/ethernet/freescale/fec_main.c +2 -0
+					GFP_KERNEL);
+#endif
   		if (!txq->tso_hdrs) {
   			ret = -ENOMEM;
   			goto alloc_failed;

And what about the dmam_alloc_coherent() call in fec_enet_init()? Does that need changing too?

Quoted 1 line

If that's actually use by the FEC implementations on coldire: yes.

It is. Testing these changes failed at boot without changing this one too.

Quoted 2 lines

But maybe I need even more help on how the cache flushing is suppoѕed to actually work here.

Regards
Greg

#7 lore raw

PATCH 2/2 m68k: return NULL from dma_alloc_coherent for nommu or coldfire

In reply to Christoph Hellwig

m68knommu and coldfire platforms can't support allocating coherent DMA memory. Instead of just returning noncoherent memory, just fail the allocation.

---
 arch/m68k/kernel/dma.c | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)
arch/m68k/kernel/dma.c +2 -12
index 2e192a5df949b..63618fce5dc7f 100644
@@ -37,23 +37,13 @@ pgprot_t pgprot_dmacoherent(pgprot_t prot)
 void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
 		gfp_t gfp, unsigned long attrs)
 {
-	void *ret;
-
-	if (dev == NULL || (*dev->dma_mask < 0xffffffff))
-		gfp |= GFP_DMA;
-	ret = (void *)__get_free_pages(gfp, get_order(size));
-
-	if (ret != NULL) {
-		memset(ret, 0, size);
-		*dma_handle = virt_to_phys(ret);
-	}
-	return ret;
+	return NULL;
 }
 
 void arch_dma_free(struct device *dev, size_t size, void *vaddr,
 		dma_addr_t dma_handle, unsigned long attrs)
 {
-	free_pages((unsigned long)vaddr, get_order(size));
+	WARN_ON_ONCE(1);
 }
 
 #endif /* CONFIG_MMU && !CONFIG_COLDFIRE */
Signature · 1 line
2.30.2