mtd: rawnand: cadence: support 64-bit slave dma interface

index 9dac3ca69d57..0309a6b0d3d3 100644
@@ -1869,6 +1869,46 @@ static int cadence_nand_slave_dma_transfer(struct cdns_nand_ctrl *cdns_ctrl,
 	return -EIO;
 }
 
+static inline void cadence_nand_readsq(struct cdns_nand_ctrl *cdns_ctrl,
+				       void *buffer, unsigned int count)
+{
+#ifdef __raw_readq
+	const volatile void __iomem *addr = cdns_ctrl->io.virt;
+
+        if (count) {
+                u64 *buf = buffer;
+
+                do {
+                        u64 x = __raw_readq(addr);
+                        *buf++ = x;
+                } while (--count);
+        }
+#else
+	dev_err(cdns_ctrl->dev,
+		"cannot read 64-bit sdma on !64-bit architectures");
+#endif
+}
+
+static inline void cadence_nand_writesq(struct cdns_nand_ctrl *cdns_ctrl,
+					const void *buffer,
+					unsigned int count)
+{
+#ifdef __raw_writeq
+	volatile void __iomem *addr = cdns_ctrl->io.virt;
+
+        if (count) {
+                const u64 *buf = buffer;
+
+                do {
+                        __raw_writeq(*buf++, addr);
+                } while (--count);
+        }
+#else
+	dev_err(cdns_ctrl->dev,
+		"cannot write 64-bit sdma on !64-bit architectures");
+#endif
+}
+
 static int cadence_nand_read_buf(struct cdns_nand_ctrl *cdns_ctrl,
 				 u8 *buf, int len)
 {

@@ -1882,17 +1922,33 @@ static int cadence_nand_read_buf(struct cdns_nand_ctrl *cdns_ctrl,
 		return status;
 
 	if (!cdns_ctrl->caps1->has_dma) {
-		int len_in_words = len >> 2;
+		u8 data_dma_width = cdns_ctrl->caps2.data_dma_width;
+
+		int len_in_words = (data_dma_width == 4) ? len >> 2 : len >> 3;
 
 		/* read alingment data */
-		ioread32_rep(cdns_ctrl->io.virt, buf, len_in_words);
+		if (data_dma_width == 4)
+			ioread32_rep(cdns_ctrl->io.virt, buf, len_in_words);
+		else
+			cadence_nand_readsq(cdns_ctrl, buf, len_in_words);
+
 		if (sdma_size > len) {
+			int read_bytes = (data_dma_width == 4) ?
+				len_in_words << 2 : len_in_words << 3;
+
 			/* read rest data from slave DMA interface if any */
-			ioread32_rep(cdns_ctrl->io.virt, cdns_ctrl->buf,
-				     sdma_size / 4 - len_in_words);
+			if (data_dma_width == 4)
+				ioread32_rep(cdns_ctrl->io.virt,
+					     cdns_ctrl->buf,
+					     sdma_size / 4 - len_in_words);
+			else
+				cadence_nand_readsq(cdns_ctrl,
+					     cdns_ctrl->buf,
+					     sdma_size / 8 - len_in_words);
+
 			/* copy rest of data */
-			memcpy(buf + (len_in_words << 2), cdns_ctrl->buf,
-			       len - (len_in_words << 2));
+			memcpy(buf + read_bytes, cdns_ctrl->buf,
+			       len - read_bytes);
 		}
 		return 0;
 	}

@@ -1936,16 +1992,32 @@ static int cadence_nand_write_buf(struct cdns_nand_ctrl *cdns_ctrl,
 		return status;
 
 	if (!cdns_ctrl->caps1->has_dma) {
-		int len_in_words = len >> 2;
+		u8 data_dma_width = cdns_ctrl->caps2.data_dma_width;
+
+		int len_in_words = (data_dma_width == 4) ? len >> 2 : len >> 3;
+
+		if (data_dma_width == 4)
+			iowrite32_rep(cdns_ctrl->io.virt, buf, len_in_words);
+		else
+			cadence_nand_writesq(cdns_ctrl, buf, len_in_words);
 
-		iowrite32_rep(cdns_ctrl->io.virt, buf, len_in_words);
 		if (sdma_size > len) {
+			int written_bytes = (data_dma_width == 4) ?
+				len_in_words << 2 : len_in_words << 3;
+
 			/* copy rest of data */
-			memcpy(cdns_ctrl->buf, buf + (len_in_words << 2),
-			       len - (len_in_words << 2));
+			memcpy(cdns_ctrl->buf, buf + written_bytes,
+			       len - written_bytes);
+
 			/* write all expected by nand controller data */
-			iowrite32_rep(cdns_ctrl->io.virt, cdns_ctrl->buf,
-				      sdma_size / 4 - len_in_words);
+			if (data_dma_width == 4)
+				iowrite32_rep(cdns_ctrl->io.virt,
+					      cdns_ctrl->buf,
+					      sdma_size / 4 - len_in_words);
+			else
+				cadence_nand_writesq(cdns_ctrl,
+					      cdns_ctrl->buf,
+					      sdma_size / 8 - len_in_words);
 		}
 
 		return 0;