ARM926EJS: Implement cache operations

index 504f604..5b23e3a 100644
@@ -23,29 +23,71 @@
 #include <common.h>
 
 #ifndef CONFIG_SYS_DCACHE_OFF
-static inline void dcache_noop(void)
+
+#ifndef CONFIG_SYS_CACHELINE_SIZE
+#define CONFIG_SYS_CACHELINE_SIZE	32
+#endif
+
+void invalidate_dcache_all(void)
 {
-	if (dcache_status()) {
-		puts("WARNING: cache operations are not implemented!\n"
-		     "WARNING: disabling D-Cache now, you can re-enable it"
-		     "later with 'dcache on' command\n");
-		dcache_disable();
-	}
+	asm volatile("mcr p15, 0, %0, c7, c6, 0\n"::"r"(0));
 }
 
-void invalidate_dcache_all(void)
+void flush_dcache_all(void)
 {
-	dcache_noop();
+	asm volatile(
+		"0:"
+		"mrc p15, 0, r15, c7, c14, 3\n"
+		"bne 0b\n"
+		"mcr p15, 0, %0, c7, c10, 4\n"
+		::"r"(0):"memory"
+	);
+}
+
+static int check_cache_range(unsigned long start, unsigned long stop)
+{
+	int ok = 1;
+
+	if (start & (CONFIG_SYS_CACHELINE_SIZE - 1))
+		ok = 0;
+
+	if (stop & (CONFIG_SYS_CACHELINE_SIZE - 1))
+		ok = 0;
+
+	if (!ok)
+		printf("CACHE: Misaligned operation at range [%08lx, %08lx]\n",
+			start, stop);
+
+	return ok;
 }
 
 void invalidate_dcache_range(unsigned long start, unsigned long stop)
 {
-	dcache_noop();
+	if (!check_cache_range(start, stop))
+		return;
+
+	while (start < stop) {
+		asm volatile("mcr p15, 0, %0, c7, c6, 1\n"::"r"(start));
+		start += CONFIG_SYS_CACHELINE_SIZE;
+	}
 }
 
 void flush_dcache_range(unsigned long start, unsigned long stop)
 {
-	dcache_noop();
+	if (!check_cache_range(start, stop))
+		return;
+
+	while (start < stop) {
+		asm volatile("mcr p15, 0, %0, c7, c14, 1\n"::"r"(start));
+		start += CONFIG_SYS_CACHELINE_SIZE;
+	}
+
+	asm("mcr p15, 0, %0, c7, c10, 4\n"::"r"(0));
+}
+
+void flush_cache(unsigned long start, unsigned long size)
+{
+	flush_dcache_range(start, start + size);
 }
 #else /* #ifndef CONFIG_SYS_DCACHE_OFF */
 void invalidate_dcache_all(void)

@@ -64,7 +106,7 @@ void flush_dcache_range(unsigned long start, unsigned long stop)
 {
 }
 
-void  flush_cache(unsigned long start, unsigned long size)
+void flush_cache(unsigned long start, unsigned long size)
 {
 }
 #endif /* #ifndef CONFIG_SYS_DCACHE_OFF */

index 9bfd83b..cf6d4e9 100644
@@ -63,6 +63,16 @@ void reset_cpu(ulong ignored)
 		;
 }
 
+void enable_caches(void)
+{
+#ifndef CONFIG_SYS_ICACHE_OFF
+	icache_enable();
+#endif
+#ifndef CONFIG_SYS_DCACHE_OFF
+	dcache_enable();
+#endif
+}
+
 int mx28_wait_mask_set(struct mx28_register_32 *reg, uint32_t mask, int timeout)
 {
 	while (--timeout) {

index e85f5fe..c086629 100644
@@ -93,6 +93,21 @@ static int mxs_dma_read_semaphore(int channel)
 	return tmp;
 }
 
+#ifndef	CONFIG_SYS_DCACHE_OFF
+void mxs_dma_flush_desc(struct mxs_dma_desc *desc)
+{
+	uint32_t addr;
+	uint32_t size;
+
+	addr = (uint32_t)desc;
+	size = roundup(sizeof(struct mxs_dma_desc), MXS_DMA_ALIGNMENT);
+
+	flush_dcache_range(addr, addr + size);
+}
+#else
+inline void mxs_dma_flush_desc(struct mxs_dma_desc *desc) {}
+#endif
+
 /*
  * Enable a DMA channel.
  *

@@ -329,8 +344,10 @@ static int mxs_dma_release(int channel)
 struct mxs_dma_desc *mxs_dma_desc_alloc(void)
 {
 	struct mxs_dma_desc *pdesc;
+	uint32_t size;
 
-	pdesc = memalign(MXS_DMA_ALIGNMENT, sizeof(struct mxs_dma_desc));
+	size = roundup(sizeof(struct mxs_dma_desc), MXS_DMA_ALIGNMENT);
+	pdesc = memalign(MXS_DMA_ALIGNMENT, size);
 
 	if (pdesc == NULL)
 		return NULL;

@@ -415,12 +432,16 @@ int mxs_dma_desc_append(int channel, struct mxs_dma_desc *pdesc)
 
 		last->cmd.next = mxs_dma_cmd_address(pdesc);
 		last->cmd.data |= MXS_DMA_DESC_CHAIN;
+
+		mxs_dma_flush_desc(last);
 	}
 	pdesc->flags |= MXS_DMA_DESC_READY;
 	if (pdesc->flags & MXS_DMA_DESC_FIRST)
 		pchan->pending_num++;
 	list_add_tail(&pdesc->node, &pchan->active);
 
+	mxs_dma_flush_desc(pdesc);
+
 	return ret;
 }
 

index ce2a326..4b1297a 100644
@@ -50,6 +50,7 @@ struct mxs_nand_info {
 	int		cur_chip;
 
 	uint32_t	cmd_queue_len;
+	uint32_t	data_buf_size;
 
 	uint8_t		*cmd_buf;
 	uint8_t		*data_buf;

@@ -73,6 +74,36 @@ struct mxs_nand_info {
 
 struct nand_ecclayout fake_ecc_layout;
 
+/*
+ * Cache management functions
+ */
+#ifndef	CONFIG_SYS_DCACHE_OFF
+static void mxs_nand_flush_data_buf(struct mxs_nand_info *info)
+{
+	uint32_t addr = (uint32_t)info->data_buf;
+
+	flush_dcache_range(addr, addr + info->data_buf_size);
+}
+
+static void mxs_nand_inval_data_buf(struct mxs_nand_info *info)
+{
+	uint32_t addr = (uint32_t)info->data_buf;
+
+	invalidate_dcache_range(addr, addr + info->data_buf_size);
+}
+
+static void mxs_nand_flush_cmd_buf(struct mxs_nand_info *info)
+{
+	uint32_t addr = (uint32_t)info->cmd_buf;
+
+	flush_dcache_range(addr, addr + MXS_NAND_COMMAND_BUFFER_SIZE);
+}
+#else
+static inline void mxs_nand_flush_data_buf(struct mxs_nand_info *info) {}
+static inline void mxs_nand_inval_data_buf(struct mxs_nand_info *info) {}
+static inline void mxs_nand_flush_cmd_buf(struct mxs_nand_info *info) {}
+#endif
+
 static struct mxs_dma_desc *mxs_nand_get_dma_desc(struct mxs_nand_info *info)
 {
 	struct mxs_dma_desc *desc;

@@ -286,6 +317,9 @@ static void mxs_nand_cmd_ctrl(struct mtd_info *mtd, int data, unsigned int ctrl)
 
 	mxs_dma_desc_append(channel, d);
 
+	/* Flush caches */
+	mxs_nand_flush_cmd_buf(nand_info);
+
 	/* Execute the DMA chain. */
 	ret = mxs_dma_go(channel);
 	if (ret)

@@ -435,6 +469,9 @@ static void mxs_nand_read_buf(struct mtd_info *mtd, uint8_t *buf, int length)
 		goto rtn;
 	}
 
+	/* Invalidate caches */
+	mxs_nand_inval_data_buf(nand_info);
+
 	memcpy(buf, nand_info->data_buf, length);
 
 rtn:

@@ -484,6 +521,9 @@ static void mxs_nand_write_buf(struct mtd_info *mtd, const uint8_t *buf,
 
 	mxs_dma_desc_append(channel, d);
 
+	/* Flush caches */
+	mxs_nand_flush_data_buf(nand_info);
+
 	/* Execute the DMA chain. */
 	ret = mxs_dma_go(channel);
 	if (ret)

@@ -600,6 +640,9 @@ static int mxs_nand_ecc_read_page(struct mtd_info *mtd, struct nand_chip *nand,
 		goto rtn;
 	}
 
+	/* Invalidate caches */
+	mxs_nand_inval_data_buf(nand_info);
+
 	/* Read DMA completed, now do the mark swapping. */
 	mxs_nand_swap_block_mark(mtd, nand_info->data_buf, nand_info->oob_buf);
 

@@ -687,6 +730,9 @@ static void mxs_nand_ecc_write_page(struct mtd_info *mtd,
 
 	mxs_dma_desc_append(channel, d);
 
+	/* Flush caches */
+	mxs_nand_flush_data_buf(nand_info);
+
 	/* Execute the DMA chain. */
 	ret = mxs_dma_go(channel);
 	if (ret) {

@@ -978,18 +1024,19 @@ int mxs_nand_alloc_buffers(struct mxs_nand_info *nand_info)
 	uint8_t *buf;
 	const int size = NAND_MAX_PAGESIZE + NAND_MAX_OOBSIZE;
 
+	nand_info->data_buf_size = roundup(size, MXS_DMA_ALIGNMENT);
+
 	/* DMA buffers */
-	buf = memalign(MXS_DMA_ALIGNMENT, size);
+	buf = memalign(MXS_DMA_ALIGNMENT, nand_info->data_buf_size);
 	if (!buf) {
 		printf("MXS NAND: Error allocating DMA buffers\n");
 		return -ENOMEM;
 	}
 
-	memset(buf, 0, size);
+	memset(buf, 0, nand_info->data_buf_size);
 
 	nand_info->data_buf = buf;
 	nand_info->oob_buf = buf + NAND_MAX_PAGESIZE;
-
 	/* Command buffers */
 	nand_info->cmd_buf = memalign(MXS_DMA_ALIGNMENT,
 				MXS_NAND_COMMAND_BUFFER_SIZE);