Embed dtc and libfdt in the kernel

Here's the latest spin of my patch to embed and use dtc in the kernel source, rather than relying on it as an external dependency. In addition, this adds patches to use the libfdt from the embedded copy of dtc in the bootwrapper, in preference to the less flexible flatdevtree.c library.

David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

This very large patch incorporates a copy of dtc (including libfdt) into the kernel source, in arch/powerpc/boot/dtc-src. This patch only imports the upstream sources verbatim, later patches are needed to actually link it into the kernel Makefiles and use the embedded code during the kernel build.

---
 arch/powerpc/boot/dtc-src/Makefile.dtc             |   25 
 arch/powerpc/boot/dtc-src/checks.c                 |  460 ++++
 arch/powerpc/boot/dtc-src/data.c                   |  351 +++
 arch/powerpc/boot/dtc-src/dtc-lexer.l              |  341 +++
 arch/powerpc/boot/dtc-src/dtc-lexer.lex.c_shipped  | 2184 +++++++++++++++++++++
 arch/powerpc/boot/dtc-src/dtc-parser.tab.c_shipped | 1925 ++++++++++++++++++
 arch/powerpc/boot/dtc-src/dtc-parser.tab.h_shipped |  110 +
 arch/powerpc/boot/dtc-src/dtc-parser.y             |  304 ++
 arch/powerpc/boot/dtc-src/dtc.c                    |  240 ++
 arch/powerpc/boot/dtc-src/dtc.h                    |  253 ++
 arch/powerpc/boot/dtc-src/flattree.c               |  959 +++++++++
 arch/powerpc/boot/dtc-src/fstree.c                 |   94 
 arch/powerpc/boot/dtc-src/libfdt/Makefile.libfdt   |   14 
 arch/powerpc/boot/dtc-src/libfdt/fdt.c             |  156 +
 arch/powerpc/boot/dtc-src/libfdt/fdt.h             |   60 
 arch/powerpc/boot/dtc-src/libfdt/fdt_ro.c          |  562 +++++
 arch/powerpc/boot/dtc-src/libfdt/fdt_rw.c          |  447 ++++
 arch/powerpc/boot/dtc-src/libfdt/fdt_strerror.c    |   96 
 arch/powerpc/boot/dtc-src/libfdt/fdt_sw.c          |  258 ++
 arch/powerpc/boot/dtc-src/libfdt/fdt_wip.c         |  144 +
 arch/powerpc/boot/dtc-src/libfdt/libfdt.h          |  593 +++++
 arch/powerpc/boot/dtc-src/libfdt/libfdt_internal.h |   89 
 arch/powerpc/boot/dtc-src/livetree.c               |  350 +++
 arch/powerpc/boot/dtc-src/srcpos.c                 |  105 +
 arch/powerpc/boot/dtc-src/srcpos.h                 |   75 
 arch/powerpc/boot/dtc-src/treesource.c             |  236 ++
 arch/powerpc/boot/dtc-src/version_gen.h            |    1 
 27 files changed, 10432 insertions(+)

Much too big for the list. Full patch at
http://ozlabs.org/~dgibson/home/merge-dtc.patch

David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

This patch alters the kernel makefiles to build dtc from the sources embedded in the previous patch. It also changes the arch/powerpc/boot/wrapper script to use the embedded dtc, rather than expecting a copy of dtc already installed on the system.

---
 arch/powerpc/boot/Makefile |   41 ++++++++++++++++++++++++++++++++++++++---
 arch/powerpc/boot/wrapper  |    2 +-
 2 files changed, 39 insertions(+), 4 deletions(-)

Index: working-2.6/arch/powerpc/boot/Makefile
===================================================================
@@ -108,17 +108,52 @@ $(patsubst %.S,%.o, $(filter %.S, $(src-
 $(obj)/wrapper.a: $(obj-wlib) FORCE
 	$(call if_changed,bootar)
 
-hostprogs-y	:= addnote addRamDisk hack-coff mktree
+hostprogs-y	:= addnote addRamDisk hack-coff mktree dtc
 
 targets		+= $(patsubst $(obj)/%,%,$(obj-boot) wrapper.a)
 extra-y		:= $(obj)/wrapper.a $(obj-plat) $(obj)/empty.o \
 		   $(obj)/zImage.lds $(obj)/zImage.coff.lds $(obj)/zImage.ps3.lds
 
 wrapper		:=$(srctree)/$(src)/wrapper
-wrapperbits	:= $(extra-y) $(addprefix $(obj)/,addnote hack-coff mktree) \
+wrapperbits	:= $(extra-y) $(addprefix $(obj)/,addnote hack-coff mktree dtc) \
 			$(wrapper) FORCE
 
 #############
+# Bits for building dtc
+# DTC_GENPARSER      := 1    # Uncomment to rebuild flex/bison output
+
+dtc-objs := dtc.o flattree.o fstree.o data.o livetree.o treesource.o srcpos.o checks.o
+dtc-objs += dtc-lexer.lex.o dtc-parser.tab.o
+dtc-objs := $(addprefix dtc-src/, $(dtc-objs))
+
+# prerequisites on generated files needs to be explicit
+$(obj)/dtc-src/dtc-parser.tab.o: $(obj)/dtc-src/dtc-parser.tab.c $(obj)/dtc-src/dtc-parser.tab.h
+$(obj)/dtc-src/dtc-lexer.lex.o:  $(obj)/dtc-src/dtc-lexer.lex.c $(obj)/dtc-src/dtc-parser.tab.h
+
+HOSTCFLAGS += -I$(src)/dtc-src/libfdt/
+
+targets += dtc-src/dtc-parser.tab.c
+targets += dtc-src/dtc-lexer.lex.c
+
+ifdef DTC_GENPARSER
+BISON = bison
+FLEX = flex
+
+quiet_cmd_bison = BISON   $@
+      cmd_bison = $(BISON) -o$@ -d $<; cp $@ $@_shipped
+quiet_cmd_flex = FLEX    $@
+      cmd_flex = $(FLEX) -o$@ $<; cp $@ $@_shipped
+
+$(obj)/dtc-src/dtc-parser.tab.c: $(src)/dtc-src/dtc-parser.y FORCE
+     $(call if_changed,bison)
+
+$(obj)/dtc-src/dtc-parser.tab.h: $(obj)/dtc-src/dtc-parser.tab.c
+
+$(obj)/dtc-src/dtc-lexer.lex.c: $(src)/dtc-src/dtc-lexer.l FORCE
+     $(call if_changed,flex)
+endif
+
+#############
 # Bits for building various flavours of zImage
 
 ifneq ($(CROSS32_COMPILE),)

@@ -236,7 +271,7 @@ install: $(CONFIGURE) $(addprefix $(obj)
 
 # anything not in $(targets)
 clean-files += $(image-) $(initrd-) zImage zImage.initrd cuImage.* treeImage.* \
-	otheros.bld
+	otheros.bld $(dtc-clean-files)
 
 # clean up files cached by wrapper
 clean-kernel := vmlinux.strip vmlinux.bin

Index: working-2.6/arch/powerpc/boot/wrapper
===================================================================
@@ -111,7 +111,7 @@ if [ -n "$dts" ]; then
     if [ -z "$dtb" ]; then
 	dtb="$platform.dtb"
     fi
-    dtc -O dtb -o "$dtb" -b 0 -V 16 "$dts" || exit 1
+    $object/dtc -O dtb -o "$dtb" -b 0 "$dts" || exit 1
 fi
 
 if [ -z "$kernel" ]; then

This patch incorporates libfdt (from the source embedded in an earlier patch) into the wrapper.a library used by the bootwrapper. This includes adding a libfdt_env.h file, which the libfdt sources need in order to integrate into the bootwrapper environment, and a libfdt-wrapper.c which provides glue to connect the bootwrappers abstract device tree callbacks to the libfdt functions.

In addition, this patch changes the various wrapper and platform files to use libfdt functions instead of the older flatdevtree.c library.

---
 arch/powerpc/boot/Makefile          |    4 
 arch/powerpc/boot/bamboo.c          |    2 
 arch/powerpc/boot/cuboot-52xx.c     |    2 
 arch/powerpc/boot/cuboot-83xx.c     |    2 
 arch/powerpc/boot/cuboot-85xx.c     |    2 
 arch/powerpc/boot/cuboot-8xx.c      |    2 
 arch/powerpc/boot/cuboot-hpc2.c     |    2 
 arch/powerpc/boot/cuboot-pq2.c      |    2 
 arch/powerpc/boot/cuboot-sequoia.c  |    2 
 arch/powerpc/boot/ebony.c           |    2 
 arch/powerpc/boot/ep88xc.c          |    2 
 arch/powerpc/boot/holly.c           |    2 
 arch/powerpc/boot/libfdt-wrapper.c  |  167 ++++++++++++++++++++++++++++++++++++
 arch/powerpc/boot/libfdt_env.h      |   17 +++
 arch/powerpc/boot/ops.h             |    1 
 arch/powerpc/boot/prpmc2800.c       |    3 
 arch/powerpc/boot/ps3.c             |    2 
 arch/powerpc/boot/treeboot-walnut.c |    2 
 18 files changed, 202 insertions(+), 16 deletions(-)

Index: working-2.6/arch/powerpc/boot/Makefile
===================================================================
@@ -33,7 +33,7 @@ ifeq ($(call cc-option-yn, -fstack-prote
 BOOTCFLAGS	+= -fno-stack-protector
 endif
 
-BOOTCFLAGS	+= -I$(obj) -I$(srctree)/$(obj)
+BOOTCFLAGS	+= -I$(obj) -I$(srctree)/$(obj) -I$(src)/dtc-src/libfdt
 
 $(obj)/4xx.o: BOOTCFLAGS += -mcpu=440
 $(obj)/ebony.o: BOOTCFLAGS += -mcpu=440

@@ -46,7 +46,9 @@ zliblinuxheader := zlib.h zconf.h zutil.
 $(addprefix $(obj)/,$(zlib) gunzip_util.o main.o): \
 	$(addprefix $(obj)/,$(zliblinuxheader)) $(addprefix $(obj)/,$(zlibheader))
 
+src-libfdt := fdt.c fdt_ro.c fdt_wip.c fdt_sw.c fdt_rw.c fdt_strerror.c
 src-wlib := string.S crt0.S stdio.c main.c flatdevtree.c flatdevtree_misc.c \
+		$(addprefix dtc-src/libfdt/,$(src-libfdt)) libfdt-wrapper.c \
 		ns16550.c serial.c simple_alloc.c div64.S util.S \
 		gunzip_util.c elf_util.c $(zlib) devtree.c oflib.c ofconsole.c \
 		4xx.c ebony.c mv64x60.c mpsc.c mv64x60_i2c.c cuboot.c bamboo.c \

Index: working-2.6/arch/powerpc/boot/libfdt_env.h
===================================================================
@@ -0,0 +1,17 @@
+#ifndef _ARCH_POWERPC_BOOT_LIBFDT_ENV_H
+#define _ARCH_POWERPC_BOOT_LIBFDT_ENV_H
+
+#include <types.h>
+#include <string.h>
+
+typedef u32 uint32_t;
+typedef u64 uint64_t;
+
+#define fdt16_to_cpu(x)		(x)
+#define cpu_to_fdt16(x)		(x)
+#define fdt32_to_cpu(x)		(x)
+#define cpu_to_fdt32(x)		(x)
+#define fdt64_to_cpu(x)		(x)
+#define cpu_to_fdt64(x)		(x)
+
+#endif /* _ARCH_POWERPC_BOOT_LIBFDT_ENV_H */

Index: working-2.6/arch/powerpc/boot/bamboo.c
===================================================================
@@ -42,6 +42,6 @@ void bamboo_init(void *mac0, void *mac1)
 	platform_ops.exit = ibm44x_dbcr_reset;
 	bamboo_mac0 = mac0;
 	bamboo_mac1 = mac1;
-	ft_init(_dtb_start, 0, 32);
+	fdt_init(_dtb_start);
 	serial_console_init();
 }

Index: working-2.6/arch/powerpc/boot/cuboot-52xx.c
===================================================================
@@ -53,7 +53,7 @@ void platform_init(unsigned long r3, uns
                    unsigned long r6, unsigned long r7)
 {
 	CUBOOT_INIT();
-	ft_init(_dtb_start, _dtb_end - _dtb_start, 32);
+	fdt_init(_dtb_start);
 	serial_console_init();
 	platform_ops.fixups = platform_fixups;
 }

Index: working-2.6/arch/powerpc/boot/cuboot-83xx.c
===================================================================
@@ -52,7 +52,7 @@ void platform_init(unsigned long r3, uns
                    unsigned long r6, unsigned long r7)
 {
 	CUBOOT_INIT();
-	ft_init(_dtb_start, _dtb_end - _dtb_start, 32);
+	fdt_init(_dtb_start);
 	serial_console_init();
 	platform_ops.fixups = platform_fixups;
 }

Index: working-2.6/arch/powerpc/boot/cuboot-85xx.c
===================================================================
@@ -53,7 +53,7 @@ void platform_init(unsigned long r3, uns
                    unsigned long r6, unsigned long r7)
 {
 	CUBOOT_INIT();
-	ft_init(_dtb_start, _dtb_end - _dtb_start, 32);
+	fdt_init(_dtb_start);
 	serial_console_init();
 	platform_ops.fixups = platform_fixups;
 }

Index: working-2.6/arch/powerpc/boot/cuboot-8xx.c
===================================================================
@@ -41,7 +41,7 @@ void platform_init(unsigned long r3, uns
                    unsigned long r6, unsigned long r7)
 {
 	CUBOOT_INIT();
-	ft_init(_dtb_start, _dtb_end - _dtb_start, 32);
+	fdt_init(_dtb_start);
 	serial_console_init();
 	platform_ops.fixups = platform_fixups;
 }

Index: working-2.6/arch/powerpc/boot/cuboot-hpc2.c
===================================================================
@@ -42,7 +42,7 @@ void platform_init(unsigned long r3, uns
 		unsigned long r6, unsigned long r7)
 {
 	CUBOOT_INIT();
-	ft_init(_dtb_start, _dtb_end - _dtb_start, 32);
+	fdt_init(_dtb_start);
 	serial_console_init();
 	platform_ops.fixups = platform_fixups;
 }

Index: working-2.6/arch/powerpc/boot/cuboot-pq2.c
===================================================================
@@ -255,7 +255,7 @@ void platform_init(unsigned long r3, uns
                    unsigned long r6, unsigned long r7)
 {
 	CUBOOT_INIT();
-	ft_init(_dtb_start, _dtb_end - _dtb_start, 32);
+	fdt_init(_dtb_start);
 	serial_console_init();
 	platform_ops.fixups = pq2_platform_fixups;
 }

Index: working-2.6/arch/powerpc/boot/cuboot-sequoia.c
===================================================================
@@ -51,6 +51,6 @@ void platform_init(unsigned long r3, uns
 	CUBOOT_INIT();
 	platform_ops.fixups = sequoia_fixups;
 	platform_ops.exit = ibm44x_dbcr_reset;
-	ft_init(_dtb_start, 0, 32);
+	fdt_init(_dtb_start);
 	serial_console_init();
 }

Index: working-2.6/arch/powerpc/boot/ebony.c
===================================================================
@@ -146,6 +146,6 @@ void ebony_init(void *mac0, void *mac1)
 	platform_ops.exit = ibm44x_dbcr_reset;
 	ebony_mac0 = mac0;
 	ebony_mac1 = mac1;
-	ft_init(_dtb_start, _dtb_end - _dtb_start, 32);
+	fdt_init(_dtb_start);
 	serial_console_init();
 }

Index: working-2.6/arch/powerpc/boot/ep88xc.c
===================================================================
@@ -45,7 +45,7 @@ void platform_init(unsigned long r3, uns
 	mem_size *= 1024 * 1024;
 	simple_alloc_init(_end, mem_size - (unsigned long)_end, 32, 64);
 
-	ft_init(_dtb_start, _dtb_end - _dtb_start, 32);
+	fdt_init(_dtb_start);
 
 	planetcore_set_stdout_path(table);
 

Index: working-2.6/arch/powerpc/boot/holly.c
===================================================================
@@ -28,6 +28,6 @@ void platform_init(unsigned long r3, uns
 	u32 heapsize = 0x8000000 - (u32)_end; /* 128M */
 
 	simple_alloc_init(_end, heapsize, 32, 64);
-	ft_init(_dtb_start, 0, 4);
+	fdt_init(_dtb_start);
 	serial_console_init();
 }

Index: working-2.6/arch/powerpc/boot/ops.h
===================================================================
@@ -80,6 +80,7 @@ extern struct loader_info loader_info;
 
 void start(void);
 int ft_init(void *dt_blob, unsigned int max_size, unsigned int max_find_device);
+void fdt_init(void *blob);
 int serial_console_init(void);
 int ns16550_console_init(void *devp, struct serial_console_data *scdp);
 int mpsc_console_init(void *devp, struct serial_console_data *scdp);

Index: working-2.6/arch/powerpc/boot/prpmc2800.c
===================================================================
@@ -547,8 +547,7 @@ void platform_init(unsigned long r3, uns
 	if (!dtb)
 		exit();
 	memmove(dtb, _dtb_start, dt_size);
-	if (ft_init(dtb, dt_size, 16))
-		exit();
+	fdt_init(dtb);
 
 	bridge_base = mv64x60_get_bridge_base();
 

Index: working-2.6/arch/powerpc/boot/ps3.c
===================================================================
@@ -131,7 +131,7 @@ void platform_init(void)
 	printf("\n-- PS3 bootwrapper --\n");
 
 	simple_alloc_init(_end, heapsize, 32, 64);
-	ft_init(_dtb_start, 0, 4);
+	fdt_init(_dtb_start);
 
 	chosen = finddevice("/chosen");
 

Index: working-2.6/arch/powerpc/boot/treeboot-walnut.c
===================================================================
@@ -128,6 +128,6 @@ void platform_init(void)
 	simple_alloc_init(_end, avail_ram, 32, 32);
 	platform_ops.fixups = walnut_fixups;
 	platform_ops.exit = ibm40x_dbcr_reset;
-	ft_init(_dtb_start, _dtb_end - _dtb_start, 32);
+	fdt_init(_dtb_start);
 	serial_console_init();
 }

Won't we need to change the dtc invocation in the wrapper to reserve some space now?

Speaking of which, it seems dtc still only supports setting the total size, as opposed to specifying the amount of additional space. It's still a bit crappy having to guess how much space to add, but it's better than needing to know how big the dtb will be without additional space.

How hard would it be to get libfdt to dynamically allocate any extra space it needs? This is a regression from the current flat device tree code...

-Scott

Uh.. it already does. Or rather, the shims in libfdt-wrapper.c do so, when libfdt functions which can expand the tree report that they've run out of room.

Ah... except that I haven't properly placed an fdt_open_into() with possible expansion somewhere to make sure we can handle v16 or dtbs with the blocks in unusual order.

I'll need to fix that before we merge.

David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

Ah, good -- I was looking in libfdt itself, not the wrapper. Now if only we could get something similar into u-boot... maybe libfdt proper could accept an optional realloc() function pointer in fdt_init(), and eliminate the need for the caller to provide such a wrapper?

-Scott

I've considered something like it (more likely an optional realloc()ing wrapper layer that comes with libfdt).

For the bootwrapper itself, however, I'm still hoping to get rid of malloc() entirely...

David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

Hi Scott,

My concern from the u-boot side is that u-boot has to know exactly *where* to put the expanded blob because it has to pass it to linux and keep it out of linux' way so it doesn't get "stepped on." Linux has an advantage in that it "owns" all of memory and can allocate and deallocate whatever and wherever and it won't step on itself (hopefully).

I'm assuming your boot wedge has the advantage of being able to use linux's malloc() and thus don't have to worry about coordinating with linux on memory allocation.

With the current u-boot fdt command, you can resize the blob, and this can be done in a script with all the (somewhat limited) capabilities of the u-boot shell (an adaptation of hush).

In the u-boot world, we fixate on memory maps and putting things in specific places. Maybe I'm making a problem where one doesn't exist, but an arbitrary malloc() in u-boot (as opposed to linux's malloc) seems like a Bad Thing[tm] because it is uncontrolled and may end up in a very bad place when linux starts (e.g. the linux start up script expands linux on top of the blob).

Best regards,
gvb

Linux is pretty careful not to step on the device tree; it marks the area as reserved, and moves it if it really has to. The only scenario I think would be problematic is if the kernel is started somewhere other than zero, and has to relocate itself, and the relocation clobbers the device tree. That's easily avoided by making sure the device tree is at a higher address than the kernel -- and is really not much different than the old bd_t mechanism, which didn't use a user provided address AFAIK.

No, it uses its own malloc.

OK, I'm just going by the behavior of the default boot commands on (at least some of) our boards, which just give you an error if the blob isn't already enlarged.

Maybe I should just poke Kim et al. about fixing our default environment -- though I'm guessing it'd still have to know from the script how much to enlarge the blob.

I've noticed. :-)

-Scott

That is good to know, makes things much easier.

Currently it is a manual thing, my point was that it is potentially scriptable (but would probably take a genius or 3 days of experimentation to make the script work ;-). If we can settle on a safe way to expand the blob automatically, I'm all for that.

Yeah, I had a ruder term than "fixate" but changed it for public consumption. ;-)

Best regards,
gvb

Now that earlier patches have switched the bootwrapper to using libfdt for device tree manipulation, this patch removes the now unused flatdevtree.c and related files.

---
 arch/powerpc/boot/Makefile           |    2 
 arch/powerpc/boot/flatdevtree.c      | 1036 -----------------------------------
 arch/powerpc/boot/flatdevtree.h      |  113 ---
 arch/powerpc/boot/flatdevtree_misc.c |   79 --
 arch/powerpc/boot/main.c             |    1 
 arch/powerpc/boot/ops.h              |    1 
 6 files changed, 1 insertion(+), 1231 deletions(-)

Index: working-2.6/arch/powerpc/boot/Makefile
===================================================================
@@ -47,7 +47,7 @@ $(addprefix $(obj)/,$(zlib) gunzip_util.
 	$(addprefix $(obj)/,$(zliblinuxheader)) $(addprefix $(obj)/,$(zlibheader))
 
 src-libfdt := fdt.c fdt_ro.c fdt_wip.c fdt_sw.c fdt_rw.c fdt_strerror.c
-src-wlib := string.S crt0.S stdio.c main.c flatdevtree.c flatdevtree_misc.c \
+src-wlib := string.S crt0.S stdio.c main.c \
 		$(addprefix dtc-src/libfdt/,$(src-libfdt)) libfdt-wrapper.c \
 		ns16550.c serial.c simple_alloc.c div64.S util.S \
 		gunzip_util.c elf_util.c $(zlib) devtree.c oflib.c ofconsole.c \

Index: working-2.6/arch/powerpc/boot/main.c
===================================================================
@@ -16,7 +16,6 @@
 #include "stdio.h"
 #include "ops.h"
 #include "gunzip_util.h"
-#include "flatdevtree.h"
 #include "reg.h"
 
 static struct gunzip_state gzstate;

Index: working-2.6/arch/powerpc/boot/ops.h
===================================================================
@@ -79,7 +79,6 @@ struct loader_info {
 extern struct loader_info loader_info;
 
 void start(void);
-int ft_init(void *dt_blob, unsigned int max_size, unsigned int max_find_device);
 void fdt_init(void *blob);
 int serial_console_init(void);
 int ns16550_console_init(void *devp, struct serial_console_data *scdp);

This sort of patch really appears a lot cleaner and nicer using git and the -M flag to simply state which files are removed outright.

*sigh*

jdl