Add gdb python scripts as kernel debugging helpers

index 253a455..fb18794 100644
@@ -774,6 +774,9 @@ endif
 ifdef CONFIG_BUILD_DOCSRC
 	$(Q)$(MAKE) $(build)=Documentation
 endif
+ifdef CONFIG_DEBUG_INFO
+	$(Q)ln -fsn $(srctree)/scripts/gdb/vmlinux-gdb.py
+endif
 	+$(call if_changed,link-vmlinux)
 
 # The actual objects are generated when descending, 

@@ -1019,7 +1022,7 @@ MRPROPER_FILES += .config .config.old .version .old_version $(version_h) \
 		  Module.symvers tags TAGS cscope* GPATH GTAGS GRTAGS GSYMS \
 		  signing_key.priv signing_key.x509 x509.genkey		\
 		  extra_certificates signing_key.x509.keyid		\
-		  signing_key.x509.signer
+		  signing_key.x509.signer vmlinux-gdb.py
 
 # clean - Delete most, but leave enough to build external modules
 #

index 01e7adb..3204b91 100644
@@ -37,6 +37,7 @@ subdir-$(CONFIG_MODVERSIONS) += genksyms
 subdir-y                     += mod
 subdir-$(CONFIG_SECURITY_SELINUX) += selinux
 subdir-$(CONFIG_DTC)         += dtc
+subdir-$(CONFIG_DEBUG_INFO)  += gdb
 
 # Let clean descend into subdirs
-subdir-	+= basic kconfig package selinux
+subdir-	+= basic kconfig package selinux gdb

new file mode 100644
index 0000000..34ccd06
@@ -0,0 +1,9 @@
+always := gdb-scripts
+
+$(obj)/gdb-scripts:
+ifneq ($(KBUILD_SRC),)
+	$(Q)ln -fsn $(srctree)/$(obj)/*.py $(objtree)/$(obj)
+endif
+	@:
+
+clean-files := *.pyc $(if $(KBUILD_SRC),*.py)

new file mode 100644
index 0000000..65a2e91
@@ -0,0 +1,17 @@
+#
+# gdb helper commands and functions for Linux kernel debugging
+#
+#  common utilities
+#
+# Copyright (c) Siemens AG, 2011, 2012
+#
+# Authors:
+#  Jan Kiszka <[email protected]>
+#
+# This work is licensed under the terms of the GNU GPL version 2.
+#
+
+import gdb
+import re
+
+gdb_version = re.sub("^[^0-9]*", "", gdb.VERSION)

new file mode 100644
index 0000000..bcb45cc
@@ -0,0 +1,22 @@
+#
+# gdb helper commands and functions for Linux kernel debugging
+#
+#  loader module
+#
+# Copyright (c) Siemens AG, 2012
+#
+# Authors:
+#  Jan Kiszka <[email protected]>
+#
+# This work is licensed under the terms of the GNU GPL version 2.
+#
+
+import os
+
+sys.path = [ os.path.dirname(__file__) + "/scripts/gdb" ] + sys.path
+
+from utils import gdb_version
+
+if gdb_version < "7.1":
+	print "NOTE: gdb 7.1 or later required for Linux helper scripts " \
+	      "to work."

index 65a2e91..2f6eafb 100644
@@ -15,3 +15,44 @@ import gdb
 import re
 
 gdb_version = re.sub("^[^0-9]*", "", gdb.VERSION)
+
+long_type = None
+
+def get_type(type_name):
+	t = gdb.lookup_type(type_name)
+	if t == None:
+		raise gdb.GdbError("cannot resolve type '%s'" % type_name)
+	return t
+
+def get_long_type():
+	global long_type
+	if long_type == None:
+		long_type = get_type("long")
+	return long_type
+
+
+def offset_of(typeobj, field):
+	element = gdb.Value(0).cast(typeobj)
+	return int(str(element[field].address).split()[0], 16)
+
+def container_of(ptr, typeobj, member):
+	return (ptr.cast(get_long_type()) -
+		offset_of(typeobj, member)).cast(typeobj)
+
+
+class ContainerOf(gdb.Function):
+	__doc__ = "Return pointer to containing data structure.\n" \
+		  "\n" \
+		  "$container_of(PTR, \"TYPE\", \"ELEMENT\"): Given PTR, return a pointer to the\n" \
+		  "data structure of the type TYPE in which PTR is the address of ELEMENT.\n" \
+		  "Note that TYPE and ELEMENT have to be quoted as strings."
+
+	def __init__(self):
+		super(ContainerOf, self).__init__("container_of")
+
+	def invoke(self, ptr, typename, elementname):
+		return container_of(ptr,
+				    gdb.lookup_type(typename.string()).pointer(),
+				    elementname.string())
+
+ContainerOf()

index bcb45cc..8ef76c9 100644
@@ -20,3 +20,5 @@ from utils import gdb_version
 if gdb_version < "7.1":
 	print "NOTE: gdb 7.1 or later required for Linux helper scripts " \
 	      "to work."
+else:
+	import utils

new file mode 100644
index 0000000..c50d5e1
@@ -0,0 +1,153 @@
+#
+# gdb helper commands and functions for Linux kernel debugging
+#
+#  load kernel and module symbols
+#
+# Copyright (c) Siemens AG, 2011-2013
+#
+# Authors:
+#  Jan Kiszka <[email protected]>
+#
+# This work is licensed under the terms of the GNU GPL version 2.
+#
+
+import gdb
+import os
+import re
+import string
+
+from utils import *
+
+class LinuxSymbols(gdb.Command):
+	__doc__ = "(Re-)load symbols of Linux kernel and currently loaded modules.\n" \
+		  "\n" \
+		  "The kernel (vmlinux) is taken from the current working directly. Modules (.ko)\n" \
+		  "are scanned recursively, starting in the same directory. Optionally, the module\n" \
+		  "search path can be extended by a space separated list of paths passed to the\n" \
+		  "lx-symbols command."
+
+	module_files = []
+	breakpoint = None
+
+	def __init__(self):
+		super(LinuxSymbols, self).__init__("lx-symbols",
+						   gdb.COMMAND_FILES,
+						   gdb.COMPLETE_FILENAME)
+
+	def _load_module_symbols(self, module):
+		module_name = module['name'].string()
+		module_addr = str(module['module_core']).split()[0]
+		module_pattern = ".*/" + \
+			string.replace(module_name, "_", r"[_\-]") + r"\.ko$"
+		module_path = ""
+		for name in self.module_files:
+			if re.match(module_pattern, name):
+				module_path = name
+				break
+
+		if module_path:
+			print "loading @" + module_addr + ": " + module_path
+			if gdb.VERSION >= "7.2":
+				gdb.execute("add-symbol-file " + \
+					    module_path + " " + module_addr,
+					    to_string = True)
+			else:
+				gdb.execute("add-symbol-file " + \
+					    module_path + " " + module_addr)
+		else:
+			print "no module object found for '" + \
+			      module_name + "'"
+
+	if gdb_version >= "7.3":
+		class _LoadModuleBreakpoint(gdb.Breakpoint):
+			def __init__(self, spec, gdb_command):
+				super(LinuxSymbols._LoadModuleBreakpoint,
+				      self).__init__(spec, internal = True)
+				self.silent = True
+				self.gdb_command = gdb_command
+
+			def stop(self):
+				module = gdb.parse_and_eval("mod")
+				self.gdb_command._load_module_symbols(module)
+				return False
+
+		def _find_breakpoint_location(self):
+			breakpoint_match = "^[0-9]*[\t]*err = parse_args\(.*"
+
+			src = gdb.execute("list kernel/module.c:load_module",
+					  to_string = True)
+			done = False
+			lineno = None
+			while not done:
+				src = gdb.execute("list", to_string = True)
+				for line in src.splitlines():
+					if re.match(breakpoint_match, line):
+						done = True
+						lineno = line.split()[0]
+						break
+					elif re.match("^[0-9]*\t}$", line):
+						done = True
+						break
+			return lineno
+
+	def _load_all_module_symbols(self, arg):
+		modules = gdb.parse_and_eval("modules")
+		entry = modules['next']
+		if entry == modules.address:
+			print "no modules found"
+			return
+
+		self.module_files = []
+		paths = arg.split()
+		paths.append(os.getcwd())
+		for path in paths:
+			print "scanning for modules in " + path
+			for root, dirs, files in os.walk(path):
+				for name in files:
+					if re.match(r".*\.ko$", name):
+						self.module_files.append(
+							root + "/" + name)
+
+		module_type = gdb.lookup_type("struct module").pointer()
+
+		while entry != modules.address:
+			module = container_of(entry, module_type, "list")
+			self._load_module_symbols(module)
+			entry = entry['next']
+
+	def invoke(self, arg, from_tty):
+		print "loading vmlinux"
+
+		saved_states = []
+		if (gdb.breakpoints() != None):
+			for breakpoint in gdb.breakpoints():
+				saved_states.append({
+					'breakpoint': breakpoint,
+					'enabled': breakpoint.enabled })
+
+		# drop all current symbols and reload vmlinux
+		gdb.execute("symbol-file", to_string = True)
+		gdb.execute("symbol-file vmlinux")
+
+		self._load_all_module_symbols(arg)
+
+		for saved_state in saved_states:
+			saved_state['breakpoint'].enabled = \
+				saved_state['enabled']
+
+		if gdb_version < "7.3":
+			print "Note: symbol update on module loading not " \
+			      "supported with this gdb version"
+		else:
+			if self.breakpoint != None:
+				self.breakpoint.delete()
+				self.breakpoint = None
+			lineno = self._find_breakpoint_location()
+			if lineno != None:
+				self.breakpoint = self._LoadModuleBreakpoint(
+					lineno, self)
+			else:
+				print "unable to detect breakpoint location " \
+				      "of load module completion"
+
+LinuxSymbols()

index 8ef76c9..3553b7d 100644
@@ -22,3 +22,4 @@ if gdb_version < "7.1":
 	      "to work."
 else:
 	import utils
+	import symbols

index 2f6eafb..44daa29c 100644
@@ -56,3 +56,20 @@ class ContainerOf(gdb.Function):
 				    elementname.string())
 
 ContainerOf()
+
+
+BIG_ENDIAN = 0
+LITTLE_ENDIAN = 1
+target_endianness = None
+
+def get_target_endianness():
+	global target_endianness
+	if target_endianness == None:
+		endian = gdb.execute("show endian", False, True)
+		if endian.find("little endian") >= 0:
+			target_endianness = LITTLE_ENDIAN
+		elif endian.find("big endian") >= 0:
+			target_endianness = BIG_ENDIAN
+		else:
+			raise gdb.GdgError("unknown endianness '%s'" % endian)
+	return target_endianness

index 44daa29c..8d54241 100644
@@ -73,3 +73,21 @@ def get_target_endianness():
 		else:
 			raise gdb.GdgError("unknown endianness '%s'" % endian)
 	return target_endianness
+
+def read_u16(buffer):
+	if get_target_endianness() == LITTLE_ENDIAN:
+		return ord(buffer[0]) + (ord(buffer[1]) << 8)
+	else:
+		return ord(buffer[1]) + (ord(buffer[0]) << 8)
+
+def read_u32(buffer):
+	if get_target_endianness() == LITTLE_ENDIAN:
+		return read_u16(buffer[0:2]) + (read_u16(buffer[2:4]) << 16)
+	else:
+		return read_u16(buffer[2:4]) + (read_u16(buffer[0:2]) << 16)
+
+def read_u64(buffer):
+	if get_target_endianness() == LITTLE_ENDIAN:
+		return read_u32(buffer[0:4]) + (read_u32(buffer[4:8]) << 32)
+	else:
+		return read_u32(buffer[4:8]) + (read_u32(buffer[0:4]) << 32)

new file mode 100644
index 0000000..1bc3368
@@ -0,0 +1,63 @@
+#
+# gdb helper commands and functions for Linux kernel debugging
+#
+#  kernel log buffer dump
+#
+# Copyright (c) Siemens AG, 2011, 2012
+#
+# Authors:
+#  Jan Kiszka <[email protected]>
+#
+# This work is licensed under the terms of the GNU GPL version 2.
+#
+
+import gdb
+import string
+
+from utils import *
+
+class LinuxDmesg(gdb.Command):
+	__doc__ = "Print Linux kernel log buffer."
+
+	def __init__(self):
+		super(LinuxDmesg, self).__init__("lx-dmesg", gdb.COMMAND_DATA)
+
+	def invoke(self, arg, from_tty):
+		log_buf_addr = \
+			int(str(gdb.parse_and_eval("log_buf")).split()[0], 16)
+		log_first_idx = int(gdb.parse_and_eval("log_first_idx"))
+		log_next_idx = int(gdb.parse_and_eval("log_next_idx"))
+		log_buf_len = int(gdb.parse_and_eval("log_buf_len"))
+
+		inf = gdb.inferiors()[0]
+		start = log_buf_addr + log_first_idx
+		if log_first_idx < log_next_idx:
+			log_buf_2nd_half = -1
+			length = log_next_idx - log_first_idx
+			log_buf = inf.read_memory(start, length)
+		else:
+			log_buf_2nd_half = log_buf_len - log_first_idx
+			log_buf = inf.read_memory(start, log_buf_2nd_half) + \
+				  inf.read_memory(log_buf_addr, log_next_idx)
+
+		pos = 0
+		while pos < log_buf.__len__():
+			length = read_u16(log_buf[pos + 8 : pos + 10])
+			if length == 0:
+				if log_buf_2nd_half == -1:
+					print "Corrupted log buffer!"
+					break
+				pos = log_buf_2nd_half
+				continue
+
+			text_len = read_u16(log_buf[pos + 10 : pos + 12])
+			time_stamp = read_u64(log_buf[pos : pos + 8])
+
+			for line in log_buf[pos + 16 :
+					    pos + 16 + text_len].splitlines():
+				print "[%13.6f] " % \
+				      (time_stamp / 1000000000.0) + line
+
+			pos += length
+
+LinuxDmesg()

index 3553b7d..47f296d 100644
@@ -23,3 +23,4 @@ if gdb_version < "7.1":
 else:
 	import utils
 	import symbols
+	import dmesg

new file mode 100644
index 0000000..ff05aa5
@@ -0,0 +1,40 @@
+#
+# gdb helper commands and functions for Linux kernel debugging
+#
+#  task & thread tools
+#
+# Copyright (c) Siemens AG, 2011, 2012
+#
+# Authors:
+#  Jan Kiszka <[email protected]>
+#
+# This work is licensed under the terms of the GNU GPL version 2.
+#
+
+import gdb
+
+from utils import *
+
+task_ptr_type = None
+
+def get_task_ptr_type():
+	global task_ptr_type
+	if task_ptr_type == None:
+		task_ptr_type = get_type("struct task_struct").pointer()
+	return task_ptr_type
+
+def for_each_task(func, arg = None):
+	init_task = gdb.parse_and_eval("init_task")
+	g = init_task.address
+	while True:
+		g =  container_of(g['tasks']['next'], get_task_ptr_type(),
+				  "tasks")
+		if g == init_task.address:
+			break;
+		t = g
+		while True:
+			func(t, arg)
+			t = container_of(t['thread_group']['next'],
+					 get_task_ptr_type(), "thread_group")
+			if t == g:
+				break

index ff05aa5..93e6c0c 100644
@@ -38,3 +38,32 @@ def for_each_task(func, arg = None):
 					 get_task_ptr_type(), "thread_group")
 			if t == g:
 				break
+
+def get_task_by_pid(pid):
+	def match_pid(t, arg):
+		if int(t['pid']) == arg['pid']:
+			arg['task'] = t
+
+	arg = { 'pid': pid, 'task': None }
+	for_each_task(match_pid, arg)
+
+	return arg['task']
+
+
+class LxTaskByPidFunc(gdb.Function):
+	__doc__ = "Find Linux task by PID and return the task_struct variable.\n" \
+		  "\n" \
+		  "$lx_task_by_pid(PID): Given PID, iterate over all tasks of the target and\n" \
+		  "return that task_struct variable which PID matches.\n"
+
+	def __init__(self):
+		super(LxTaskByPidFunc, self).__init__("lx_task_by_pid")
+
+	def invoke(self, pid):
+		task = get_task_by_pid(pid)
+		if task:
+			return task.dereference()
+		else:
+			raise gdb.GdbError("No task of PID " + str(pid))
+
+LxTaskByPidFunc()

index 47f296d..e78b353 100644
@@ -24,3 +24,4 @@ else:
 	import utils
 	import symbols
 	import dmesg
+	import task

index 8d54241..532d379 100644
@@ -91,3 +91,12 @@ def read_u64(buffer):
 		return read_u32(buffer[0:4]) + (read_u32(buffer[4:8]) << 32)
 	else:
 		return read_u32(buffer[4:8]) + (read_u32(buffer[0:4]) << 32)
+
+
+target_arch = None
+
+def is_target_arch(arch):
+	global target_arch
+	if target_arch == None:
+		target_arch = gdb.execute("show architecture", False, True)
+	return target_arch.find(arch) >= 0

index 93e6c0c..0a45b3c 100644
@@ -67,3 +67,42 @@ class LxTaskByPidFunc(gdb.Function):
 			raise gdb.GdbError("No task of PID " + str(pid))
 
 LxTaskByPidFunc()
+
+
+thread_info_ptr_type = None
+
+def get_thread_info_ptr_type():
+	global thread_info_ptr_type
+	if thread_info_ptr_type = None:
+		thread_info_ptr_type = get_type('struct thread_info').pointer()
+	return thread_info_ptr_type
+
+ia64_task_size = None
+
+def get_thread_info(task):
+	if is_target_arch("ia64"):
+		global ia64_task_size
+		if ia64_task_size = None:
+			ia64_task_size = gdb.parse_and_eval(
+						"sizeof(struct task_struct)")
+		thread_info_addr = task.address + ia64_task_size
+		thread_info = thread_info_addr.cast(get_thread_info_ptr_type())
+	else:
+		thread_info = task['stack'].cast(get_thread_info_ptr_type())
+	return thread_info.dereference()
+
+
+class LxThreadInfoFunc (gdb.Function):
+	# Calculate Linux thread_info from task variable.
+	__doc__ = "Calculate Linux thread_info from task variable.\n" \
+		  "\n" \
+		  "$lx_thread_info(TASK): Given TASK, return the corresponding thread_info\n" \
+		  "variable.\n"
+
+	def __init__(self):
+		super(LxThreadInfoFunc, self).__init__("lx_thread_info")
+
+	def invoke(self, task):
+		return get_thread_info(task)
+
+LxThreadInfoFunc()

index 532d379..a20f096 100644
@@ -100,3 +100,38 @@ def is_target_arch(arch):
 	if target_arch == None:
 		target_arch = gdb.execute("show architecture", False, True)
 	return target_arch.find(arch) >= 0
+
+
+GDBSERVER_QEMU = 0
+GDBSERVER_KGDB = 1
+gdbserver_type = None
+
+def get_gdbserver_type():
+	def exit_handler(event):
+		global gdbserver_type
+		gdbserver_type = None
+		gdb.events.exited.disconnect(exit_handler)
+
+	def probe_qemu():
+		try:
+			return gdb.execute("monitor info version", False,
+					   True) != ""
+		except:
+			return False
+
+	def probe_kgdb():
+		try:
+			thread_info = gdb.execute("info thread 2", False, True)
+			return thread_info.find("shadowCPU0") >= 0
+		except:
+			return False
+
+	global gdbserver_type
+	if gdbserver_type == None:
+		if probe_qemu():
+			gdbserver_type = GDBSERVER_QEMU
+		elif probe_kgdb():
+			gdbserver_type = GDBSERVER_KGDB
+		if gdbserver_type != None:
+			gdb.events.exited.connect(exit_handler)
+	return gdbserver_type

new file mode 100644
index 0000000..e2556919
@@ -0,0 +1,61 @@
+#
+# gdb helper commands and functions for Linux kernel debugging
+#
+#  per-cpu tools
+#
+# Copyright (c) Siemens AG, 2011, 2012
+#
+# Authors:
+#  Jan Kiszka <[email protected]>
+#
+# This work is licensed under the terms of the GNU GPL version 2.
+#
+
+import gdb
+
+from utils import *
+from task import *
+
+MAX_CPUS = 4096
+
+def get_current_cpu():
+	if get_gdbserver_type() = GDBSERVER_QEMU:
+		return gdb.selected_thread().num - 1
+	elif get_gdbserver_type() = GDBSERVER_KGDB:
+		tid = gdb.selected_thread().ptid[2]
+		if tid > (0x100000000 - MAX_CPUS - 2):
+			return 0x100000000 - tid - 2
+		else:
+			return get_thread_info(get_task_by_pid(tid))['cpu']
+	else:
+		raise gdb.GdbError("Sorry, obtaining the current CPU is "
+				   "not yet supported with this gdb server.")
+
+def per_cpu(var_name, cpu):
+	var = gdb.parse_and_eval("&" + var_name)
+	if cpu = -1:
+		cpu = get_current_cpu()
+	if is_target_arch("sparc:v9"):
+		offset = gdb.parse_and_eval("trap_block[" + str(cpu) +
+					    "].__per_cpu_base")
+	else:
+		offset = gdb.parse_and_eval("__per_cpu_offset[" + str(cpu) +
+					    "]")
+	pointer = var.cast(get_long_type()) + offset
+	return pointer.cast(var.type).dereference()
+
+
+class PerCpu(gdb.Function):
+	__doc__ = "Return per-cpu variable.\n" \
+		  "\n" \
+		  "$lx_per_cpu(\"VAR\"[, CPU]): Return the per-cpu variable called VAR for the\n" \
+		  "given CPU number. If CPU is omitted, the CPU of the current context is used.\n" \
+		  "Note that VAR has to be quoted as string."
+
+	def __init__(self):
+		super(PerCpu, self).__init__("lx_per_cpu")
+
+	def invoke(self, var_name, cpu = -1):
+		return per_cpu(var_name.string(), cpu)
+
+PerCpu()

index e78b353..6606247 100644
@@ -25,3 +25,4 @@ else:
 	import symbols
 	import dmesg
 	import task
+	import percpu

index e2556919..dbc84c0 100644
@@ -59,3 +59,18 @@ class PerCpu(gdb.Function):
 		return per_cpu(var_name.string(), cpu)
 
 PerCpu()
+
+
+class LxCurrentFunc(gdb.Function):
+	__doc__ = "Return current task.\n" \
+		  "\n" \
+		  "$lx_current([CPU]): Return the per-cpu task variable for the given CPU\n" \
+		  "number. If CPU is omitted, the CPU of the current context is used."
+
+	def __init__(self):
+		super(LxCurrentFunc, self).__init__("lx_current")
+
+	def invoke(self, cpu = -1):
+		return per_cpu("current_task", cpu).dereference()
+
+LxCurrentFunc()

new file mode 100644
index 0000000..bf2adbf
@@ -0,0 +1,145 @@
+Debugging kernel and modules via gdb
+====================================
+
+The kernel debugger kgdb, hypervisors like QEMU or JTAG-based hardware
+interfaces allow to debug the Linux kernel and its modules during runtime
+using gdb. Gdb comes with a powerful scripting interface for python. The
+kernel provides a collection of helper scripts that can simplify typical
+kernel debugging steps. This is a short tutorial about how to enable and use
+them. It focuses on QEMU/KVM virtual machines as target, but the examples can
+be transferred to the other gdb stubs as well.
+
+
+Requirements
+------------
+
+ o gdb 7.1+ (recommended: 7.3+) with python support enabled (typically true
+   for distributions)
+
+
+Setup
+-----
+
+ o Create a virtual Linux machine for QEMU/KVM (see www.linux-kvm.org and
+   www.qemu.org for more details)
+
+ o Build the kernel with CONFIG_DEBUG_INFO enabled, but leave
+   CONFIG_DEBUG_INFO_REDUCED off
+
+ o Install that kernel on the guest.
+
+   Alternatively, QEMU allows to boot the kernel directly using -kernel,
+   -append, -initrd command line switches. This is generally only useful if
+   you do not depend on modules. See QEMU documentation for more details on
+   this mode.
+
+ o Enable the gdb stub of QEMU/KVM, either
+    - at VM startup time by appending "-s" to the QEMU command line
+   or
+    - during runtime by issuing "gdbserver" from the QEMU monitor
+      console
+
+ o cd /path/to/linux-build
+
+ o Start gdb: gdb vmlinux
+
+ o Attach to the booted guest:
+    (gdb) target remote :1234
+
+
+Examples of using the Linux-provided gdb helpers
+------------------------------------------------
+
+ o Load module (and main kernel) symbols:
+    (gdb) lx-symbols
+    loading vmlinux
+    scanning for modules in /home/user/linux/build
+    loading @0xffffffffa0020000: /home/user/linux/build/net/netfilter/xt_tcpudp.ko
+    loading @0xffffffffa0016000: /home/user/linux/build/net/netfilter/xt_pkttype.ko
+    loading @0xffffffffa0002000: /home/user/linux/build/net/netfilter/xt_limit.ko
+    loading @0xffffffffa00ca000: /home/user/linux/build/net/packet/af_packet.ko
+    loading @0xffffffffa003c000: /home/user/linux/build/fs/fuse/fuse.ko
+    ...
+    loading @0xffffffffa0000000: /home/user/linux/build/drivers/ata/ata_generic.ko
+
+ o Set a breakpoint on some not yet loaded module function, e.g.:
+    (gdb) b btrfs_init_sysfs
+    Function "btrfs_init_sysfs" not defined.
+    Make breakpoint pending on future shared library load? (y or [n]) y
+    Breakpoint 1 (btrfs_init_sysfs) pending.
+
+ o Continue the target
+    (gdb) c
+
+ o Load the module on the target and watch the symbols being loaded as well as
+   the breakpoint hit:
+    loading @0xffffffffa0034000: /home/user/linux/build/lib/libcrc32c.ko
+    loading @0xffffffffa0050000: /home/user/linux/build/lib/lzo/lzo_compress.ko
+    loading @0xffffffffa006e000: /home/user/linux/build/lib/zlib_deflate/zlib_deflate.ko
+    loading @0xffffffffa01b1000: /home/user/linux/build/fs/btrfs/btrfs.ko
+
+    Breakpoint 1, btrfs_init_sysfs () at /home/user/linux/fs/btrfs/sysfs.c:36
+    36              btrfs_kset = kset_create_and_add("btrfs", NULL, fs_kobj);
+
+ o Dump the log buffer of the target kernel:
+    (gdb) lx-dmesg
+    [     0.000000] Initializing cgroup subsys cpuset
+    [     0.000000] Initializing cgroup subsys cpu
+    [     0.000000] Linux version 3.8.0-rc4-dbg+ (...
+    [     0.000000] Command line: root=/dev/sda2 resume=/dev/sda1 vga=0x314
+    [     0.000000] e820: BIOS-provided physical RAM map:
+    [     0.000000] BIOS-e820: [mem 0x0000000000000000-0x000000000009fbff] usable
+    [     0.000000] BIOS-e820: [mem 0x000000000009fc00-0x000000000009ffff] reserved
+    ....
+
+ o Examine fields of the current task struct:
+    (gdb) p $lx_current().pid
+    $1 = 4998
+    (gdb) p $lx_current().comm
+    $2 = "modprobe\000\000\000\000\000\000\000"
+
+ o Make use of the per-cpu helper for the current or a specified CPU:
+    (gdb) p $lx_per_cpu("runqueues").nr_running
+    $3 = 1
+    (gdb) p $lx_per_cpu("runqueues", 2).nr_running
+    $4 = 0
+
+ o Dig into hrtimers using the container_of helper:
+    (gdb) set $next = $lx_per_cpu("hrtimer_bases").clock_base[0].active.next
+    (gdb) p *$container_of($next, "struct hrtimer", "node")
+    $5 = {
+      node = {
+        node = {
+          __rb_parent_color = 18446612133355256072,
+          rb_right = 0x0 <irq_stack_union>,
+          rb_left = 0x0 <irq_stack_union>
+        },
+        expires = {
+          tv64 = 1835268000000
+        }
+      },
+      _softexpires = {
+        tv64 = 1835268000000
+      },
+      function = 0xffffffff81078232 <tick_sched_timer>,
+      base = 0xffff88003fd0d6f0,
+      state = 1,
+      start_pid = 0,
+      start_site = 0xffffffff81055c1f <hrtimer_start_range_ns+20>,
+      start_comm = "swapper/2\000\000\000\000\000\000"
+    }
+
+
+List of commands and helper
+---------------------------
+
+The number of commands and convenience helpers may evolve over the time, this
+is just a snapshot of the initial version:
+
+ (gdb) apropos lx
+ function lx_current -- Return current task
+ function lx_per_cpu -- Return per-cpu variable
+ function lx_task_by_pid -- Find Linux task by PID and return the task_struct variable
+ function lx_thread_info -- Calculate Linux thread_info from task variable
+ lx-dmesg -- Print Linux kernel log buffer
+ lx-symbols -- (Re-)load symbols of Linux kernel and currently loaded modules