Skip to contentSkip to conversation

kmmio/mmio-trace problems

PATCH Jeff Muizelaar 1 participant 1 message Jan 25, 2007

lore
Share & tools
lore
#0 lore raw

I have built a tool with the goal of logging mmio writes and reads by device drivers. I am sure anyone who has reverse engineered device drivers can immediately see the use of such a tool. In particular I have been trying to help with the nouveau project. The code for the tool is based on the work Intel did quite some time ago for device driver fault injection which some of you may remember (It was presented at OLS in 2003).

The basic mechanism involves using page faults to capture all of the accesses to an ioremapped area. It works by clearing the _PAGE_PRESENT bit on all of the pages in the mapping. On an access to the area, the faulting instruction is decoded. The _PAGE_PRESENT flag is set and the single stepping is turned on in eflags. In the debugging handler the _PAGE_PRESET flag is recleared and written or read values are logged.

The implementation seems to mostly work, however, there are some large stability problems. Occasionally, it seems that the processor is not convinced that page is actually present and will continue to fault on the same instruction.

Basically, I have a situation where if the following code is executed in the page fault handler I get a recursive fault. i.e *(int *)addr page faults.

if (page_present(addr)) {
	flush_tlb_all();
	int read_val = *(int *)addr;
	printk("read: %x\n", read_val);
	explosion;
}

static inline unsigned long page_present(unsigned long address)

{
        pgd_t *pgd = pgd_offset_k(address);
        pud_t *pud = pud_offset(pgd, address);
        pmd_t *pmd = pmd_offset(pud, address);
        if (pmd_large(*pmd)) {
                printk("4MB pages are not currently supported: %lx\n", address);
                BUG();
        }
        return pte_val(*pte_offset_kernel(pmd, address)) & _PAGE_PRESENT;
}

I have no idea why this would happen. Can anyone think of any reason it would?

Note: the addresses that cause the problem are not anything special. e.g. base+0x600140 (they don't span page boundaries)

The code for the intercepting calls to ioremap and setting up the trace region (mmio-trace) is at:
git://people.freedesktop.org/~jrmuizel/mmio-trace
http://gitweb.freedesktop.org/?p=users/jrmuizel/mmio-trace.git;a=summary
There is also more information about mmio-trace at
http://nouveau.freedesktop.org/wiki/MmioTrace

There are a whole set of logs from the debugging sessions available at
https://no.spoon.fi/~pq/home_nv28/ in the mmio directories. The most recent and probably the more interesting logs are in the mmio17 directory.

Although, the problem was found using the binary drivers. I can reproduce the problem with a simple driver that maps the io memory on an nvidia card and then pokes it.
http://infidigm.net/~jeff/nv-test-0.5.tar.bz2

I've included the relevant code here. It is pretty straight forward.

static void poke(void *base)

{
	int new;
	printk("doing a poke()\n");
	new = *(int *)(base + 0x101000);
	printk("read %x from 0x101000\n", new);
	*(int *)(base + 0x600140) = 0;
	printk("wrote 0 to 0x600140\n");
}

static int __devinit nv_test_init_one(struct pci_dev *pdev,
const struct pci_device_id *ent)

{
	int err;
	void *base;

	err = pci_enable_device(pdev);
	if (err) {
		printk(KERN_ERR "%s cannot enable PCI device\n",
				pci_name(pdev));
		goto err_out;
	}

	err = pci_request_regions(pdev, "nv-test");
	if (err) {
		printk(KERN_ERR "%s cannot obtain PCI resources\n",
				pci_name(pdev));
		goto err_out_disable_pdev;
	}

	pci_set_master(pdev);

	base = ioremap_nocache(pci_resource_start(pdev, 0), 0x1000000);

	poke(base);

	pci_set_drvdata(pdev, base);
err_out_disable_pdev:
	pci_disable_device(pdev);
	pci_set_drvdata(pdev, NULL);
err_out:
	return err;
}

Finally, here is the kmmio patch against a git kernel from about 2 weeks ago. A debugging version of the patch that the above case is from is also available at:
http://infidigm.net/~jeff/kmmio-super-debug.patch

arch/i386/Kconfig.debug +9 -0
index f68cc6f..64372f3 100644
@@ -26,6 +26,15 @@ config DEBUG_STACKOVERFLOW
 	  This option will cause messages to be printed if free stack space
 	  drops below a certain limit.
 
+config KMMIO
+	bool "KMMIO (EXPERIMENTAL)"
+	depends on EXPERIMENTAL
+	depends on KPROBES 
+	help
+	  KMMIO is a Kprobes add-ons for placing a probe on MMIO access, using
+	  register_kmmio(), and providing a callback function. This is useful
+	  for monitoring driver access specific MMIO address.
+
 config DEBUG_STACK_USAGE
 	bool "Stack utilization instrumentation"
 	depends on DEBUG_KERNEL
arch/i386/kernel/Makefile +1 -0
index 1e8988e..2b5fec2 100644
@@ -29,6 +29,7 @@ obj-$(CONFIG_CRASH_DUMP)	+= crash_dump.o
 obj-$(CONFIG_X86_NUMAQ)		+= numaq.o
 obj-$(CONFIG_X86_SUMMIT_NUMA)	+= summit.o
 obj-$(CONFIG_KPROBES)		+= kprobes.o
+obj-$(CONFIG_KMMIO)		+= kmmio.o
 obj-$(CONFIG_MODULES)		+= module.o
 obj-y				+= sysenter.o vsyscall.o
 obj-$(CONFIG_ACPI_SRAT) 	+= srat.o
arch/i386/kernel/kmmio.c new file +162 -0

162 lines not on this page · raw message

arch/i386/kernel/traps.c +5 -0
index 0efad8a..81a506a 100644
@@ -27,6 +27,7 @@
 #include <linux/utsname.h>
 #include <linux/kprobes.h>
 #include <linux/kexec.h>
+#include <linux/kmmio.h>
 #include <linux/unwind.h>
 #include <linux/uaccess.h>
 #include <linux/nmi.h>
@@ -808,9 +809,13 @@ fastcall void __kprobes do_debug(struct pt_regs * regs, long error_code)
 
 	get_debugreg(condition, 6);
 
+	if (post_kmmio_handler(condition, regs))
+		return;
+
 	if (notify_die(DIE_DEBUG, "debug", regs, condition, error_code,
 					SIGTRAP) == NOTIFY_STOP)
 		return;
+
 	/* It's safe to allow irq's after DR6 has been saved */
 	if (regs->eflags & X86_EFLAGS_IF)
 		local_irq_enable();
arch/i386/mm/fault.c +4 -0
index aaaa4d2..cbc706b 100644
@@ -22,6 +22,7 @@
 #include <linux/highmem.h>
 #include <linux/module.h>
 #include <linux/kprobes.h>
+#include <linux/kmmio.h>
 #include <linux/uaccess.h>
 
 #include <asm/system.h>
@@ -333,6 +334,9 @@ fastcall void __kprobes do_page_fault(struct pt_regs *regs,
 	/* get the address */
         address = read_cr2();
 
+	if (is_kmmio_active() && kmmio_handler(regs, address))
+		return;
+	
 	tsk = current;
 
 	si_code = SEGV_MAPERR;
include/asm-i386/kmmio.h new file +33 -0
new file mode 100644
index 0000000..6f8f034
@@ -0,0 +1,33 @@
+/*
+ * Benfit many code from kprobes
+ * (C) 2002 Louis Zhuang <[email protected]>.
+ */
+
+#ifndef _ASM_KMMIO_H
+#define _ASM_KMMIO_H
+#include <linux/types.h>
+#include <linux/ptrace.h>
+
+struct pt_regs;
+
+typedef unsigned long kmmio_addr_t;
+
+#ifdef CONFIG_KMMIO
+extern void arm_kmmio_fault_page(kmmio_addr_t page);
+extern void disarm_kmmio_fault_page(kmmio_addr_t page);
+extern int post_kmmio_handler(unsigned long condition, 
+			      struct pt_regs *regs);
+extern int kmmio_handler(struct pt_regs *regs, unsigned long addr);
+extern void *kmmio_invert_map(void *virt_addr, unsigned long bus_addr);
+#else /* !CONFIG_KMMIO */
+static inline int post_kmmio_handler(unsigned long condition, 
+				     struct pt_regs *regs) 
+{ 
+	return 0; 
+}
+static inline int kmmio_handler(struct pt_regs *regs, unsigned long addr) 
+{ 
+	return 0; 
+}
+#endif
+#endif /* _ASM_KMMIO_H */
include/linux/kmmio.h new file +69 -0

69 lines not on this page · raw message

kernel/Makefile +1 -0
index 14f4d45..ab7d291 100644
@@ -47,6 +47,7 @@ obj-$(CONFIG_GENERIC_HARDIRQS) += irq/
 obj-$(CONFIG_SECCOMP) += seccomp.o
 obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o
 obj-$(CONFIG_RELAY) += relay.o
+obj-$(CONFIG_KMMIO) += kmmio.o
 obj-$(CONFIG_UTS_NS) += utsname.o
 obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o
 obj-$(CONFIG_TASKSTATS) += taskstats.o tsacct.o
kernel/kmmio.c new file +157 -0

157 lines not on this page · raw message