IP30: SMP Help

I've recently acquired a dual R14000 CPU for the Octane, so I am trying to get SMP working again, but I can't get things setup properly. I've attached both ip30-irq.c and ip30-smp.c -- does anyone see any immediate problems (or just where I am doing it wrong)?

Most reboot cycles with this code panics because init exited with a code of 0xa or 0xb (which matches w/ SIGSEGV or SIGBUS). Randomly, I can acquire a dash shell by passing init=/bin/dash. I can't do much in it, though. A basic 'ls' either segfaults or triggers a SIGBUS. If I execute 'ls' enough times, it eventually works. Can't get much farther beyond that.

Joshua Kinard
Gentoo/MIPS
[email protected]
4096R/D25D95E3 2011-03-28

"The past tempts us, the present confuses us, the future frightens us.  And our
lives slip away, moment by moment, lost in that vast, terrible in-between."

--Emperor Turhan, Centauri Republic

/*
* This file is subject to the terms and conditions of the GNU General Public
* License. See the file "COPYING" in the main directory of this archive
* for more details.
*
* ip30-irq.c: Highlevel interrupt handling for IP30 architecture. *
* Copyright (C) 2004-2007 Stanislaw Skowronek <[email protected]>
* 2009 Johannes Dickgreber <[email protected]>
* 2007-2014 Joshua Kinard <[email protected]>
*
* Inspired by ip27-irq.c and ip32-irq.c
*/

#include <linux/init.h>
#include <linux/irq.h>
#include <linux/errno.h>
#include <linux/types.h>
#include <linux/interrupt.h>
#include <linux/tick.h>

#include <asm/irq_cpu.h>
#include <asm/pci/bridge.h>

#include <asm/mach-ip30/heart.h>
#include <asm/mach-ip30/pcibr.h>
#include <asm/mach-ip30/racermp.h>
#include <asm/mach-ip30/addrs.h>

#undef DEBUG_IRQ

static DEFINE_SPINLOCK(heart_lock);
static int heart_irq_owner[NR_IRQS];
bridge_t *ip30_irq_to_bridge[NR_IRQS];
unsigned int ip30_irq_to_slot[NR_IRQS];

//extern void ip30_clockevent_broadcast(const int cpu); //extern void ip30_clockevent_broadcast(const cpumask_t mask);

/* ----------------------------------------------------------------------- */

/**
* ip30_do_error_irq - IRQ dispatch for all HEART error IRQs (51 - 63). */
static noinline void
ip30_do_error_irq(void)

{
	unsigned long errors;
	unsigned long cause;
	int i;

	irq_enter();
	errors = readq(HEART_ISR);
	cause = readq(HEART_CAUSE);
	writeq(HEART_INT_LEVEL4, HEART_CLR_ISR);

	/* Refer to ip30-heart.h for the HC_* macros to know what the cause was */
	if (cause > 0) {
		printk(KERN_WARNING "IP30: HEART ATTACK! ISR = 0x%.16llx "
		       "CAUSE = 0x%.16llx\n", (u64)(errors),(u64)(cause));

		/* i = 63; i >= 51; i-- */
		for (i = HEART_ERR_MASK_END; i >= HEART_ERR_MASK_START; i--)
			if ((errors >> i) & 1)
				printk(KERN_CONT "  HEART Error IRQ #%d\n", i);
	}
	irq_exit();
}

/**
* ip30_do_heart_irq - IRQ dispatch for all HEART L0, L1, L2 IRQs (0 - 49). */
static noinline void
ip30_do_heart_irq(void)

{
	int irqnum = 49;
	int cpu = smp_processor_id();
	u64 heart_isr = readq(HEART_ISR);
	u64 heart_imr = readq(HEART_IMR(cpu));
	u64 irqs = (heart_isr &
	            (HEART_INT_LEVEL0 | HEART_INT_LEVEL1 | HEART_INT_LEVEL2) &
	            heart_imr);

#ifdef DEBUG_IRQ
	bridge_t *bvma = (bridge_t *)RAW_NODE_SWIN_BASE(0, 15);

	printk(KERN_INFO "IP30: received HEART IRQs: 0x%016llx "
	                 "(mask 0x%016llx) CPU%d BRIDGE %08x\n",
	                 heart_isr, heart_imr, cpu, bvma->b_int_status);

#endif

	/* Poll all IRQs in decreasing priority order */
	do {
		if (irqs & (1UL << irqnum))
			do_IRQ(irqnum);
		irqnum--;
	} while (likely(irqnum >= 0));
}

/* ----------------------------------------------------------------------- */

/**
* plat_irq_dispatch - platform IRQ dispatch.
*/
asmlinkage void
plat_irq_dispatch(void)

{
	unsigned long pending;

	pending = (read_c0_cause() & read_c0_status() & ST0_IM);

	/* L5, CPU Counter/Compare */
	if (pending & CAUSEF_IP7)
		do_IRQ(MIPS_CPU_IRQ_BASE);
	/* L3, HEART Counter/Compare */
	else if (likely(pending & CAUSEF_IP5))
		do_IRQ(HEART_L3_INT_TIMER);
	/* L0-L2, HEART normal IRQs + IPI/SMP IRQs*/
	else if (likely(pending & (CAUSEF_IP2 | CAUSEF_IP3 | CAUSEF_IP4)))
		ip30_do_heart_irq();
	/* L4, HEART Errors */
	else if (unlikely(pending & CAUSEF_IP6))
		ip30_do_error_irq();
}

/* ----------------------------------------------------------------------- */

/* ----------------------------------------------------------------------- */ /* HEART IRQ Ops */

/**
* startup_heart_irq - assigns a HEART IRQ to a CPU and/or Bridge slot.
* @d: struct irq_data containing IRQ information.
*
* Returns 0 for now (XXX: this may need review, possible pending interrupts). */
static unsigned int
startup_heart_irq(struct irq_data *d)

{
	bridge_t *bridge;
	int cpu = smp_processor_id();
	u32 device, slot;
	u64 *imr;

	spin_lock(&heart_lock);

	if (heart_irq_owner[d->irq] != -1) {
		printk(KERN_ERR "DEBUG: startup_heart_irq: bad IRQ "
		       "startup request for IRQ %d on CPU %d "
		       "(already assigned to %d)!\n",
		       d->irq, cpu, heart_irq_owner[d->irq]);
		goto out;
	}

	/* store which CPU owns this IRQ */
	heart_irq_owner[d->irq] = cpu;

#ifdef DEBUG_IRQ
	printk(KERN_INFO "DEBUG: IP30: IRQ: startup_heart_irq: "
	       "IRQ %d on CPU %d!\n", d->irq, heart_irq_owner[d->irq]);

#endif

	/* clear IRQ flag */
	writeq(HEART_VEC_TO_IBIT(d->irq), HEART_CLR_ISR);

	/* unmask IRQ */
	imr = HEART_IMR(cpu);
	writeq(readq(imr) | HEART_VEC_TO_IBIT(d->irq), imr);

	/* Handle BRIDGE IRQs. */
	bridge = ip30_irq_to_bridge[d->irq];
	if (bridge) {
		slot = ip30_irq_to_slot[d->irq];
		bridge->b_int_enable |= (1 << slot);
		bridge->b_int_mode |= (1 << slot);
		device = bridge->b_int_device;
		device &= ~BRIDGE_INT_DEV_MASK(slot);
		device |=  BRIDGE_INT_DEV_SET(slot, slot);
		bridge->b_int_device = device;
		bridge->b_widget.w_tflush;
	}

out:

	spin_unlock(&heart_lock);

	/* XXX: This is probably not right; we could have pending irqs */
	return 0;
}

/**
* shutdown_heart_irq - removes a HEART IRQ from a CPU and/or Bridge slot.
* @d: struct irq_data containing IRQ information.
*/
static void
shutdown_heart_irq(struct irq_data *d)

{
	bridge_t *bridge;
	u64 *imr;

	spin_lock(&heart_lock);

	imr = HEART_IMR(heart_irq_owner[d->irq]);
	writeq(readq(imr) & ~(HEART_VEC_TO_IBIT(d->irq)), imr);

	bridge = ip30_irq_to_bridge[d->irq];
	if (bridge)
		bridge->b_int_enable &= ~(1UL << ip30_irq_to_slot[d->irq]);

	heart_irq_owner[d->irq] = -1;

	spin_unlock(&heart_lock);
}

/**
* ack_heart_irq - acks a HEART IRQ.
* @d: struct irq_data containing IRQ information. */
static void
ack_heart_irq(struct irq_data *d)
{

	spin_lock(&heart_lock);
	writeq(HEART_VEC_TO_IBIT(d->irq), HEART_CLR_ISR);
	spin_unlock(&heart_lock);
}

/**
* mask_heart_irq - masks a HEART IRQ.
* @d: struct irq_data containing IRQ information. */
static void
mask_heart_irq(struct irq_data *d)

{
	u64 *imr;

	spin_lock(&heart_lock);
	imr = HEART_IMR(heart_irq_owner[d->irq]);
	writeq(readq(imr) & ~(HEART_VEC_TO_IBIT(d->irq)), imr);
	spin_unlock(&heart_lock);
}

/**
* mask_and_ack_heart_irq - masks and acks a HEART IRQ.
* @d: struct irq_data containing IRQ information. */
static void
mask_and_ack_heart_irq(struct irq_data *d)

{
	u64 *imr;

	spin_lock(&heart_lock);
	writeq(HEART_VEC_TO_IBIT(d->irq), HEART_CLR_ISR);
	imr = HEART_IMR(heart_irq_owner[d->irq]);
	writeq(readq(imr) & ~(HEART_VEC_TO_IBIT(d->irq)), imr);
	spin_unlock(&heart_lock);
}

/**
* unmask_heart_irq - unmasks a HEART IRQ.
* @d: struct irq_data containing IRQ information. */
static void
unmask_heart_irq(struct irq_data *d)

{
	u64 *imr;

	spin_lock(&heart_lock);
	imr = HEART_IMR(heart_irq_owner[d->irq]);
	writeq(readq(imr) | HEART_VEC_TO_IBIT(d->irq), imr);
	spin_unlock(&heart_lock);
}

/**
* struct ip30_heart_irq - HEART struct irq_chip ops.
* @irq_startup: startup function.
* @irq_shutdown: shutdown function.
* @irq_ack: ack function.
* @irq_mask: mask function.
* @irq_mask_ack: mask & ack function.
* @irq_unmask: unmask function.
*/
static struct
irq_chip ip30_heart_irq = {
.name = "HEART",
.irq_startup = startup_heart_irq,
.irq_shutdown = shutdown_heart_irq,
.irq_ack = ack_heart_irq,
.irq_mask = mask_heart_irq,
.irq_mask_ack = mask_and_ack_heart_irq,
.irq_unmask = unmask_heart_irq,
};
/* ----------------------------------------------------------------------- */

static inline void ip30_unmask_boot_cpu_irq(struct irq_data *d)

{
	set_c0_status(0x100 << (d->irq - MIPS_CPU_IRQ_BASE));
}

static inline void ip30_mask_boot_cpu_irq(struct irq_data *d)

{
	clear_c0_status(0x100 << (d->irq - MIPS_CPU_IRQ_BASE));
}

static struct irq_chip ip30_boot_cpu_irq_controller = {
	.name		= "IP30-CPU0-IRQ",
	.irq_ack	= ip30_mask_boot_cpu_irq,
	.irq_mask	= ip30_mask_boot_cpu_irq,
	.irq_mask_ack	= ip30_mask_boot_cpu_irq,
	.irq_unmask	= ip30_unmask_boot_cpu_irq,
	.irq_eoi	= ip30_unmask_boot_cpu_irq,
};

/* ----------------------------------------------------------------------- */ /* Arch IRQ initialization - runs on CPU0 only. */

/**
* arch_init_irq - arch initialization function.
*/
void __init
arch_init_irq(void)

{
	int irq;

	/* Ack everything */
	writeq(HEART_ACK_ALL_MASK, HEART_CLR_ISR);

	/* Mask all IRQs */
	writeq(HEART_CLR_ALL_MASK, HEART_IMR(0));
	writeq(HEART_CLR_ALL_MASK, HEART_IMR(1));
	writeq(HEART_CLR_ALL_MASK, HEART_IMR(2));
	writeq(HEART_CLR_ALL_MASK, HEART_IMR(3));

	/* Leave HEART errors enabled */
	writeq(HEART_BR_ERR_MASK, HEART_IMR(0));

	for (irq = HEART_IRQ_BASE; irq < HEART_IRQS; irq++) {
		heart_irq_owner[irq] = -1;
		irq_set_chip_and_handler(irq, &ip30_heart_irq, handle_level_irq);
	}

	/* Init CPU0 IRQs */
//	mips_cpu_irq_init();
	irq_set_chip_and_handler(MIPS_CPU_IRQ_BASE,
				 &ip30_boot_cpu_irq_controller,
				 handle_percpu_irq);
	change_c0_status(ST0_IM, (STATUSF_IP2 | STATUSF_IP3 | STATUSF_IP4 |
	                          STATUSF_IP5 | STATUSF_IP6 | STATUSF_IP7));

	printk(KERN_INFO "IP30: HEART interrupt controller initialized.\n");
}

/* ----------------------------------------------------------------------- */

/*
* This file is subject to the terms and conditions of the GNU General Public
* License. See the file "COPYING" in the main directory of this archive
* for more details.
*
* ip30-smp.c: SMP on IP30 architecture.
*
* Copyright (C) 2005-2007 Stanislaw Skowronek <[email protected]>
* 2006-2007 Joshua Kinard <[email protected]>
* 2009 Johannes Dickgreber <[email protected]>
*/

#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/tick.h>
#include <linux/spinlock_types.h>

#include <asm/time.h>
#include <asm/cacheflush.h>

#include <asm/mach-ip30/heart.h>
#include <asm/mach-ip30/racermp.h>

static DEFINE_SPINLOCK(ip30_ipi_lock); //static DEFINE_SPINLOCK(ip30_timer_lock); static DEFINE_PER_CPU(int, ip30_ipi_action_mask); static DEFINE_PER_CPU(char[15], ip30_ipi_name); //static DEFINE_PER_CPU(char[17], ip30_timer_name);

//void ip30_clockevent_broadcast(const int cpu)
//{
//#ifdef CONFIG_SMP
// /* There can only ever be 2 CPUs in an Octane, so keep it simple. */
//#endif
//}

/* ----------------------------------------------------------------------- */ /* SMP IPI Ops */

/* Runs on CPUx and sends an IPI to CPUy. */
static void ip30_send_ipi_single(int cpu, unsigned int action) {
// unsigned long flags;

// printk(KERN_INFO "DEBUG: IP30: SMP: ip30_send_ipi_single CPU%d -> CPU%d: action: 0x%.8x\n", smp_processor_id(), cpu, action);

//	spin_lock_irqsave(&ip30_ipi_lock, flags);
	spin_lock(&ip30_ipi_lock);
	per_cpu(ip30_ipi_action_mask, cpu) |= action;
	writeq(HEART_VEC_TO_IBIT(HEART_L2_INT_IPI_CPU(cpu)), HEART_SET_ISR);
	spin_unlock(&ip30_ipi_lock);

// spin_unlock_irqrestore(&ip30_ipi_lock, flags);
}

/* IRQ handler that runs on CPUy and services the IPI. */ static irqreturn_t ip30_ipi_irq(int irq, void *dev_id)

{
	int action;
	int cpu = smp_processor_id();

// unsigned long flags;

//	spin_lock_irqsave(&ip30_ipi_lock, flags);
	spin_lock(&ip30_ipi_lock);
	action = __get_cpu_var(ip30_ipi_action_mask);
	per_cpu(ip30_ipi_action_mask, cpu) = 0;
	writeq(HEART_VEC_TO_IBIT(HEART_L2_INT_IPI_CPU(cpu)), HEART_CLR_ISR);
	spin_unlock(&ip30_ipi_lock);

// spin_unlock_irqrestore(&ip30_ipi_lock, flags);

// printk(KERN_INFO "DEBUG: IP30: SMP: ip30_ipi_irq: cpu%d: action: 0x%.8x\n", cpu, action);

	if (action & SMP_RESCHEDULE_YOURSELF)
		scheduler_ipi();

	if (action & SMP_CALL_FUNCTION)
		smp_call_function_interrupt();

	return IRQ_HANDLED;
}

///* IRQ handler that runs on CPUy and services the CPU Timer. */ //static irqreturn_t ip30_timer_irq(int irq, void *dev_id) //{
// int cpu = smp_processor_id();
// struct clock_event_device *cd;
//
// spin_lock(&ip30_timer_lock);
// cd = tick_get_device(cpu)->evtdev;
// cd->event_handler(cd);
//// printk(KERN_INFO "DEBUG: IP30: SMP: ip30_timer_irq: CPU%d: %s\n", cpu, cd->name);
// spin_unlock(&ip30_timer_lock);
// writeq(HEART_VEC_TO_IBIT(HEART_L2_INT_TIMER_CPU(cpu)), HEART_CLR_ISR);
//
// return IRQ_HANDLED;
//}

static void ip30_send_ipi_mask(const struct cpumask *mask, unsigned int action)

{
	u32 i;

	for_each_cpu(i, mask)
		ip30_send_ipi_single(i, action);
}

static void __init ip30_smp_setup(void)

{
	int i;
	int ncpu = 0;
	int vcpu;

	for (i = 0; i < NR_CPUS; i++) {
		if (readl(MP_MAGIC(i)) == MPCONF_MAGIC) {
			vcpu = readl(MP_VIRTID(i));
			__cpu_number_map[i] = vcpu;
			__cpu_logical_map[vcpu] = i;
			set_cpu_possible(i, true);
			set_cpu_present(i, true);
			printk(KERN_INFO "IP30: Slot: %d, PrID: %.8x, PhyID: %d, VirtID: %d\n",
			       i, readl(MP_PRID(i)), readl(MP_PHYSID(i)), vcpu);
			ncpu++;
		}
	}
	printk(KERN_INFO "IP30: Detected %d CPU(s) present.\n", ncpu);
}

/* Runs on CPU0 */
static void __init ip30_prepare_cpus(unsigned int max_cpus)

{
	int cpu = smp_processor_id();
	unsigned char *ipi_name = per_cpu(ip30_ipi_name, cpu);

// unsigned char *timer_name = per_cpu(ip30_timer_name, cpu);

	printk(KERN_INFO "DEBUG: IP30: SMP: ip30_prepare_cpus\n");

	/* Request an IRQ number for CPU0 IPI. */
	snprintf(ipi_name, 15, "ip30-cpu%d-ipi", cpu);
	if (request_irq(HEART_L2_INT_IPI_CPU(cpu), ip30_ipi_irq, IRQF_PERCPU,
	                ipi_name, NULL))
		panic("Can't request CPU%d IPI interrupt", cpu);

// snprintf(timer_name, 17, "ip30-cpu%d-timer", cpu);
// if (request_irq(HEART_L2_INT_TIMER_CPU(cpu), ip30_timer_irq, IRQF_PERCPU,
// timer_name, NULL))
// panic("Can't request CPU%d Broadcast Timer interrupt", cpu);

	/* Enable the IPI interrupt on CPU0 IP4. */

// change_c0_status(ST0_IM, STATUSF_IP4);
}

/* Runs on CPU0 and boots CPUx, where x > 0 */
static void ip30_boot_secondary(int cpu, struct task_struct *idle)

{
	int boot_cpu = smp_processor_id();

// struct thread_info *gp = task_thread_info(idle);

	printk(KERN_INFO "DEBUG: IP30: SMP: ip30_boot_secondary: CPU%d -> CPU%d\n", boot_cpu, cpu);

	/* Stack pointer (sp). */
	writeq(__KSTK_TOS(idle), MP_STACKADDR(cpu));

	/* Global pointer (gp). */
	writeq((unsigned long)task_thread_info(idle), MP_LPARM(cpu));

// writeq((unsigned long)gp, MP_LPARM(cpu));

	/* XXX: I don't know if this is needed or not.  Copied from elsewhere. */

// flush_icache_range((unsigned long)gp,
// (unsigned long)(gp + sizeof(struct thread_info)));

	/* Boot CPU1. */
	writeq((unsigned long)ip30_smp_bootstrap, MP_LAUNCH(cpu));

// mb();

	/* XXX: This is needed to "kick" the CPU awake.  Bug? */
	pr_info("SMP: Booting CPU%d...\n", cpu);
}

/* Runs on CPUx, where x > 0 */
static void ip30_init_secondary(void)

{
	int cpu = smp_processor_id();

	printk(KERN_INFO "DEBUG: IP30: SMP: ip30_init_secondary on CPU%d\n", cpu);

}

static inline void ip30_unmask_2nd_cpu_irq(struct irq_data *d)

{
	set_c0_status(0x100 << (d->irq - (MIPS_CPU_IRQ_BASE + 1)));
}

static inline void ip30_mask_2nd_cpu_irq(struct irq_data *d)

{
	clear_c0_status(0x100 << (d->irq - (MIPS_CPU_IRQ_BASE + 1)));
}

static struct irq_chip ip30_2nd_cpu_irq_controller = {
	.name		= "IP30-CPU1-IRQ",
	.irq_ack	= ip30_mask_2nd_cpu_irq,
	.irq_mask	= ip30_mask_2nd_cpu_irq,
	.irq_mask_ack	= ip30_mask_2nd_cpu_irq,
	.irq_unmask	= ip30_unmask_2nd_cpu_irq,
	.irq_eoi	= ip30_unmask_2nd_cpu_irq,
};

static void ip30_smp_finish(void)

{
	int cpu = smp_processor_id();
	unsigned char *ipi_name = per_cpu(ip30_ipi_name, cpu);

// unsigned char *timer_name = per_cpu(ip30_timer_name, cpu);

	printk(KERN_INFO "DEBUG: IP30: SMP: ip30_smp_finish: CPU%d\n", cpu);

	irq_set_chip_and_handler((MIPS_CPU_IRQ_BASE + 1),
				 &ip30_2nd_cpu_irq_controller,
				 handle_percpu_irq);
//	write_c0_compare(read_c0_count() + mips_hpt_frequency / HZ);
	change_c0_status(ST0_IM, (STATUSF_IP2 | STATUSF_IP3 | STATUSF_IP4 |
	                          STATUSF_IP5 | STATUSF_IP6 | STATUSF_IP7));
	local_irq_enable();

	/* Request an IRQ number for CPU1 IPI. */
	snprintf(ipi_name, 15, "ip30-cpu%d-ipi", cpu);
	if (request_irq(HEART_L2_INT_IPI_CPU(cpu), ip30_ipi_irq, IRQF_PERCPU,
	                ipi_name, NULL))
		panic("Can't request CPU%d IPI interrupt", cpu);

// snprintf(timer_name, 17, "ip30-cpu%d-timer", cpu);
// if (request_irq(HEART_L2_INT_TIMER_CPU(cpu), ip30_timer_irq, IRQF_PERCPU,
// timer_name, NULL))
// panic("Can't request CPU%d Broadcast Timer interrupt", cpu);

	pr_info("SMP: CPU%d is running\n", smp_processor_id());
}

struct plat_smp_ops ip30_smp_ops = {
	.send_ipi_single	= ip30_send_ipi_single,
	.send_ipi_mask		= ip30_send_ipi_mask,
	.init_secondary		= ip30_init_secondary,
	.smp_finish		= ip30_smp_finish,
	.boot_secondary		= ip30_boot_secondary,
	.smp_setup		= ip30_smp_setup,
	.prepare_cpus		= ip30_prepare_cpus,
};

I take it no one has any feedback or tips on this?

I think one of the problems is I'm not syncing the CPU timers (IP7) correctly.
The old IP30 SMP code used a timer broadcast trick yo do this, sharing a
single IRQ, #63. However, 63 is one of the hardware error IRQs. Still not sure how that ever worked.

I've tried using the sync-r4k module...that just hangs in the sync function. atomics seemed messed up (I wonder if PR61538 has something to do...). Also tried re-implementing the timer broadcast but that just hangs because the two CPUs get into a deadlock situation w/ each trying to tell the other about the timer broadcast event. Not even sure if I should be using spin_lock/spin_unlock or spin_lock_irqsave/spin_unlock_irqrestore for the HEART irq code or SMP IRQ code. Each MIPS SMP machine seems to use a completely different mechanism in the kernel. IP27 doesn't even enable the CPU timer IRQs, it looks, and relies solely on the HUB timer present on each nodeboard.

So, yeah, out of ideas.

Joshua Kinard
Gentoo/MIPS
[email protected]
4096R/D25D95E3 2011-03-28

"The past tempts us, the present confuses us, the future frightens us.  And our
lives slip away, moment by moment, lost in that vast, terrible in-between."

--Emperor Turhan, Centauri Republic

What is wrong with these stack addresses? This is the result of disabling CPU1 in the PROM and booting an SMP kernel. It's like both the low 32-bits and high 32-bits of the data in the CPU registers are getting merged together somehow when they're added to the stack.

I can't think of anything in Octane's code doing this. Has anyone seen something like this before? This is likely the cause of the SIGSEGV/SIGBUS signals I keep getting.

CPU: 0 PID: 54 Comm: grep Not tainted 3.18.0-rc4 #194
task: a800000059b80000 ti: a8000000595c4000 task.ti: a8000000595c4000
$ 0   : 0000000000000000 ffffffff9004fce0 ffffffffffffffff ffffffffffffffff
$ 4   : 0000000077d809a0 0000000000000000 ffffffffffffffff ffffffffffffffff
$ 8   : 0000000000440f14 0000000000439bdc 0000000000000058 0000000000000000
$12   : 0000000000000000 a800000059b88aa0 a8000000200d22d0 001c450400000018
$16   : 0000000077d809a0 0000000077e3f000 0000000000000000 0000000000000000
$20   : 0000000077d803b4 0000000000000001 0000000077d82604 000000007ff808f8
$24   : 0018460400000000 0000000077c7aa90
$28   : 0000000077d88e10 000000007ff807c0 000000007ff807c0 0000000077c68bdc
Hi    : 0000000000061170
Lo    : 00000000000205d0
epc   : 0000000077c7ab00 0x77c7ab00
    Not tainted
ra    : 0000000077c68bdc 0x77c68bdc
Status: 8004fcf3    KX SX UX KERNEL EXL IE
Cause : 00000018
PrId  : 00000f24 (R14000)
Process grep (pid: 54, threadinfo=a8000000595c4000, task=a800000059b80000, tls=0000000077e46490)
Stack : 0000000000000000 77d88e1077d809a0 77d88e1077d809a0 77e3f00077d809a0
        7ff807e877c68bdc 0000000000000000 0000000000000009 77d88e1000000001
        0000000000000000 0000000200000000 7ff808700041e7a4 0000000000000000
        0000003d202fbf00 0043f0d07ff80830 0000003d00000003 00000004004134f4
        77d88e1000000000 0000000300000004 0000000200000000 0043f0d000000001
        0000000200000003 0000000477c34698 0043000000424fd0 000000027ff808f8
        77d88e1077c29488 000000007ff80ae4 0000000200000000 7ff80fc600430000
        00424fd000000002 7ff808b077c34748 000000027ff80fc6 0043000000424fd0
        77d88e107ff808f8 77d8012800403788 0000000077d5638c 0000000077e4028c
        000000007ff808e8 0000000000000000 0043f0d077e101dc 0000000100000000
        ...
Call Trace:
 (Bad stack address)

Code: 30420040  5040000a  82020046  <03c0e821>  8f998750  00002821  8fbf0024  02002021  8fbe0020

--J

Is `grep' a 64-bit (n64 or n32) process? If no then, there is nothing
wrong here, 32-bit (o32) processes will store registers on the stack as 32-bit quantities. I doubt that has anything to do with SIGSEGV/SIGBUS.

There is definitely something wrong here though, the contents of
registers include pointers to the kernel-only XKPHYS memory segment ($13 and $14) that shouldn't have leaked from the kernel, so it looks to me like the user context isn't handled correctly. Of course any attempt to dereference these pointers will cause an exception and in the response the process will be treated with an appropriate signal, and, usually, killed.

Maciej

This is an o32 userland. So that means, given 64-bit wide registers, o32 is going to stuff two 32-bit quantities into them? I have an n32 chroot on a different partition, but I haven't tried it w/ CONFIG_SMP yet.

Of the two XKPHYS addresses, a8000000200d22d0 points directly at SyS_munmap. Couple other crashes pointed at compat_SyS_fcntl64, as well as a few other addresses in XKPHYS that I couldn't find a specific function for in System.map.
Seems to be random leakage.

I thought it might be improper use of spinlocks (w/ & w/o irqsave/irqrestore) in the IRQ code, but I commented out all spinlocks in the core IP30 code, then after still triggering fatal crashes, commented out all of the spinlocks in IOC3 (evil driver) and Impact (video driver). For a while, I couldn't crash the kernel until I uncommented Impact's spinlocks, but it looks like that was a fluke after subsequently commenting them out again and still crashing. I'll probably swap in the Odyssey board in the next day or so and see if that exhibits similar problems, just to rule out the framebuffer code/drivers. Then pull more memory. Then my hair...

--J

[...]

No, user registers have to be treated as 32-bit, that is sign-extended
from bit #31 (bits 63:32 are a copy of bit #31). Using 32-bit operations only guarantees that, e.g. when you load a register from its stack slot using the 32-bit LW instruction, then on a 64-bit processor it will get sign-extended in the hardware register from bit #31 through to bit #63.

Of course an o32 bit program does not see it, it sees the environment as
it would on a 32-bit processor as it is supposed to run the same on a true 32-bit processor. Well, strictly speaking this is not completely true on Linux, for that to be the case the UX bit you see set in the Status register dumped above would have to be cleared, but this is a historical artefact and nobody has had the incentive to clean this up yet (in a reference environment you want UX clear for o32, UX set for n64 and PX set for n32 where available, otherwise UX set). Clearing the UX bit disables all the 64-bit instructions in the user mode making a user program unable to see or modify the upper 32 bits of any general register (they're still sign-extended automatically).

Please do yourself a favour and read a good MIPS architecture book;
Dominic Sweetman's "See MIPS Run" would be my recommendation and its second edition has some focus on Linux too that might help. Without a good understanding of the architecture you'll be having a very, very hard time debugging such low-level issues.

Maciej

[...]

I've been aware of the sign-extending bit for some time. Hence the 0xffffffffxxxxxxxx-style addresses. Regardless if the stack contains 77abcdef77f00ba7, an LW insn loading that value will actually store ffffffff77f00ba7 into the hardware register, right?

It just seemed odd that from the 64-bit stack addresses, it looked like there were two 32-bit virtual addresses (77abcdef and 77f00ba7). I wasn't sure if that was normal behavior or not for 64-bit Linux/MIPS. Or a symptom of whatever bug is present in the IP30 code I've yet to find.

A lot of the binaries on the Octane's filesystem are compiled w/ MIPS-IV. Under o32, I know that doesn't allow for use of 64bit instructions, but I do believe that o32 can still make use of newer instructions that were added in the MIPS-III or MIPS-IV ISA (iirc, conditional move was one of those).

The problem seems to be, though, that there's some kind of subtle memory corruption happening when I compile an IP30 kernel w/ CONFIG_SMP. Without CONFIG_SMP, it runs fine. I was focusing on the spinlocks because I know that without CONFIG_SMP, they compile away into nothing, but with CONFIG_SMP, even on a uniprocessor machine, they'll convert to instructions and can still highlight locking problems in the code.

It's just the question of where in the code is the corruption happening? I may not fully comprehend all the low-level bit-frobbing that goes on, but once I get an idea of the code region causing it, I can usually figure something out.
Problem is, like IP27, the serial ports are behind the IOC3 device, and the
old IOC3/kgdb interface code was removed in ~2008 (commit 8d60a903) and the new kgdb interface code doesn't work w/ IOC3 (NULL pointer deref). I've got a rough idea on how to talk to the UART chip behind the IOC3 to make KGDB work again, just not a good enough understanding of the serial-8250 core to know how to add it. And then I have to hope I don't end up debugging the code added to debug the code...

So, for now, my post-analysis crash-tool is a Canon DSLR camera mounted on a tripod to capture whatever debug data fits onto a single 1024x768 screen.

I've had a copy of the 2nd edition for a few years now. It's an excellent general reference, though I admit I've skipped over the TLB chapter a few times. Hopefully a third edition comes out at some point and has more in-depth coverage of Linux running on MIPS, in addition to the general MIPS overview.

I've also got Robert Love's Linux Kernel Development, 3rd edition, but that's 2010 and is from the 2.6.x era. I don't know of anything newer for the 3.x era.

--J

Nope, it'll be either 0x0000000077f00ba7 or 0x0000000077abcdef, depending
on the endianness. Bit #31 is clear here and so will be bits #63:32. All user software must have these bits clear in addresses as KUSEG spans 0x00000000-0x7fffffff.

This is 32-bit user code, it's (approximately) unaware it's running on a
64-bit OS.

Nope, not exactly right. You can indeed make use of MIPS IV ISA
additions, that are address size agnostic, however you cannot make use of any MIPS III additions as all of these depend on the 64-bit mode: integer operations require CP0.Status.UX to be set that is supposed not to be for o32 user software and floating-point additions require CP0.Status.FR to be set that is likewise supposed not to be for o32 user software.

Additionally on some processors, including the R10k family you are
concerned about, CP0.Status.XX has to be set for MIPS IV ISA additions to be enabled, but I believe it is under Linux.

I highly doubt spinlocks have any significance here, they're used and
work just fine across many systems. If anything this will probably be either a bug in platform code somewhere or a critical part of hardware having not been correctly initialised.

Does the system have any standard bus like PCI? If so then you could get
an NVRAM card then and log some activity there like CPU status on entry to exception handlers. Once a crash has happened you could boot with that logging disabled and retrieve your data. Accessing hardware is easy on MIPS, you can do it via XKPHYS without a need to have the MMU working, IOW you'll be able to poke at hardware even if your TLB/page tables got botched for some reason. And you can bypass the cache too, which is another possible place for breakage.

Of course if you have PCI then you can add an ordinary serial port card
there as well if the onboard port is difficult to access for some reason, but serial port logging has its limitations, mainly the complexity to access it and throughput.

Given the symptoms I'd first take a look at context switching code
though, you may be able to get away without fancy tools.

I don't think it will, by now Dominic has retired. But I don't think you
need anything beyond that, there's nothing specific about the MIPS port of Linux that wouldn't be covered by a generic Linux book, except from the processor architecture itself -- and you've got it covered there. For the rest, like newer stuff, just study the sources.

Maciej

Yes. The XX bit is primarily about the additional FPU condition bits. An OS needs to be aware of those or branch emulation won't work right. Linux supports that - always has - even on 32 bit kernels.

For testing purposes one could disable the secondary CPU - I believe that's possible on IP30, too? Then build an SMP kernel with NR_CPUS set to 1. That's basically a glorified uniprocessor kernel on glorified uniprocessor hardware then.

IP27 reserves a part of its FLASH memory for logging. However Linux doesn't support that.

While the system has 16550 UARTs a PCI card might indeed make things slightly sinpler - the setup of the IOC3 + SuperIO combo is complex, even for a simple PIO driver. However I think he shouldn't have to go to such an extreme!

See MIPS Run is actually one of the few CS books that are fun to read. And as a new Britton at the time I also learned a lot about fish & chips shops!

Ralf

That's what I did, actually. "disable 1" in the PROM and a reset to turn off the extra CPU. The register dump in my earlier mail in this thread is from an SMP kernel running with CPU1 disabled in PROM. NR_CPUS=2 in that build, though, but Octane has some data available via MP_CONF registers that can tell you if a CPU is online or offline, and the SMP setup code uses that to enumerate the possible CPUs.

I can still trigger the corrupted memory addresses in that situation. Which is why I'm thinking there is something else, possibly not in the core IP30 code, that's causing the problem. I've stripped the test kernel of everything else (no block drivers, no SCSI, no networking, etc). I guess I can remove PCI & IOC3 support next and see if loading an initramfs triggers the memory corruption. Might have to look closer at the memory probing code, too.

I did add a different RTC to my Octane, DS17887, which has 8KB of NVRAM available. The driver I wrote for it can access that NVRAM, too. Uses PIO to write an address to a port register and then reads a data register to get data in/out from the RTC (unlike O2, which can ioremap the RTC registers directly).
Can't store much in 8KB, though. I'll look for an NVRAM card in that case then.

Is it possible for Linux, upon a kernel crash, to actually create a full dump of all available memory and write that out somewhere? NetWare had this capability to designate a spare volume as a crash volume, which a core dump could be written to using low-level access, and then you could do offline analysis of the dump via the NW kernel debugger on a separate workstation (after rebooting and copying the crash dump out).

Already got a PCI serial card installed in the PCI 'shoebox' module. Unfortunately, it's a Moschip, so the driver for that is part of the parallel port code, and that doesn't appear to be big-endian safe. Last time I tried the driver, it crashed when probing for the card. Seems most of the cheap PCI Serial card these days are Moschips.

--J

Joshua Kinard
Gentoo/MIPS
[email protected]
4096R/D25D95E3 2011-03-28

"The past tempts us, the present confuses us, the future frightens us.  And our
lives slip away, moment by moment, lost in that vast, terrible in-between."

--Emperor Turhan, Centauri Republic