diff -urpN -X /home/fletch/.diff.exclude 220-thread_info_cleanup/arch/i386/Kconfig 221-interrupt_stacks/arch/i386/Kconfig
--- 220-thread_info_cleanup/arch/i386/Kconfig Wed Jul 2 22:02:16 2003
+++ 221-interrupt_stacks/arch/i386/Kconfig Wed Jul 2 22:02:19 2003
@@ -397,6 +397,11 @@ config X86_OOSTORE
depends on MWINCHIP3D || MWINCHIP2 || MWINCHIPC6
default y
+config X86_CMOV
+ bool
+ depends on M686 || MPENTIUMII || MPENTIUMIII || MPENTIUM4 || MK8 || MCRUSOE
+ default y
+
config HUGETLB_PAGE
bool "Huge TLB Page Support"
help
diff -urpN -X /home/fletch/.diff.exclude 220-thread_info_cleanup/arch/i386/kernel/apic.c 221-interrupt_stacks/arch/i386/kernel/apic.c
--- 220-thread_info_cleanup/arch/i386/kernel/apic.c Wed Jul 2 21:59:03 2003
+++ 221-interrupt_stacks/arch/i386/kernel/apic.c Wed Jul 2 22:02:19 2003
@@ -1037,7 +1037,8 @@ inline void smp_local_timer_interrupt(st
* interrupt as well. Thus we cannot inline the local irq ... ]
*/
-void smp_apic_timer_interrupt(struct pt_regs regs)
+struct pt_regs * IRQHANDLER(smp_apic_timer_interrupt(struct pt_regs* regs));
+struct pt_regs * smp_apic_timer_interrupt(struct pt_regs* regs)
{
int cpu = smp_processor_id();
@@ -1057,14 +1058,16 @@ void smp_apic_timer_interrupt(struct pt_
* interrupt lock, which is the WrongThing (tm) to do.
*/
irq_enter();
- smp_local_timer_interrupt(®s);
+ smp_local_timer_interrupt(regs);
irq_exit();
+ return regs;
}
/*
* This interrupt should _never_ happen with our APIC/SMP architecture
*/
-asmlinkage void smp_spurious_interrupt(void)
+struct pt_regs * IRQHANDLER(smp_spurious_interrupt(struct pt_regs* regs));
+struct pt_regs * smp_spurious_interrupt(struct pt_regs* regs)
{
unsigned long v;
@@ -1082,13 +1085,15 @@ asmlinkage void smp_spurious_interrupt(v
printk(KERN_INFO "spurious APIC interrupt on CPU#%d, should never happen.\n",
smp_processor_id());
irq_exit();
+ return regs;
}
/*
* This interrupt should never happen with our APIC/SMP architecture
*/
-asmlinkage void smp_error_interrupt(void)
+struct pt_regs * IRQHANDLER(smp_error_interrupt(struct pt_regs* regs));
+struct pt_regs * smp_error_interrupt(struct pt_regs* regs)
{
unsigned long v, v1;
@@ -1113,6 +1118,7 @@ asmlinkage void smp_error_interrupt(void
printk (KERN_INFO "APIC error on CPU%d: %02lx(%02lx)\n",
smp_processor_id(), v , v1);
irq_exit();
+ return regs;
}
/*
diff -urpN -X /home/fletch/.diff.exclude 220-thread_info_cleanup/arch/i386/kernel/cpu/mcheck/p4.c 221-interrupt_stacks/arch/i386/kernel/cpu/mcheck/p4.c
--- 220-thread_info_cleanup/arch/i386/kernel/cpu/mcheck/p4.c Sun Apr 20 19:34:56 2003
+++ 221-interrupt_stacks/arch/i386/kernel/cpu/mcheck/p4.c Wed Jul 2 22:02:19 2003
@@ -61,11 +61,13 @@ static void intel_thermal_interrupt(stru
/* Thermal interrupt handler for this CPU setup */
static void (*vendor_thermal_interrupt)(struct pt_regs *regs) = unexpected_thermal_interrupt;
-asmlinkage void smp_thermal_interrupt(struct pt_regs regs)
+struct pt_regs * IRQHANDLER(smp_thermal_interrupt(struct pt_regs* regs));
+struct pt_regs * smp_thermal_interrupt(struct pt_regs* regs)
{
irq_enter();
vendor_thermal_interrupt(®s);
irq_exit();
+ return regs;
}
/* P4/Xeon Thermal regulation detect and init */
diff -urpN -X /home/fletch/.diff.exclude 220-thread_info_cleanup/arch/i386/kernel/entry.S 221-interrupt_stacks/arch/i386/kernel/entry.S
--- 220-thread_info_cleanup/arch/i386/kernel/entry.S Wed Jul 2 22:02:18 2003
+++ 221-interrupt_stacks/arch/i386/kernel/entry.S Wed Jul 2 22:02:19 2003
@@ -394,17 +394,78 @@ ENTRY(irq_entries_start)
vector=vector+1
.endr
+
+# lets play optimizing compiler...
+#ifdef CONFIG_X86_CMOV
+#define COND_MOVE cmovnz %esi,%esp;
+#else
+#define COND_MOVE \
+ jz 1f; \
+ mov %esi,%esp; \
+1:
+#endif
+
+# These macros will switch you to, and from a per-cpu interrupt stack
+# They take the pt_regs arg and move it from the normal place on the
+# stack to %eax. Any handler function can retrieve it using regparm(1).
+# The handlers are expected to return the stack to switch back to in
+# the same register.
+#
+# This means that the irq handlers need to return their arg
+#
+# SWITCH_TO_IRQSTACK clobbers %ebx, %ecx, %edx, %esi
+# old stack gets put in %eax
+
+.macro SWITCH_TO_IRQSTACK
+ GET_THREAD_INFO(%ebx);
+ movl TI_IRQ_STACK(%ebx),%ecx;
+ movl TI_TASK(%ebx),%edx;
+ movl %esp,%eax;
+
+ # %ecx+THREAD_SIZE is next stack -4 keeps us in the right one
+ leal (THREAD_SIZE-4)(%ecx),%esi;
+
+ # is there a valid irq_stack?
+ testl %ecx,%ecx;
+ COND_MOVE;
+
+ # update the task pointer in the irq stack
+ GET_THREAD_INFO(%esi);
+ movl %edx,TI_TASK(%esi);
+
+ # update the preempt count in the irq stack
+ movl TI_PRE_COUNT(%ebx),%ecx;
+ movl %ecx,TI_PRE_COUNT(%esi);
+.endm
+
+# copy flags from the irq stack back into the task's thread_info
+# %esi is saved over the irq handler call and contains the irq stack's
+# thread_info pointer
+# %eax was returned from the handler, as described above
+# %ebx contains the original thread_info pointer
+
+.macro RESTORE_FROM_IRQSTACK
+ movl %eax,%esp;
+ movl TI_FLAGS(%esi),%eax;
+ movl $0,TI_FLAGS(%esi);
+ LOCK orl %eax,TI_FLAGS(%ebx);
+.endm
+
ALIGN
common_interrupt:
SAVE_ALL
+ SWITCH_TO_IRQSTACK
call do_IRQ
+ RESTORE_FROM_IRQSTACK
jmp ret_from_intr
#define BUILD_INTERRUPT(name, nr) \
ENTRY(name) \
pushl $nr-256; \
SAVE_ALL \
- call smp_/**/name; \
+ SWITCH_TO_IRQSTACK; \
+ call smp_/**/name; \
+ RESTORE_FROM_IRQSTACK; \
jmp ret_from_intr;
/* The include is where all of the SMP etc. interrupts come from */
diff -urpN -X /home/fletch/.diff.exclude 220-thread_info_cleanup/arch/i386/kernel/init_task.c 221-interrupt_stacks/arch/i386/kernel/init_task.c
--- 220-thread_info_cleanup/arch/i386/kernel/init_task.c Thu Feb 13 11:08:02 2003
+++ 221-interrupt_stacks/arch/i386/kernel/init_task.c Wed Jul 2 22:02:19 2003
@@ -14,6 +14,10 @@ static struct signal_struct init_signals
static struct sighand_struct init_sighand = INIT_SIGHAND(init_sighand);
struct mm_struct init_mm = INIT_MM(init_mm);
+union thread_union init_irq_union
+ __attribute__((__section__(".data.init_task")));
+
+
/*
* Initial thread structure.
*
diff -urpN -X /home/fletch/.diff.exclude 220-thread_info_cleanup/arch/i386/kernel/irq.c 221-interrupt_stacks/arch/i386/kernel/irq.c
--- 220-thread_info_cleanup/arch/i386/kernel/irq.c Wed Jul 2 22:02:08 2003
+++ 221-interrupt_stacks/arch/i386/kernel/irq.c Wed Jul 2 22:02:19 2003
@@ -403,7 +403,8 @@ void enable_irq(unsigned int irq)
* SMP cross-CPU interrupts have their own specific
* handlers).
*/
-asmlinkage unsigned int do_IRQ(struct pt_regs regs)
+struct pt_regs * IRQHANDLER(do_IRQ(struct pt_regs *regs));
+struct pt_regs * do_IRQ(struct pt_regs *regs)
{
/*
* We ack quickly, we don't want the irq controller
@@ -415,7 +416,7 @@ asmlinkage unsigned int do_IRQ(struct pt
* 0 return value means that this irq is already being
* handled by some other CPU. (or is disabled)
*/
- int irq = regs.orig_eax & 0xff; /* high bits used in ret_from_ code */
+ int irq = regs->orig_eax & 0xff; /* high bits used in ret_from_ code */
int cpu = smp_processor_id();
irq_desc_t *desc = irq_desc + irq;
struct irqaction * action;
@@ -482,7 +483,7 @@ asmlinkage unsigned int do_IRQ(struct pt
irqreturn_t action_ret;
spin_unlock(&desc->lock);
- action_ret = handle_IRQ_event(irq, ®s, action);
+ action_ret = handle_IRQ_event(irq, regs, action);
spin_lock(&desc->lock);
if (!noirqdebug)
note_interrupt(irq, desc, action_ret);
@@ -502,7 +503,7 @@ out:
irq_exit();
- return 1;
+ return regs;
}
/**
diff -urpN -X /home/fletch/.diff.exclude 220-thread_info_cleanup/arch/i386/kernel/process.c 221-interrupt_stacks/arch/i386/kernel/process.c
--- 220-thread_info_cleanup/arch/i386/kernel/process.c Tue Jun 24 21:29:16 2003
+++ 221-interrupt_stacks/arch/i386/kernel/process.c Wed Jul 2 22:02:19 2003
@@ -449,6 +449,7 @@ struct task_struct * __switch_to(struct
/* never put a printk in __switch_to... printk() calls wake_up*() indirectly */
+ next_p->thread_info->irq_stack = prev_p->thread_info->irq_stack;
unlazy_fpu(prev_p);
/*
diff -urpN -X /home/fletch/.diff.exclude 220-thread_info_cleanup/arch/i386/kernel/smp.c 221-interrupt_stacks/arch/i386/kernel/smp.c
--- 220-thread_info_cleanup/arch/i386/kernel/smp.c Wed Jul 2 22:02:16 2003
+++ 221-interrupt_stacks/arch/i386/kernel/smp.c Wed Jul 2 22:02:19 2003
@@ -305,7 +305,8 @@ static inline void leave_mm (unsigned lo
* 2) Leave the mm if we are in the lazy tlb mode.
*/
-asmlinkage void smp_invalidate_interrupt (void)
+struct pt_regs * IRQHANDLER(smp_invalidate_interrupt(struct pt_regs *regs));
+struct pt_regs * smp_invalidate_interrupt(struct pt_regs *regs)
{
unsigned long cpu;
@@ -336,6 +337,7 @@ asmlinkage void smp_invalidate_interrupt
out:
put_cpu_no_resched();
+ return regs;
}
static void flush_tlb_others (unsigned long cpumask, struct mm_struct *mm,
@@ -581,12 +583,15 @@ void smp_send_stop(void)
* all the work is done automatically when
* we return from the interrupt.
*/
-asmlinkage void smp_reschedule_interrupt(void)
+struct pt_regs * IRQHANDLER(smp_reschedule_interrupt(struct pt_regs *regs));
+struct pt_regs * smp_reschedule_interrupt(struct pt_regs *regs)
{
ack_APIC_irq();
+ return regs;
}
-asmlinkage void smp_call_function_interrupt(struct pt_regs regs)
+struct pt_regs * IRQHANDLER(smp_call_function_interrupt(struct pt_regs *regs));
+struct pt_regs * smp_call_function_interrupt(struct pt_regs *regs)
{
void (*func) (void *info, struct pt_regs *) = (void (*)(void *, struct pt_regs*))call_data->func;
void *info = call_data->info;
@@ -610,5 +615,6 @@ asmlinkage void smp_call_function_interr
mb();
atomic_inc(&call_data->finished);
}
+ return regs;
}
diff -urpN -X /home/fletch/.diff.exclude 220-thread_info_cleanup/arch/i386/kernel/smpboot.c 221-interrupt_stacks/arch/i386/kernel/smpboot.c
--- 220-thread_info_cleanup/arch/i386/kernel/smpboot.c Wed Jul 2 22:01:57 2003
+++ 221-interrupt_stacks/arch/i386/kernel/smpboot.c Wed Jul 2 22:02:19 2003
@@ -71,6 +71,11 @@ static unsigned long smp_commenced_mask;
/* Per CPU bogomips and other parameters */
struct cpuinfo_x86 cpu_data[NR_CPUS] __cacheline_aligned;
+/* Per CPU interrupt stacks */
+extern union thread_union init_irq_union;
+union thread_union *irq_stacks[NR_CPUS] __cacheline_aligned =
+ { &init_irq_union, };
+
/* Set when the idlers are all forked */
int smp_threads_ready;
@@ -770,6 +775,28 @@ wakeup_secondary_cpu(int phys_apicid, un
}
#endif /* WAKE_SECONDARY_VIA_INIT */
+static void __init setup_irq_stack(struct task_struct *p, int cpu)
+{
+ unsigned long stk;
+
+ stk = __get_free_pages(GFP_KERNEL, THREAD_ORDER);
+ if (!stk)
+ panic("I can't seem to allocate my irq stack. Oh well, giving up.");
+
+ irq_stacks[cpu] = (void *)stk;
+ memset(irq_stacks[cpu], 0, THREAD_SIZE);
+ irq_stacks[cpu]->thread_info.cpu = cpu;
+ irq_stacks[cpu]->thread_info.preempt_count = 1;
+ /* interrupts are not preemptable */
+ p->thread_info->irq_stack = &irq_stacks[cpu]->thread_info;
+
+ /* If we want to make the irq stack more than one unit
+ * deep, we can chain then off of the irq_stack pointer
+ * here.
+ */
+}
+
+
extern unsigned long cpu_initialized;
static int __init do_boot_cpu(int apicid)
@@ -793,6 +820,7 @@ static int __init do_boot_cpu(int apicid
idle = fork_by_hand();
if (IS_ERR(idle))
panic("failed fork for CPU %d", cpu);
+ setup_irq_stack(idle, cpu);
wake_up_forked_process(idle);
/*
diff -urpN -X /home/fletch/.diff.exclude 220-thread_info_cleanup/include/asm-i386/linkage.h 221-interrupt_stacks/include/asm-i386/linkage.h
--- 220-thread_info_cleanup/include/asm-i386/linkage.h Sun Nov 17 20:29:46 2002
+++ 221-interrupt_stacks/include/asm-i386/linkage.h Wed Jul 2 22:02:19 2003
@@ -3,6 +3,7 @@
#define asmlinkage CPP_ASMLINKAGE __attribute__((regparm(0)))
#define FASTCALL(x) x __attribute__((regparm(3)))
+#define IRQHANDLER(x) x __attribute__((regparm(1)))
#ifdef CONFIG_X86_ALIGNMENT_16
#define __ALIGN .align 16,0x90
diff -urpN -X /home/fletch/.diff.exclude 220-thread_info_cleanup/include/asm-i386/thread_info.h 221-interrupt_stacks/include/asm-i386/thread_info.h
--- 220-thread_info_cleanup/include/asm-i386/thread_info.h Wed Jul 2 22:02:18 2003
+++ 221-interrupt_stacks/include/asm-i386/thread_info.h Wed Jul 2 22:02:19 2003
@@ -31,9 +31,11 @@ struct thread_info {
__s32 preempt_count; /* 0 => preemptable, <0 => BUG */
mm_segment_t addr_limit; /* thread address space:
+ 0 for interrupts: illegal
0-0xBFFFFFFF for user-thead
0-0xFFFFFFFF for kernel-thread
*/
+ struct thread_info *irq_stack; /* pointer to cpu irq stack */
struct restart_block restart_block;
__u8 supervisor_stack[0];
@@ -49,7 +51,8 @@ struct thread_info {
#define TI_CPU 0x00000010
#define TI_PRE_COUNT 0x00000014
#define TI_ADDR_LIMIT 0x00000018
-#define TI_RESTART_BLOCK 0x000001C
+#define TI_IRQ_STACK 0x0000001C
+#define TI_RESTART_BLOCK 0x0000026
#endif
@@ -65,17 +68,18 @@ struct thread_info {
#ifndef __ASSEMBLY__
-#define INIT_THREAD_INFO(tsk) \
-{ \
- .task = &tsk, \
- .exec_domain = &default_exec_domain, \
- .flags = 0, \
- .cpu = 0, \
- .preempt_count = 1, \
- .addr_limit = KERNEL_DS, \
- .restart_block = { \
- .fn = do_no_restart_syscall, \
- }, \
+#define INIT_THREAD_INFO(tsk) \
+{ \
+ .task = &tsk, \
+ .exec_domain = &default_exec_domain, \
+ .flags = 0, \
+ .cpu = 0, \
+ .preempt_count = 1, \
+ .addr_limit = KERNEL_DS, \
+ .irq_stack = &init_irq_union.thread_info, \
+ .restart_block = { \
+ .fn = do_no_restart_syscall, \
+ } \
}
#define init_thread_info (init_thread_union.thread_info)