From: Andi Kleen Turn per cpu TSS into per cpu data. Signed-off-by: Andi Kleen Signed-off-by: Andrew Morton --- 25-akpm/arch/x86_64/kernel/init_task.c | 3 +-- 25-akpm/arch/x86_64/kernel/ioport.c | 2 +- 25-akpm/arch/x86_64/kernel/process.c | 4 ++-- 25-akpm/arch/x86_64/kernel/setup64.c | 2 +- 25-akpm/arch/x86_64/kernel/smpboot.c | 2 +- 25-akpm/arch/x86_64/kernel/suspend.c | 2 +- 25-akpm/arch/x86_64/kernel/traps.c | 5 +++-- 25-akpm/include/asm-x86_64/processor.h | 7 ++++--- 8 files changed, 14 insertions(+), 13 deletions(-) diff -puN arch/x86_64/kernel/init_task.c~x86-64-turn-tss-into-per-cpu-data arch/x86_64/kernel/init_task.c --- 25/arch/x86_64/kernel/init_task.c~x86-64-turn-tss-into-per-cpu-data 2004-09-15 02:32:32.583014856 -0700 +++ 25-akpm/arch/x86_64/kernel/init_task.c 2004-09-15 02:32:32.601012120 -0700 @@ -44,8 +44,7 @@ EXPORT_SYMBOL(init_task); * section. Since TSS's are completely CPU-local, we want them * on exact cacheline boundaries, to eliminate cacheline ping-pong. */ -struct tss_struct init_tss[NR_CPUS] __cacheline_aligned; - +DEFINE_PER_CPU(struct tss_struct, init_tss) ____cacheline_maxaligned_in_smp; #define ALIGN_TO_4K __attribute__((section(".data.init_task"))) diff -puN arch/x86_64/kernel/ioport.c~x86-64-turn-tss-into-per-cpu-data arch/x86_64/kernel/ioport.c --- 25/arch/x86_64/kernel/ioport.c~x86-64-turn-tss-into-per-cpu-data 2004-09-15 02:32:32.585014552 -0700 +++ 25-akpm/arch/x86_64/kernel/ioport.c 2004-09-15 02:32:32.597012728 -0700 @@ -61,7 +61,7 @@ asmlinkage long sys_ioperm(unsigned long * do it in the per-thread copy and in the TSS ... */ set_bitmap(t->io_bitmap_ptr, from, num, !turn_on); - tss = init_tss + get_cpu(); + tss = &per_cpu(init_tss, get_cpu()); if (tss->io_bitmap_base == IO_BITMAP_OFFSET) { /* already active? */ set_bitmap(tss->io_bitmap, from, num, !turn_on); } else { diff -puN arch/x86_64/kernel/process.c~x86-64-turn-tss-into-per-cpu-data arch/x86_64/kernel/process.c --- 25/arch/x86_64/kernel/process.c~x86-64-turn-tss-into-per-cpu-data 2004-09-15 02:32:32.586014400 -0700 +++ 25-akpm/arch/x86_64/kernel/process.c 2004-09-15 02:32:32.598012576 -0700 @@ -251,7 +251,7 @@ void exit_thread(void) { struct task_struct *me = current; if (me->thread.io_bitmap_ptr) { - struct tss_struct *tss = init_tss + get_cpu(); + struct tss_struct *tss = &per_cpu(init_tss,get_cpu()); kfree(me->thread.io_bitmap_ptr); me->thread.io_bitmap_ptr = NULL; tss->io_bitmap_base = INVALID_IO_BITMAP_OFFSET; @@ -404,7 +404,7 @@ struct task_struct *__switch_to(struct t struct thread_struct *prev = &prev_p->thread, *next = &next_p->thread; int cpu = smp_processor_id(); - struct tss_struct *tss = init_tss + cpu; + struct tss_struct *tss = &per_cpu(init_tss, cpu); unlazy_fpu(prev_p); diff -puN arch/x86_64/kernel/setup64.c~x86-64-turn-tss-into-per-cpu-data arch/x86_64/kernel/setup64.c --- 25/arch/x86_64/kernel/setup64.c~x86-64-turn-tss-into-per-cpu-data 2004-09-15 02:32:32.588014096 -0700 +++ 25-akpm/arch/x86_64/kernel/setup64.c 2004-09-15 02:32:32.599012424 -0700 @@ -237,7 +237,7 @@ void __init cpu_init (void) #else int cpu = smp_processor_id(); #endif - struct tss_struct * t = &init_tss[cpu]; + struct tss_struct *t = &per_cpu(init_tss, cpu); unsigned long v; char *estacks = NULL; struct task_struct *me; diff -puN arch/x86_64/kernel/smpboot.c~x86-64-turn-tss-into-per-cpu-data arch/x86_64/kernel/smpboot.c --- 25/arch/x86_64/kernel/smpboot.c~x86-64-turn-tss-into-per-cpu-data 2004-09-15 02:32:32.590013792 -0700 +++ 25-akpm/arch/x86_64/kernel/smpboot.c 2004-09-15 02:32:32.601012120 -0700 @@ -575,7 +575,7 @@ static void __init do_boot_cpu (int apic start_rip = setup_trampoline(); init_rsp = idle->thread.rsp; - init_tss[cpu].rsp0 = init_rsp; + per_cpu(init_tss,cpu).rsp0 = init_rsp; initial_code = start_secondary; clear_ti_thread_flag(idle->thread_info, TIF_FORK); diff -puN arch/x86_64/kernel/suspend.c~x86-64-turn-tss-into-per-cpu-data arch/x86_64/kernel/suspend.c --- 25/arch/x86_64/kernel/suspend.c~x86-64-turn-tss-into-per-cpu-data 2004-09-15 02:32:32.591013640 -0700 +++ 25-akpm/arch/x86_64/kernel/suspend.c 2004-09-15 02:32:32.598012576 -0700 @@ -129,7 +129,7 @@ void restore_processor_state(void) void fix_processor_context(void) { int cpu = smp_processor_id(); - struct tss_struct * t = init_tss + cpu; + struct tss_struct *t = &per_cpu(init_tss, cpu); set_tss_desc(cpu,t); /* This just modifies memory; should not be neccessary. But... This is neccessary, because 386 hardware has concept of busy TSS or some similar stupidity. */ diff -puN arch/x86_64/kernel/traps.c~x86-64-turn-tss-into-per-cpu-data arch/x86_64/kernel/traps.c --- 25/arch/x86_64/kernel/traps.c~x86-64-turn-tss-into-per-cpu-data 2004-09-15 02:32:32.593013336 -0700 +++ 25-akpm/arch/x86_64/kernel/traps.c 2004-09-15 02:32:32.600012272 -0700 @@ -113,9 +113,10 @@ unsigned long *in_exception_stack(int cp { int k; for (k = 0; k < N_EXCEPTION_STACKS; k++) { - unsigned long end = init_tss[cpu].ist[k] + EXCEPTION_STKSZ; + struct tss_struct *tss = &per_cpu(init_tss, cpu); + unsigned long end = tss->ist[k] + EXCEPTION_STKSZ; - if (stack >= init_tss[cpu].ist[k] && stack <= end) + if (stack >= tss->ist[k] && stack <= end) return (unsigned long *)end; } return NULL; diff -puN include/asm-x86_64/processor.h~x86-64-turn-tss-into-per-cpu-data include/asm-x86_64/processor.h --- 25/include/asm-x86_64/processor.h~x86-64-turn-tss-into-per-cpu-data 2004-09-15 02:32:32.594013184 -0700 +++ 25-akpm/include/asm-x86_64/processor.h 2004-09-15 02:32:32.602011968 -0700 @@ -18,6 +18,7 @@ #include #include #include +#include #include #define TF_MASK 0x00000100 @@ -77,9 +78,6 @@ struct cpuinfo_x86 { #define X86_VENDOR_NUM 8 #define X86_VENDOR_UNKNOWN 0xff -extern struct cpuinfo_x86 boot_cpu_data; -extern struct tss_struct init_tss[NR_CPUS]; - #ifdef CONFIG_SMP extern struct cpuinfo_x86 cpu_data[]; #define current_cpu_data cpu_data[smp_processor_id()] @@ -229,6 +227,9 @@ struct tss_struct { unsigned long io_bitmap[IO_BITMAP_LONGS + 1]; } __attribute__((packed)) ____cacheline_aligned; +extern struct cpuinfo_x86 boot_cpu_data; +DECLARE_PER_CPU(struct tss_struct,init_tss); + #define ARCH_MIN_TASKALIGN 16 struct thread_struct { _