x86-64: Move TLB state from PDA to per-cpu and consolidate with 32-bit.
Signed-off-by: Brian Gerst <brgerst@gmail.com> Signed-off-by: Tejun Heo <tj@kernel.org>
This commit is contained in:
Brian Gerst
Tejun Heo
parent
1b437c8c73
commit
9eb912d1aa
7 changed files with 20 additions and 38 deletions
|
|
@ -1,13 +1,11 @@
|
|||
#ifndef _ASM_X86_MMU_CONTEXT_64_H
|
||||
#define _ASM_X86_MMU_CONTEXT_64_H
|
||||
|
||||
#include <asm/pda.h>
|
||||
|
||||
static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
|
||||
{
|
||||
#ifdef CONFIG_SMP
|
||||
if (read_pda(mmu_state) == TLBSTATE_OK)
|
||||
write_pda(mmu_state, TLBSTATE_LAZY);
|
||||
if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
|
||||
percpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
|
@ -19,8 +17,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
|
|||
/* stop flush ipis for the previous mm */
|
||||
cpu_clear(cpu, prev->cpu_vm_mask);
|
||||
#ifdef CONFIG_SMP
|
||||
write_pda(mmu_state, TLBSTATE_OK);
|
||||
write_pda(active_mm, next);
|
||||
percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
|
||||
percpu_write(cpu_tlbstate.active_mm, next);
|
||||
#endif
|
||||
cpu_set(cpu, next->cpu_vm_mask);
|
||||
load_cr3(next->pgd);
|
||||
|
|
@ -30,9 +28,9 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
|
|||
}
|
||||
#ifdef CONFIG_SMP
|
||||
else {
|
||||
write_pda(mmu_state, TLBSTATE_OK);
|
||||
if (read_pda(active_mm) != next)
|
||||
BUG();
|
||||
percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
|
||||
BUG_ON(percpu_read(cpu_tlbstate.active_mm) != next);
|
||||
|
||||
if (!cpu_test_and_set(cpu, next->cpu_vm_mask)) {
|
||||
/* We were in lazy tlb mode and leave_mm disabled
|
||||
* tlb flush IPI delivery. We must reload CR3
|
||||
|
|
|
|||
|
|
@ -25,9 +25,7 @@ struct x8664_pda {
|
|||
char *irqstackptr;
|
||||
short nodenumber; /* number of current node (32k max) */
|
||||
short in_bootmem; /* pda lives in bootmem */
|
||||
short mmu_state;
|
||||
short isidle;
|
||||
struct mm_struct *active_mm;
|
||||
} ____cacheline_aligned_in_smp;
|
||||
|
||||
DECLARE_PER_CPU(struct x8664_pda, __pda);
|
||||
|
|
|
|||
|
|
@ -148,20 +148,17 @@ void native_flush_tlb_others(const struct cpumask *cpumask,
|
|||
#define TLBSTATE_OK 1
|
||||
#define TLBSTATE_LAZY 2
|
||||
|
||||
#ifdef CONFIG_X86_32
|
||||
struct tlb_state {
|
||||
struct mm_struct *active_mm;
|
||||
int state;
|
||||
char __cacheline_padding[L1_CACHE_BYTES-8];
|
||||
};
|
||||
DECLARE_PER_CPU(struct tlb_state, cpu_tlbstate);
|
||||
|
||||
void reset_lazy_tlbstate(void);
|
||||
#else
|
||||
static inline void reset_lazy_tlbstate(void)
|
||||
{
|
||||
percpu_write(cpu_tlbstate.state, 0);
|
||||
percpu_write(cpu_tlbstate.active_mm, &init_mm);
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* SMP */
|
||||
|
||||
|
|
|
|||
|
|
@ -897,8 +897,6 @@ void __cpuinit pda_init(int cpu)
|
|||
pda->irqcount = -1;
|
||||
pda->kernelstack = (unsigned long)stack_thread_info() -
|
||||
PDA_STACKOFFSET + THREAD_SIZE;
|
||||
pda->active_mm = &init_mm;
|
||||
pda->mmu_state = 0;
|
||||
|
||||
if (cpu == 0) {
|
||||
/* others are initialized in smpboot.c */
|
||||
|
|
|
|||
|
|
@ -4,8 +4,8 @@
|
|||
|
||||
#include <asm/tlbflush.h>
|
||||
|
||||
DEFINE_PER_CPU(struct tlb_state, cpu_tlbstate)
|
||||
____cacheline_aligned = { &init_mm, 0, };
|
||||
DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate)
|
||||
= { &init_mm, 0, };
|
||||
|
||||
/* must come after the send_IPI functions above for inlining */
|
||||
#include <mach_ipi.h>
|
||||
|
|
@ -231,14 +231,6 @@ void flush_tlb_all(void)
|
|||
on_each_cpu(do_flush_tlb_all, NULL, 1);
|
||||
}
|
||||
|
||||
void reset_lazy_tlbstate(void)
|
||||
{
|
||||
int cpu = raw_smp_processor_id();
|
||||
|
||||
per_cpu(cpu_tlbstate, cpu).state = 0;
|
||||
per_cpu(cpu_tlbstate, cpu).active_mm = &init_mm;
|
||||
}
|
||||
|
||||
static int init_flush_cpumask(void)
|
||||
{
|
||||
alloc_cpumask_var(&flush_cpumask, GFP_KERNEL);
|
||||
|
|
|
|||
|
|
@ -18,6 +18,9 @@
|
|||
#include <asm/uv/uv_hub.h>
|
||||
#include <asm/uv/uv_bau.h>
|
||||
|
||||
DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate)
|
||||
= { &init_mm, 0, };
|
||||
|
||||
#include <mach_ipi.h>
|
||||
/*
|
||||
* Smarter SMP flushing macros.
|
||||
|
|
@ -62,9 +65,9 @@ static DEFINE_PER_CPU(union smp_flush_state, flush_state);
|
|||
*/
|
||||
void leave_mm(int cpu)
|
||||
{
|
||||
if (read_pda(mmu_state) == TLBSTATE_OK)
|
||||
if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
|
||||
BUG();
|
||||
cpu_clear(cpu, read_pda(active_mm)->cpu_vm_mask);
|
||||
cpu_clear(cpu, percpu_read(cpu_tlbstate.active_mm)->cpu_vm_mask);
|
||||
load_cr3(swapper_pg_dir);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(leave_mm);
|
||||
|
|
@ -142,8 +145,8 @@ asmlinkage void smp_invalidate_interrupt(struct pt_regs *regs)
|
|||
* BUG();
|
||||
*/
|
||||
|
||||
if (f->flush_mm == read_pda(active_mm)) {
|
||||
if (read_pda(mmu_state) == TLBSTATE_OK) {
|
||||
if (f->flush_mm == percpu_read(cpu_tlbstate.active_mm)) {
|
||||
if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
|
||||
if (f->flush_va == TLB_FLUSH_ALL)
|
||||
local_flush_tlb();
|
||||
else
|
||||
|
|
@ -281,7 +284,7 @@ static void do_flush_tlb_all(void *info)
|
|||
unsigned long cpu = smp_processor_id();
|
||||
|
||||
__flush_tlb_all();
|
||||
if (read_pda(mmu_state) == TLBSTATE_LAZY)
|
||||
if (percpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
|
||||
leave_mm(cpu);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1063,11 +1063,7 @@ static void drop_other_mm_ref(void *info)
|
|||
struct mm_struct *mm = info;
|
||||
struct mm_struct *active_mm;
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
active_mm = read_pda(active_mm);
|
||||
#else
|
||||
active_mm = __get_cpu_var(cpu_tlbstate).active_mm;
|
||||
#endif
|
||||
active_mm = percpu_read(cpu_tlbstate.active_mm);
|
||||
|
||||
if (active_mm == mm)
|
||||
leave_mm(smp_processor_id());
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue