evie/android_kernel_oneplus_msm8998
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'pm-cpuidle'

Browse source 
* pm-cpuidle:
  cpuidle: coupled: fix race condition between pokes and safe state
  cpuidle: coupled: abort idle if pokes are pending
  cpuidle: coupled: disable interrupts after entering safe state
This commit is contained in:
Rafael J. Wysocki 2013-08-30 14:15:16 +02:00
commit 499aa70a01
1 changed files with 98 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

129
drivers/cpuidle/coupled.c
View file

@ -106,6 +106,7 @@ struct cpuidle_coupled {
cpumask_t coupled_cpus;
int requested_state[NR_CPUS];
atomic_t ready_waiting_counts;
atomic_t abort_barrier;
int online_count;
int refcnt;
int prevent;
@ -122,12 +123,19 @@ static DEFINE_MUTEX(cpuidle_coupled_lock);
static DEFINE_PER_CPU(struct call_single_data, cpuidle_coupled_poke_cb);
/*
* The cpuidle_coupled_poked_mask mask is used to avoid calling
* The cpuidle_coupled_poke_pending mask is used to avoid calling
* __smp_call_function_single with the per cpu call_single_data struct already
* in use. This prevents a deadlock where two cpus are waiting for each others
* call_single_data struct to be available
*/
static cpumask_t cpuidle_coupled_poked_mask;
static cpumask_t cpuidle_coupled_poke_pending;
/*
* The cpuidle_coupled_poked mask is used to ensure that each cpu has been poked
* once to minimize entering the ready loop with a poke pending, which would
* require aborting and retrying.
*/
static cpumask_t cpuidle_coupled_poked;
/**
* cpuidle_coupled_parallel_barrier - synchronize all online coupled cpus
@ -291,10 +299,11 @@ static inline int cpuidle_coupled_get_state(struct cpuidle_device *dev,
return state;
}
static void cpuidle_coupled_poked(void *info)
static void cpuidle_coupled_handle_poke(void *info)
{
int cpu = (unsigned long)info;
cpumask_clear_cpu(cpu, &cpuidle_coupled_poked_mask);
cpumask_set_cpu(cpu, &cpuidle_coupled_poked);
cpumask_clear_cpu(cpu, &cpuidle_coupled_poke_pending);
}
/**
@ -313,7 +322,7 @@ static void cpuidle_coupled_poke(int cpu)
{
struct call_single_data *csd = &per_cpu(cpuidle_coupled_poke_cb, cpu);
if (!cpumask_test_and_set_cpu(cpu, &cpuidle_coupled_poked_mask))
if (!cpumask_test_and_set_cpu(cpu, &cpuidle_coupled_poke_pending))
__smp_call_function_single(cpu, csd, 0);
}
@ -340,30 +349,19 @@ static void cpuidle_coupled_poke_others(int this_cpu,
* @coupled: the struct coupled that contains the current cpu
* @next_state: the index in drv->states of the requested state for this cpu
*
* Updates the requested idle state for the specified cpuidle device,
* poking all coupled cpus out of idle if necessary to let them see the new
* state.
* Updates the requested idle state for the specified cpuidle device.
* Returns the number of waiting cpus.
*/
static void cpuidle_coupled_set_waiting(int cpu,
static int cpuidle_coupled_set_waiting(int cpu,
struct cpuidle_coupled *coupled, int next_state)
{
int w;
coupled->requested_state[cpu] = next_state;
/*
* If this is the last cpu to enter the waiting state, poke
* all the other cpus out of their waiting state so they can
* enter a deeper state. This can race with one of the cpus
* exiting the waiting state due to an interrupt and
* decrementing waiting_count, see comment below.
*
* The atomic_inc_return provides a write barrier to order the write
* to requested_state with the later write that increments ready_count.
*/
w = atomic_inc_return(&coupled->ready_waiting_counts) & WAITING_MASK;
if (w == coupled->online_count)
cpuidle_coupled_poke_others(cpu, coupled);
return atomic_inc_return(&coupled->ready_waiting_counts) & WAITING_MASK;
}
/**
@ -410,19 +408,33 @@ static void cpuidle_coupled_set_done(int cpu, struct cpuidle_coupled *coupled)
* been processed and the poke bit has been cleared.
*
* Other interrupts may also be processed while interrupts are enabled, so
* need_resched() must be tested after turning interrupts off again to make sure
* need_resched() must be tested after this function returns to make sure
* the interrupt didn't schedule work that should take the cpu out of idle.
*
* Returns 0 if need_resched was false, -EINTR if need_resched was true.
* Returns 0 if no poke was pending, 1 if a poke was cleared.
*/
static int cpuidle_coupled_clear_pokes(int cpu)
{
if (!cpumask_test_cpu(cpu, &cpuidle_coupled_poke_pending))
return 0;
local_irq_enable();
while (cpumask_test_cpu(cpu, &cpuidle_coupled_poked_mask))
while (cpumask_test_cpu(cpu, &cpuidle_coupled_poke_pending))
cpu_relax();
local_irq_disable();
return need_resched() ? -EINTR : 0;
return 1;
}
static bool cpuidle_coupled_any_pokes_pending(struct cpuidle_coupled *coupled)
{
cpumask_t cpus;
int ret;
cpumask_and(&cpus, cpu_online_mask, &coupled->coupled_cpus);
ret = cpumask_and(&cpus, &cpuidle_coupled_poke_pending, &cpus);
return ret;
}
/**
@ -449,31 +461,56 @@ int cpuidle_enter_state_coupled(struct cpuidle_device *dev,
{
int entered_state = -1;
struct cpuidle_coupled *coupled = dev->coupled;
int w;
if (!coupled)
return -EINVAL;
while (coupled->prevent) {
if (cpuidle_coupled_clear_pokes(dev->cpu)) {
cpuidle_coupled_clear_pokes(dev->cpu);
if (need_resched()) {
local_irq_enable();
return entered_state;
}
entered_state = cpuidle_enter_state(dev, drv,
dev->safe_state_index);
local_irq_disable();
}
/* Read barrier ensures online_count is read after prevent is cleared */
smp_rmb();
cpuidle_coupled_set_waiting(dev->cpu, coupled, next_state);
reset:
cpumask_clear_cpu(dev->cpu, &cpuidle_coupled_poked);
w = cpuidle_coupled_set_waiting(dev->cpu, coupled, next_state);
/*
* If this is the last cpu to enter the waiting state, poke
* all the other cpus out of their waiting state so they can
* enter a deeper state. This can race with one of the cpus
* exiting the waiting state due to an interrupt and
* decrementing waiting_count, see comment below.
*/
if (w == coupled->online_count) {
cpumask_set_cpu(dev->cpu, &cpuidle_coupled_poked);
cpuidle_coupled_poke_others(dev->cpu, coupled);
}
retry:
/*
* Wait for all coupled cpus to be idle, using the deepest state
* allowed for a single cpu.
* allowed for a single cpu. If this was not the poking cpu, wait
* for at least one poke before leaving to avoid a race where
* two cpus could arrive at the waiting loop at the same time,
* but the first of the two to arrive could skip the loop without
* processing the pokes from the last to arrive.
*/
while (!cpuidle_coupled_cpus_waiting(coupled)) {
if (cpuidle_coupled_clear_pokes(dev->cpu)) {
while (!cpuidle_coupled_cpus_waiting(coupled) ||
!cpumask_test_cpu(dev->cpu, &cpuidle_coupled_poked)) {
if (cpuidle_coupled_clear_pokes(dev->cpu))
continue;
if (need_resched()) {
cpuidle_coupled_set_not_waiting(dev->cpu, coupled);
goto out;
}
@ -485,13 +522,21 @@ retry:
entered_state = cpuidle_enter_state(dev, drv,
dev->safe_state_index);
local_irq_disable();
}
if (cpuidle_coupled_clear_pokes(dev->cpu)) {
cpuidle_coupled_clear_pokes(dev->cpu);
if (need_resched()) {
cpuidle_coupled_set_not_waiting(dev->cpu, coupled);
goto out;
}
/*
* Make sure final poke status for this cpu is visible before setting
* cpu as ready.
*/
smp_wmb();
/*
* All coupled cpus are probably idle. There is a small chance that
* one of the other cpus just became active. Increment the ready count,
@ -511,6 +556,28 @@ retry:
cpu_relax();
}
/*
* Make sure read of all cpus ready is done before reading pending pokes
*/
smp_rmb();
/*
* There is a small chance that a cpu left and reentered idle after this
* cpu saw that all cpus were waiting. The cpu that reentered idle will
* have sent this cpu a poke, which will still be pending after the
* ready loop. The pending interrupt may be lost by the interrupt
* controller when entering the deep idle state. It's not possible to
* clear a pending interrupt without turning interrupts on and handling
* it, and it's too late to turn on interrupts here, so reset the
* coupled idle state of all cpus and retry.
*/
if (cpuidle_coupled_any_pokes_pending(coupled)) {
cpuidle_coupled_set_done(dev->cpu, coupled);
/* Wait for all cpus to see the pending pokes */
cpuidle_coupled_parallel_barrier(dev, &coupled->abort_barrier);
goto reset;
}
/* all cpus have acked the coupled state */
next_state = cpuidle_coupled_get_state(dev, coupled);
@ -596,7 +663,7 @@ have_coupled:
coupled->refcnt++;
csd = &per_cpu(cpuidle_coupled_poke_cb, dev->cpu);
csd->func = cpuidle_coupled_poked;
csd->func = cpuidle_coupled_handle_poke;
csd->info = (void *)(unsigned long)dev->cpu;
return 0;