sched: Move around code
Move up chunk of code to be defined early. This helps a subsequent patch that needs update_min_max_capacity() Change-Id: I9403c7b4dcc74ba4ef1034327241c81df97b01ea Signed-off-by: Srivatsa Vaddagiri <vatsa@codeaurora.org> Signed-off-by: Syed Rameez Mustafa <rameezmustafa@codeaurora.org> [joonwoop@codeaurora.org: fixed trivial conflicts.] Signed-off-by: Joonwoo Park <joonwoop@codeaurora.org>
This commit is contained in:
Srivatsa Vaddagiri
David Keitel
parent
01cb5df240
commit
db573f06e8
1 changed files with 207 additions and 211 deletions
|
|
@ -1102,6 +1102,27 @@ unsigned int min_possible_efficiency = 1024;
|
|||
__read_mostly int sysctl_sched_freq_inc_notify_slack_pct;
|
||||
__read_mostly int sysctl_sched_freq_dec_notify_slack_pct = 25;
|
||||
|
||||
/*
|
||||
* Maximum possible frequency across all cpus. Task demand and cpu
|
||||
* capacity (cpu_power) metrics are scaled in reference to it.
|
||||
*/
|
||||
unsigned int max_possible_freq = 1;
|
||||
|
||||
/*
|
||||
* Minimum possible max_freq across all cpus. This will be same as
|
||||
* max_possible_freq on homogeneous systems and could be different from
|
||||
* max_possible_freq on heterogenous systems. min_max_freq is used to derive
|
||||
* capacity (cpu_power) of cpus.
|
||||
*/
|
||||
unsigned int min_max_freq = 1;
|
||||
|
||||
unsigned int max_capacity = 1024; /* max(rq->capacity) */
|
||||
unsigned int min_capacity = 1024; /* min(rq->capacity) */
|
||||
|
||||
static unsigned int sync_cpu;
|
||||
static u64 sched_init_jiffy;
|
||||
static u64 sched_clock_at_init_jiffy;
|
||||
|
||||
/* Returns how undercommitted a CPU is given its current frequency and
|
||||
* task load (as measured in the previous window). Returns this value
|
||||
* as a percentage of the CPU's maximum frequency. A negative value
|
||||
|
|
@ -1401,10 +1422,6 @@ static inline void mark_task_starting(struct task_struct *p)
|
|||
p->ravg.prev_window = p->ravg.demand;
|
||||
}
|
||||
|
||||
static unsigned int sync_cpu;
|
||||
static u64 sched_init_jiffy;
|
||||
static u64 sched_clock_at_init_jiffy;
|
||||
|
||||
static inline void set_window_start(struct rq *rq)
|
||||
{
|
||||
int cpu = cpu_of(rq);
|
||||
|
|
@ -1513,6 +1530,192 @@ void sched_set_window(u64 window_start, unsigned int window_size)
|
|||
local_irq_restore(flags);
|
||||
}
|
||||
|
||||
/* Keep track of max/min capacity possible across CPUs "currently" */
|
||||
static void update_min_max_capacity(void)
|
||||
{
|
||||
int i;
|
||||
int max = 0, min = INT_MAX;
|
||||
|
||||
for_each_possible_cpu(i) {
|
||||
if (cpu_rq(i)->capacity > max)
|
||||
max = cpu_rq(i)->capacity;
|
||||
if (cpu_rq(i)->capacity < min)
|
||||
min = cpu_rq(i)->capacity;
|
||||
}
|
||||
|
||||
max_capacity = max;
|
||||
min_capacity = min;
|
||||
}
|
||||
|
||||
/*
|
||||
* Return 'capacity' of a cpu in reference to "least" efficient cpu, such that
|
||||
* least efficient cpu gets capacity of 1024
|
||||
*/
|
||||
unsigned long capacity_scale_cpu_efficiency(int cpu)
|
||||
{
|
||||
return (1024 * cpu_rq(cpu)->efficiency) / min_possible_efficiency;
|
||||
}
|
||||
|
||||
/*
|
||||
* Return 'capacity' of a cpu in reference to cpu with lowest max_freq
|
||||
* (min_max_freq), such that one with lowest max_freq gets capacity of 1024.
|
||||
*/
|
||||
unsigned long capacity_scale_cpu_freq(int cpu)
|
||||
{
|
||||
return (1024 * cpu_rq(cpu)->max_freq) / min_max_freq;
|
||||
}
|
||||
|
||||
/*
|
||||
* Return load_scale_factor of a cpu in reference to "most" efficient cpu, so
|
||||
* that "most" efficient cpu gets a load_scale_factor of 1
|
||||
*/
|
||||
static inline unsigned long load_scale_cpu_efficiency(int cpu)
|
||||
{
|
||||
return (1024 * max_possible_efficiency) / cpu_rq(cpu)->efficiency;
|
||||
}
|
||||
|
||||
/*
|
||||
* Return load_scale_factor of a cpu in reference to cpu with best max_freq
|
||||
* (max_possible_freq), so that one with best max_freq gets a load_scale_factor
|
||||
* of 1.
|
||||
*/
|
||||
static inline unsigned long load_scale_cpu_freq(int cpu)
|
||||
{
|
||||
return (1024 * max_possible_freq) / cpu_rq(cpu)->max_freq;
|
||||
}
|
||||
|
||||
static int compute_capacity(int cpu)
|
||||
{
|
||||
int capacity = 1024;
|
||||
|
||||
capacity *= capacity_scale_cpu_efficiency(cpu);
|
||||
capacity >>= 10;
|
||||
|
||||
capacity *= capacity_scale_cpu_freq(cpu);
|
||||
capacity >>= 10;
|
||||
|
||||
return capacity;
|
||||
}
|
||||
|
||||
static int compute_load_scale_factor(int cpu)
|
||||
{
|
||||
int load_scale = 1024;
|
||||
|
||||
/*
|
||||
* load_scale_factor accounts for the fact that task load
|
||||
* is in reference to "best" performing cpu. Task's load will need to be
|
||||
* scaled (up) by a factor to determine suitability to be placed on a
|
||||
* (little) cpu.
|
||||
*/
|
||||
load_scale *= load_scale_cpu_efficiency(cpu);
|
||||
load_scale >>= 10;
|
||||
|
||||
load_scale *= load_scale_cpu_freq(cpu);
|
||||
load_scale >>= 10;
|
||||
|
||||
return load_scale;
|
||||
}
|
||||
|
||||
static int cpufreq_notifier_policy(struct notifier_block *nb,
|
||||
unsigned long val, void *data)
|
||||
{
|
||||
struct cpufreq_policy *policy = (struct cpufreq_policy *)data;
|
||||
int i;
|
||||
unsigned int min_max = min_max_freq;
|
||||
const struct cpumask *cpus = policy->related_cpus;
|
||||
int orig_min_max_freq = min_max_freq;
|
||||
|
||||
if (val != CPUFREQ_NOTIFY)
|
||||
return 0;
|
||||
|
||||
for_each_cpu(i, policy->related_cpus) {
|
||||
cpumask_copy(&cpu_rq(i)->freq_domain_cpumask,
|
||||
policy->related_cpus);
|
||||
cpu_rq(i)->min_freq = policy->min;
|
||||
cpu_rq(i)->max_freq = policy->max;
|
||||
cpu_rq(i)->max_possible_freq = policy->cpuinfo.max_freq;
|
||||
}
|
||||
|
||||
max_possible_freq = max(max_possible_freq, policy->cpuinfo.max_freq);
|
||||
if (min_max_freq == 1)
|
||||
min_max = UINT_MAX;
|
||||
min_max_freq = min(min_max, policy->cpuinfo.max_freq);
|
||||
BUG_ON(!min_max_freq);
|
||||
BUG_ON(!policy->max);
|
||||
|
||||
if (min_max_freq != orig_min_max_freq)
|
||||
cpus = cpu_online_mask;
|
||||
|
||||
/*
|
||||
* Changed load_scale_factor can trigger reclassification of tasks as
|
||||
* big or small. Make this change "atomic" so that tasks are accounted
|
||||
* properly due to changed load_scale_factor
|
||||
*/
|
||||
pre_big_small_task_count_change();
|
||||
for_each_cpu(i, cpus) {
|
||||
struct rq *rq = cpu_rq(i);
|
||||
|
||||
rq->capacity = compute_capacity(i);
|
||||
rq->load_scale_factor = compute_load_scale_factor(i);
|
||||
}
|
||||
|
||||
update_min_max_capacity();
|
||||
post_big_small_task_count_change();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int cpufreq_notifier_trans(struct notifier_block *nb,
|
||||
unsigned long val, void *data)
|
||||
{
|
||||
struct cpufreq_freqs *freq = (struct cpufreq_freqs *)data;
|
||||
unsigned int cpu = freq->cpu, new_freq = freq->new;
|
||||
struct rq *rq = cpu_rq(cpu);
|
||||
unsigned long flags;
|
||||
|
||||
if (val != CPUFREQ_POSTCHANGE)
|
||||
return 0;
|
||||
|
||||
BUG_ON(!new_freq);
|
||||
|
||||
raw_spin_lock_irqsave(&rq->lock, flags);
|
||||
update_task_ravg(rq->curr, rq, TASK_UPDATE, sched_clock(), NULL);
|
||||
cpu_rq(cpu)->cur_freq = new_freq;
|
||||
raw_spin_unlock_irqrestore(&rq->lock, flags);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static struct notifier_block notifier_policy_block = {
|
||||
.notifier_call = cpufreq_notifier_policy
|
||||
};
|
||||
|
||||
static struct notifier_block notifier_trans_block = {
|
||||
.notifier_call = cpufreq_notifier_trans
|
||||
};
|
||||
|
||||
static int register_sched_callback(void)
|
||||
{
|
||||
int ret;
|
||||
|
||||
ret = cpufreq_register_notifier(¬ifier_policy_block,
|
||||
CPUFREQ_POLICY_NOTIFIER);
|
||||
|
||||
if (!ret)
|
||||
ret = cpufreq_register_notifier(¬ifier_trans_block,
|
||||
CPUFREQ_TRANSITION_NOTIFIER);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* cpufreq callbacks can be registered at core_initcall or later time.
|
||||
* Any registration done prior to that is "forgotten" by cpufreq. See
|
||||
* initialization of variable init_cpufreq_transition_notifier_list_called
|
||||
* for further information.
|
||||
*/
|
||||
core_initcall(register_sched_callback);
|
||||
|
||||
#else /* CONFIG_SCHED_FREQ_INPUT || CONFIG_SCHED_HMP */
|
||||
|
||||
static inline void
|
||||
|
|
@ -8005,213 +8208,6 @@ void __init sched_init_smp(void)
|
|||
#endif /* CONFIG_SMP */
|
||||
|
||||
|
||||
#if defined(CONFIG_SCHED_FREQ_INPUT) || defined(CONFIG_SCHED_HMP)
|
||||
|
||||
/*
|
||||
* Maximum possible frequency across all cpus. Task demand and cpu
|
||||
* capacity (cpu_power) metrics are scaled in reference to it.
|
||||
*/
|
||||
unsigned int max_possible_freq = 1;
|
||||
|
||||
/*
|
||||
* Minimum possible max_freq across all cpus. This will be same as
|
||||
* max_possible_freq on homogeneous systems and could be different from
|
||||
* max_possible_freq on heterogenous systems. min_max_freq is used to derive
|
||||
* capacity (cpu_power) of cpus.
|
||||
*/
|
||||
unsigned int min_max_freq = 1;
|
||||
|
||||
unsigned int max_capacity = 1024; /* max(rq->capacity) */
|
||||
unsigned int min_capacity = 1024; /* min(rq->capacity) */
|
||||
|
||||
/* Keep track of max/min capacity possible across CPUs "currently" */
|
||||
static void update_min_max_capacity(void)
|
||||
{
|
||||
int i;
|
||||
int max = 0, min = INT_MAX;
|
||||
|
||||
for_each_possible_cpu(i) {
|
||||
if (cpu_rq(i)->capacity > max)
|
||||
max = cpu_rq(i)->capacity;
|
||||
if (cpu_rq(i)->capacity < min)
|
||||
min = cpu_rq(i)->capacity;
|
||||
}
|
||||
|
||||
max_capacity = max;
|
||||
min_capacity = min;
|
||||
}
|
||||
|
||||
/*
|
||||
* Return 'capacity' of a cpu in reference to "least" efficient cpu, such that
|
||||
* least efficient cpu gets capacity of 1024
|
||||
*/
|
||||
unsigned long capacity_scale_cpu_efficiency(int cpu)
|
||||
{
|
||||
return (1024 * cpu_rq(cpu)->efficiency) / min_possible_efficiency;
|
||||
}
|
||||
|
||||
/*
|
||||
* Return 'capacity' of a cpu in reference to cpu with lowest max_freq
|
||||
* (min_max_freq), such that one with lowest max_freq gets capacity of 1024.
|
||||
*/
|
||||
unsigned long capacity_scale_cpu_freq(int cpu)
|
||||
{
|
||||
return (1024 * cpu_rq(cpu)->max_freq) / min_max_freq;
|
||||
}
|
||||
|
||||
/*
|
||||
* Return load_scale_factor of a cpu in reference to "most" efficient cpu, so
|
||||
* that "most" efficient cpu gets a load_scale_factor of 1
|
||||
*/
|
||||
static inline unsigned long load_scale_cpu_efficiency(int cpu)
|
||||
{
|
||||
return (1024 * max_possible_efficiency) / cpu_rq(cpu)->efficiency;
|
||||
}
|
||||
|
||||
/*
|
||||
* Return load_scale_factor of a cpu in reference to cpu with best max_freq
|
||||
* (max_possible_freq), so that one with best max_freq gets a load_scale_factor
|
||||
* of 1.
|
||||
*/
|
||||
static inline unsigned long load_scale_cpu_freq(int cpu)
|
||||
{
|
||||
return (1024 * max_possible_freq) / cpu_rq(cpu)->max_freq;
|
||||
}
|
||||
|
||||
static int compute_capacity(int cpu)
|
||||
{
|
||||
int capacity = 1024;
|
||||
|
||||
capacity *= capacity_scale_cpu_efficiency(cpu);
|
||||
capacity >>= 10;
|
||||
|
||||
capacity *= capacity_scale_cpu_freq(cpu);
|
||||
capacity >>= 10;
|
||||
|
||||
return capacity;
|
||||
}
|
||||
|
||||
static int compute_load_scale_factor(int cpu)
|
||||
{
|
||||
int load_scale = 1024;
|
||||
|
||||
/*
|
||||
* load_scale_factor accounts for the fact that task load
|
||||
* is in reference to "best" performing cpu. Task's load will need to be
|
||||
* scaled (up) by a factor to determine suitability to be placed on a
|
||||
* (little) cpu.
|
||||
*/
|
||||
load_scale *= load_scale_cpu_efficiency(cpu);
|
||||
load_scale >>= 10;
|
||||
|
||||
load_scale *= load_scale_cpu_freq(cpu);
|
||||
load_scale >>= 10;
|
||||
|
||||
return load_scale;
|
||||
}
|
||||
|
||||
static int cpufreq_notifier_policy(struct notifier_block *nb,
|
||||
unsigned long val, void *data)
|
||||
{
|
||||
struct cpufreq_policy *policy = (struct cpufreq_policy *)data;
|
||||
int i;
|
||||
unsigned int min_max = min_max_freq;
|
||||
const struct cpumask *cpus = policy->related_cpus;
|
||||
int orig_min_max_freq = min_max_freq;
|
||||
|
||||
if (val != CPUFREQ_NOTIFY)
|
||||
return 0;
|
||||
|
||||
for_each_cpu(i, policy->related_cpus) {
|
||||
cpumask_copy(&cpu_rq(i)->freq_domain_cpumask,
|
||||
policy->related_cpus);
|
||||
cpu_rq(i)->min_freq = policy->min;
|
||||
cpu_rq(i)->max_freq = policy->max;
|
||||
cpu_rq(i)->max_possible_freq = policy->cpuinfo.max_freq;
|
||||
}
|
||||
|
||||
max_possible_freq = max(max_possible_freq, policy->cpuinfo.max_freq);
|
||||
if (min_max_freq == 1)
|
||||
min_max = UINT_MAX;
|
||||
min_max_freq = min(min_max, policy->cpuinfo.max_freq);
|
||||
BUG_ON(!min_max_freq);
|
||||
BUG_ON(!policy->max);
|
||||
|
||||
if (min_max_freq != orig_min_max_freq)
|
||||
cpus = cpu_online_mask;
|
||||
|
||||
/*
|
||||
* Changed load_scale_factor can trigger reclassification of tasks as
|
||||
* big or small. Make this change "atomic" so that tasks are accounted
|
||||
* properly due to changed load_scale_factor
|
||||
*/
|
||||
pre_big_small_task_count_change();
|
||||
for_each_cpu(i, cpus) {
|
||||
struct rq *rq = cpu_rq(i);
|
||||
|
||||
rq->capacity = compute_capacity(i);
|
||||
rq->load_scale_factor = compute_load_scale_factor(i);
|
||||
}
|
||||
|
||||
update_min_max_capacity();
|
||||
post_big_small_task_count_change();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int cpufreq_notifier_trans(struct notifier_block *nb,
|
||||
unsigned long val, void *data)
|
||||
{
|
||||
struct cpufreq_freqs *freq = (struct cpufreq_freqs *)data;
|
||||
unsigned int cpu = freq->cpu, new_freq = freq->new;
|
||||
struct rq *rq = cpu_rq(cpu);
|
||||
unsigned long flags;
|
||||
|
||||
if (val != CPUFREQ_POSTCHANGE)
|
||||
return 0;
|
||||
|
||||
BUG_ON(!new_freq);
|
||||
|
||||
raw_spin_lock_irqsave(&rq->lock, flags);
|
||||
update_task_ravg(rq->curr, rq, TASK_UPDATE, sched_clock(), NULL);
|
||||
cpu_rq(cpu)->cur_freq = new_freq;
|
||||
raw_spin_unlock_irqrestore(&rq->lock, flags);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static struct notifier_block notifier_policy_block = {
|
||||
.notifier_call = cpufreq_notifier_policy
|
||||
};
|
||||
|
||||
static struct notifier_block notifier_trans_block = {
|
||||
.notifier_call = cpufreq_notifier_trans
|
||||
};
|
||||
|
||||
static int register_sched_callback(void)
|
||||
{
|
||||
int ret;
|
||||
|
||||
ret = cpufreq_register_notifier(¬ifier_policy_block,
|
||||
CPUFREQ_POLICY_NOTIFIER);
|
||||
|
||||
if (!ret)
|
||||
ret = cpufreq_register_notifier(¬ifier_trans_block,
|
||||
CPUFREQ_TRANSITION_NOTIFIER);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* cpufreq callbacks can be registered at core_initcall or later time.
|
||||
* Any registration done prior to that is "forgotten" by cpufreq. See
|
||||
* initialization of variable init_cpufreq_transition_notifier_list_called
|
||||
* for further information.
|
||||
*/
|
||||
core_initcall(register_sched_callback);
|
||||
|
||||
#endif /* CONFIG_SCHED_FREQ_INPUT || CONFIG_SCHED_HMP */
|
||||
|
||||
int in_sched_functions(unsigned long addr)
|
||||
{
|
||||
return in_lock_functions(addr) ||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue