diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index d3dab9c4c4c8..47dd5253fd94 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -42,6 +42,7 @@ extern unsigned int sysctl_sched_child_runs_first; extern unsigned int sysctl_sched_wake_to_idle; extern unsigned int sysctl_sched_wakeup_load_threshold; extern unsigned int sysctl_sched_window_stats_policy; +extern unsigned int sysctl_sched_account_wait_time; #if defined(CONFIG_SCHED_FREQ_INPUT) || defined(CONFIG_SCHED_HMP) extern unsigned int sysctl_sched_init_task_load_pct; @@ -103,6 +104,9 @@ extern int sched_hmp_proc_update_handler(struct ctl_table *table, extern int sched_boost_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); +extern int sched_acct_wait_time_update_handler(struct ctl_table *table, + int write, void __user *buffer, size_t *lenp, loff_t *ppos); + extern int sched_window_stats_policy_update_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 55ec953fb4cb..2d08b9e505a0 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1195,7 +1195,14 @@ unsigned int min_possible_efficiency = 1024; __read_mostly int sysctl_sched_freq_inc_notify_slack_pct; __read_mostly int sysctl_sched_freq_dec_notify_slack_pct = 25; + static __read_mostly unsigned int sched_account_wait_time = 1; +/* + * Copy of sched_account_wait_time, used to change it atomically. + * Initialize both variables to same value!! + */ +__read_mostly unsigned int sysctl_sched_account_wait_time = 1; + static __read_mostly unsigned int sched_io_is_busy; /* @@ -1682,7 +1689,7 @@ unsigned long sched_get_busy(int cpu) /* Called with IRQs disabled */ void reset_all_window_stats(u64 window_start, unsigned int window_size, - int policy) + int policy, int acct_wait_time) { int cpu; u64 wallclock; @@ -1728,6 +1735,9 @@ void reset_all_window_stats(u64 window_start, unsigned int window_size, if (policy >= 0) sched_window_stats_policy = policy; + if (acct_wait_time >= 0) + sched_account_wait_time = acct_wait_time; + for_each_online_cpu(cpu) { struct rq *rq = cpu_rq(cpu); raw_spin_unlock(&rq->lock); @@ -1765,7 +1775,7 @@ int sched_set_window(u64 window_start, unsigned int window_size) BUG_ON(sched_clock() < ws); - reset_all_window_stats(ws, window_size, -1); + reset_all_window_stats(ws, window_size, -1, -1); local_irq_restore(flags); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 7f8ee0b7b4c7..26839998ebec 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -3202,6 +3202,38 @@ void post_big_small_task_count_change(void) static DEFINE_MUTEX(policy_mutex); +int sched_acct_wait_time_update_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret; + unsigned int *data = (unsigned int *)table->data; + unsigned int old_val; + unsigned long flags; + + if (!sched_enable_hmp) + return -EINVAL; + + mutex_lock(&policy_mutex); + + old_val = *data; + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret || !write || (write && old_val == *data)) + goto done; + + local_irq_save(flags); + + reset_all_window_stats(0, 0, -1, sysctl_sched_account_wait_time); + + local_irq_restore(flags); + +done: + mutex_unlock(&policy_mutex); + + return ret; +} + int sched_window_stats_policy_update_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos) @@ -3224,7 +3256,7 @@ int sched_window_stats_policy_update_handler(struct ctl_table *table, int write, local_irq_save(flags); - reset_all_window_stats(0, 0, sysctl_sched_window_stats_policy); + reset_all_window_stats(0, 0, sysctl_sched_window_stats_policy, -1); local_irq_restore(flags); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 3369d8709835..f1fa00c2448c 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1061,7 +1061,7 @@ extern void dec_nr_big_small_task(struct rq *rq, struct task_struct *p); extern void set_hmp_defaults(void); extern unsigned int power_cost_at_freq(int cpu, unsigned int freq); extern void reset_all_window_stats(u64 window_start, unsigned int window_size, - int policy); + int policy, int acct_wait_time); extern void boost_kick(int cpu); #else /* CONFIG_SCHED_HMP */ diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 2c464882e2da..cbdeb753f1c4 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -309,6 +309,13 @@ static struct ctl_table kern_table[] = { }, #endif #if defined(CONFIG_SCHED_FREQ_INPUT) || defined(CONFIG_SCHED_HMP) + { + .procname = "sched_account_wait_time", + .data = &sysctl_sched_account_wait_time, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_acct_wait_time_update_handler, + }, { .procname = "sched_window_stats_policy", .data = &sysctl_sched_window_stats_policy,