diff --git a/drivers/mmc/core/core.c b/drivers/mmc/core/core.c index f4acc56bb2c8..ae96334b9c8f 100644 --- a/drivers/mmc/core/core.c +++ b/drivers/mmc/core/core.c @@ -30,6 +30,7 @@ #include #include #include +#include #include @@ -52,6 +53,8 @@ /* If the device is not responding */ #define MMC_CORE_TIMEOUT_MS (10 * 60 * 1000) /* 10 minute timeout */ +static void mmc_clk_scaling(struct mmc_host *host, bool from_wq); + /* * Background operations can take a long time, depending on the housekeeping * operations the card has to perform. @@ -139,6 +142,10 @@ void mmc_request_done(struct mmc_host *host, struct mmc_request *mrq) #ifdef CONFIG_MMC_PERF_PROFILING ktime_t diff; #endif + if (host->card && host->clk_scaling.enable) + host->clk_scaling.busy_time_us += + ktime_to_us(ktime_sub(ktime_get(), + host->clk_scaling.start_busy)); /* Flag re-tuning needed on CRC errors */ if ((cmd->opcode != MMC_SEND_TUNING_BLOCK && @@ -324,6 +331,19 @@ static int mmc_start_request(struct mmc_host *host, struct mmc_request *mrq) } mmc_host_clk_hold(host); led_trigger_event(host->led, LED_FULL); + + if (host->card && host->clk_scaling.enable) { + /* + * Check if we need to scale the clocks. Clocks + * will be scaled up immediately if necessary + * conditions are satisfied. Scaling down the + * frequency will be done after current thread + * releases host. + */ + mmc_clk_scaling(host, false); + host->clk_scaling.start_busy = ktime_get(); + } + __mmc_start_request(host, mrq); return 0; @@ -2629,6 +2649,283 @@ int mmc_hw_reset(struct mmc_host *host) } EXPORT_SYMBOL(mmc_hw_reset); +/** + * mmc_reset_clk_scale_stats() - reset clock scaling statistics + * @host: pointer to mmc host structure + */ +void mmc_reset_clk_scale_stats(struct mmc_host *host) +{ + host->clk_scaling.busy_time_us = 0; + host->clk_scaling.window_time = jiffies; +} +EXPORT_SYMBOL_GPL(mmc_reset_clk_scale_stats); + +/** + * mmc_get_max_frequency() - get max. frequency supported + * @host: pointer to mmc host structure + * + * Returns max. frequency supported by card/host. If the + * timing mode is SDR50/SDR104/HS200/DDR50 return appropriate + * max. frequency in these modes else, use the current frequency. + * Also, allow host drivers to overwrite the frequency in case + * they support "get_max_frequency" host ops. + */ +unsigned long mmc_get_max_frequency(struct mmc_host *host) +{ + unsigned long freq; + + if (host->ops && host->ops->get_max_frequency) { + freq = host->ops->get_max_frequency(host); + goto out; + } + + switch (host->ios.timing) { + case MMC_TIMING_UHS_SDR50: + freq = UHS_SDR50_MAX_DTR; + break; + case MMC_TIMING_UHS_SDR104: + freq = UHS_SDR104_MAX_DTR; + break; + case MMC_TIMING_MMC_HS200: + freq = MMC_HS200_MAX_DTR; + break; + case MMC_TIMING_UHS_DDR50: + freq = UHS_DDR50_MAX_DTR; + break; + default: + mmc_host_clk_hold(host); + freq = host->ios.clock; + mmc_host_clk_release(host); + break; + } + +out: + return freq; +} +EXPORT_SYMBOL_GPL(mmc_get_max_frequency); + +/** + * mmc_get_min_frequency() - get min. frequency supported + * @host: pointer to mmc host structure + * + * Returns min. frequency supported by card/host which doesn't impair + * performance for most usecases. If the timing mode is SDR50/SDR104/HS200 + * return 50MHz value. If timing mode is DDR50 return 25MHz so that + * throughput would be equivalent to SDR50/SDR104 in 50MHz. Also, allow + * host drivers to overwrite the frequency in case they support + * "get_min_frequency" host ops. + */ +static unsigned long mmc_get_min_frequency(struct mmc_host *host) +{ + unsigned long freq; + + if (host->ops && host->ops->get_min_frequency) { + freq = host->ops->get_min_frequency(host); + goto out; + } + + switch (host->ios.timing) { + case MMC_TIMING_UHS_SDR50: + case MMC_TIMING_UHS_SDR104: + freq = UHS_SDR25_MAX_DTR; + break; + case MMC_TIMING_MMC_HS200: + freq = MMC_HIGH_52_MAX_DTR; + break; + case MMC_TIMING_UHS_DDR50: + freq = UHS_DDR50_MAX_DTR / 2; + break; + default: + mmc_host_clk_hold(host); + freq = host->ios.clock; + mmc_host_clk_release(host); + break; + } + +out: + return freq; +} + +/* + * Scale down clocks to minimum frequency supported. + * The delayed work re-arms itself in case it cannot + * claim the host. + */ +static void mmc_clk_scale_work(struct work_struct *work) +{ + struct mmc_host *host = container_of(work, struct mmc_host, + clk_scaling.work.work); + + if (!host->card || !host->bus_ops || + !host->bus_ops->change_bus_speed || + !host->clk_scaling.enable || !host->ios.clock) + goto out; + + mmc_clk_scaling(host, true); + mmc_release_host(host); +out: + return; +} + + +/** + * mmc_clk_scaling() - clock scaling decision algorithm + * @host: pointer to mmc host structure + * @from_wq: variable that specifies the context in which + * mmc_clk_scaling() is called. + * + * Calculate load percentage based on host busy time + * and total sampling interval and decide clock scaling + * based on scale up/down thresholds. + * If load is greater than up threshold increase the + * frequency to maximum as supported by host. Else, + * if load is less than down threshold, scale down the + * frequency to minimum supported by the host. Otherwise, + * retain current frequency and do nothing. + */ +static void mmc_clk_scaling(struct mmc_host *host, bool from_wq) +{ + int err = 0; + struct mmc_card *card = host->card; + unsigned long total_time_ms = 0; + unsigned long busy_time_ms = 0; + unsigned long freq; + unsigned int up_threshold = host->clk_scaling.up_threshold; + unsigned int down_threshold = host->clk_scaling.down_threshold; + bool queue_scale_down_work = false; + + if (!card || !host->bus_ops || !host->bus_ops->change_bus_speed) { + pr_err("%s: %s: invalid entry\n", mmc_hostname(host), __func__); + goto out; + } + + /* Check if the clocks are already gated. */ + if (!host->ios.clock) + goto out; + + if (time_is_after_jiffies(host->clk_scaling.window_time + + msecs_to_jiffies(host->clk_scaling.polling_delay_ms))) + goto out; + + /* handle time wrap */ + total_time_ms = jiffies_to_msecs((long)jiffies - + (long)host->clk_scaling.window_time); + + /* Check if we re-enter during clock switching */ + if (unlikely(host->clk_scaling.in_progress)) + goto out; + + host->clk_scaling.in_progress = true; + + busy_time_ms = host->clk_scaling.busy_time_us / USEC_PER_MSEC; + + freq = host->clk_scaling.curr_freq; + + /* + * Note that the max. and min. frequency should be based + * on the timing modes that the card and host handshake + * during initialization. + */ + if ((busy_time_ms * 100 > total_time_ms * up_threshold)) { + freq = mmc_get_max_frequency(host); + } else if ((busy_time_ms * 100 < total_time_ms * down_threshold)) { + if (!from_wq) + queue_scale_down_work = true; + freq = mmc_get_min_frequency(host); + } + + if (freq != host->clk_scaling.curr_freq) { + if (!queue_scale_down_work) { + if (!from_wq) + cancel_delayed_work_sync( + &host->clk_scaling.work); + err = host->bus_ops->change_bus_speed(host, &freq); + if (!err) + host->clk_scaling.curr_freq = freq; + else + pr_err("%s: %s: failed (%d) at freq=%lu\n", + mmc_hostname(host), __func__, err, + freq); + } else { + /* + * We hold claim host while queueing the scale down + * work, so delay atleast one timer tick to release + * host and re-claim while scaling down the clocks. + */ + queue_delayed_work(system_wq, + &host->clk_scaling.work, 1); + host->clk_scaling.in_progress = false; + goto out; + } + } + + mmc_reset_clk_scale_stats(host); + host->clk_scaling.in_progress = false; +out: + return; +} + +/** + * mmc_disable_clk_scaling() - Disable clock scaling + * @host: pointer to mmc host structure + * + * Disables clock scaling temporarily by setting enable + * property to false. To disable completely, one also + * need to set 'initialized' variable to false. + */ +void mmc_disable_clk_scaling(struct mmc_host *host) +{ + cancel_delayed_work_sync(&host->clk_scaling.work); + host->clk_scaling.enable = false; +} +EXPORT_SYMBOL_GPL(mmc_disable_clk_scaling); + +/** + * mmc_can_scale_clk() - Check if clock scaling is initialized + * @host: pointer to mmc host structure + */ +bool mmc_can_scale_clk(struct mmc_host *host) +{ + return host->clk_scaling.initialized; +} +EXPORT_SYMBOL_GPL(mmc_can_scale_clk); + +/** + * mmc_init_clk_scaling() - Initialize clock scaling + * @host: pointer to mmc host structure + * + * Initialize clock scaling for supported hosts. + * It is assumed that the caller ensure clock is + * running at maximum possible frequency before + * calling this function. + */ +void mmc_init_clk_scaling(struct mmc_host *host) +{ + if (!host->card || !(host->caps2 & MMC_CAP2_CLK_SCALE)) + return; + + INIT_DELAYED_WORK(&host->clk_scaling.work, mmc_clk_scale_work); + host->clk_scaling.curr_freq = mmc_get_max_frequency(host); + mmc_reset_clk_scale_stats(host); + host->clk_scaling.enable = true; + host->clk_scaling.initialized = true; + pr_debug("%s: clk scaling enabled\n", mmc_hostname(host)); +} +EXPORT_SYMBOL_GPL(mmc_init_clk_scaling); + +/** + * mmc_exit_clk_scaling() - Disable clock scaling + * @host: pointer to mmc host structure + * + * Disable clock scaling permanently. + */ +void mmc_exit_clk_scaling(struct mmc_host *host) +{ + cancel_delayed_work_sync(&host->clk_scaling.work); + memset(&host->clk_scaling, 0, sizeof(host->clk_scaling)); +} +EXPORT_SYMBOL_GPL(mmc_exit_clk_scaling); + static int mmc_rescan_try_freq(struct mmc_host *host, unsigned freq) { host->f_init = freq; diff --git a/drivers/mmc/core/core.h b/drivers/mmc/core/core.h index d57daacbfcca..9bf804ee35a2 100644 --- a/drivers/mmc/core/core.h +++ b/drivers/mmc/core/core.h @@ -92,6 +92,13 @@ void mmc_remove_card_debugfs(struct mmc_card *card); void mmc_init_context_info(struct mmc_host *host); +extern void mmc_disable_clk_scaling(struct mmc_host *host); +extern bool mmc_can_scale_clk(struct mmc_host *host); +extern void mmc_init_clk_scaling(struct mmc_host *host); +extern void mmc_exit_clk_scaling(struct mmc_host *host); +extern void mmc_reset_clk_scale_stats(struct mmc_host *host); +extern unsigned long mmc_get_max_frequency(struct mmc_host *host); + int mmc_execute_tuning(struct mmc_card *card); int mmc_hs200_to_hs400(struct mmc_card *card); int mmc_hs400_to_hs200(struct mmc_card *card); diff --git a/drivers/mmc/core/host.c b/drivers/mmc/core/host.c index 44560b15ce57..90c863688806 100644 --- a/drivers/mmc/core/host.c +++ b/drivers/mmc/core/host.c @@ -167,6 +167,9 @@ void mmc_host_clk_hold(struct mmc_host *host) if (host->clk_gated) { spin_unlock_irqrestore(&host->clk_lock, flags); mmc_ungate_clock(host); + + /* Reset clock scaling stats as host is out of idle */ + mmc_reset_clk_scale_stats(host); spin_lock_irqsave(&host->clk_lock, flags); pr_debug("%s: ungated MCI clock\n", mmc_hostname(host)); } @@ -697,6 +700,10 @@ int mmc_add_host(struct mmc_host *host) #endif mmc_host_clk_sysfs_init(host); + host->clk_scaling.up_threshold = 35; + host->clk_scaling.down_threshold = 5; + host->clk_scaling.polling_delay_ms = 100; + err = sysfs_create_group(&host->parent->kobj, &dev_attr_grp); if (err) pr_err("%s: failed to create sysfs group with err %d\n", diff --git a/drivers/mmc/core/mmc.c b/drivers/mmc/core/mmc.c index 05b5dff1da4f..d020236477bb 100644 --- a/drivers/mmc/core/mmc.c +++ b/drivers/mmc/core/mmc.c @@ -1848,6 +1848,7 @@ static void mmc_remove(struct mmc_host *host) mmc_claim_host(host); host->card = NULL; + mmc_exit_clk_scaling(host); mmc_release_host(host); } @@ -1902,6 +1903,12 @@ static int _mmc_suspend(struct mmc_host *host, bool is_suspend) if (mmc_card_suspended(host->card)) goto out; + /* + * Disable clock scaling before suspend and enable it after resume so + * as to avoid clock scaling decisions kicking in during this window. + */ + mmc_disable_clk_scaling(host); + if (mmc_card_doing_bkops(host->card)) { err = mmc_stop_bkops(host->card); if (err) @@ -1958,15 +1965,25 @@ static int _mmc_resume(struct mmc_host *host) mmc_claim_host(host); - if (!mmc_card_suspended(host->card)) + if (!mmc_card_suspended(host->card)) { + mmc_release_host(host); goto out; + } mmc_power_up(host, host->card->ocr); err = mmc_init_card(host, host->card->ocr, host->card); mmc_card_clr_suspended(host->card); -out: mmc_release_host(host); + + /* + * We have done full initialization of the card, + * reset the clk scale stats and current frequency. + */ + if (mmc_can_scale_clk(host)) + mmc_init_clk_scaling(host); + +out: return err; } @@ -2041,6 +2058,10 @@ static int mmc_runtime_resume(struct mmc_host *host) pr_err("%s: error %d doing aggressive resume\n", mmc_hostname(host), err); + /* Initialize clock scaling only for high frequency modes */ + if (mmc_card_hs200(host->card)) + mmc_init_clk_scaling(host); + return 0; } diff --git a/drivers/mmc/core/sd.c b/drivers/mmc/core/sd.c index 29b3447bff9e..ccab284c3ce5 100644 --- a/drivers/mmc/core/sd.c +++ b/drivers/mmc/core/sd.c @@ -1113,6 +1113,7 @@ static void mmc_sd_remove(struct mmc_host *host) mmc_claim_host(host); host->card = NULL; + mmc_exit_clk_scaling(host); mmc_release_host(host); } @@ -1180,6 +1181,12 @@ static int _mmc_sd_suspend(struct mmc_host *host) BUG_ON(!host); BUG_ON(!host->card); + /* + * Disable clock scaling before suspend and enable it after resume so + * as to avoid clock scaling decisions kicking in during this window. + */ + mmc_disable_clk_scaling(host); + mmc_claim_host(host); if (mmc_card_suspended(host->card)) @@ -1293,6 +1300,13 @@ static int mmc_sd_runtime_suspend(struct mmc_host *host) pr_err("%s: error %d doing aggressive suspend\n", mmc_hostname(host), err); + /* + * We have done full initialization of the card, + * reset the clk scale stats and current frequency. + */ + if (mmc_can_scale_clk(host)) + mmc_init_clk_scaling(host); + return err; } @@ -1311,6 +1325,10 @@ static int mmc_sd_runtime_resume(struct mmc_host *host) pr_err("%s: error %d doing aggressive resume\n", mmc_hostname(host), err); + /* Initialize clock scaling only for high frequency modes */ + if (mmc_card_uhs(host->card)) + mmc_init_clk_scaling(host); + return 0; } diff --git a/include/linux/mmc/host.h b/include/linux/mmc/host.h index 1f4c2cc72b8c..dd68bc9a805d 100644 --- a/include/linux/mmc/host.h +++ b/include/linux/mmc/host.h @@ -156,6 +156,10 @@ struct mmc_host_ops { */ int (*multi_io_quirk)(struct mmc_card *card, unsigned int direction, int blk_size); + + unsigned long (*get_max_frequency)(struct mmc_host *host); + unsigned long (*get_min_frequency)(struct mmc_host *host); + int (*notify_load)(struct mmc_host *, enum mmc_load); }; @@ -303,6 +307,7 @@ struct mmc_host { #define MMC_CAP2_SDIO_IRQ_NOTHREAD (1 << 17) #define MMC_CAP2_NO_WRITE_PROTECT (1 << 18) /* No physical write protect pin, assume that card is always read-write */ #define MMC_CAP2_PACKED_WR_CONTROL (1 << 19) /* Allow write packing control */ +#define MMC_CAP2_CLK_SCALE (1 << 20) /* Allow dynamic clk scaling */ #define MMC_CAP2_NONHOTPLUG (1 << 25) /*Don't support hotplug*/ mmc_pm_flag_t pm_caps; /* supported pm features */ @@ -407,23 +412,6 @@ struct mmc_host { } embedded_sdio_data; #endif - struct { - unsigned long busy_time_us; - unsigned long window_time; - unsigned long curr_freq; - unsigned long polling_delay_ms; - unsigned int up_threshold; - unsigned int down_threshold; - ktime_t start_busy; - bool enable; - bool initialized; - bool in_progress; - /* freq. transitions are not allowed in invalid state */ - bool invalid_state; - struct delayed_work work; - enum mmc_load state; - } clk_scaling; - /* * Set to 1 to just stop the SDCLK to the card without * actually disabling the clock from it's source. @@ -441,7 +429,19 @@ struct mmc_host { } perf; bool perf_enable; #endif - + struct { + unsigned long busy_time_us; + unsigned long window_time; + unsigned long curr_freq; + unsigned long polling_delay_ms; + unsigned int up_threshold; + unsigned int down_threshold; + ktime_t start_busy; + bool enable; + bool initialized; + bool in_progress; + struct delayed_work work; + } clk_scaling; unsigned long private[0] ____cacheline_aligned; };