diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c
index 55b44559bae4..ac815c6dbac0 100644
--- a/drivers/clk/clk.c
+++ b/drivers/clk/clk.c
@@ -197,6 +197,19 @@ static void clk_unprepare_unused_subtree(struct clk_core *core)
 	hlist_for_each_entry(child, &core->children, child_node)
 		clk_unprepare_unused_subtree(child);
 
+	/*
+	 * setting CLK_ENABLE_HAND_OFF flag triggers this conditional
+	 *
+	 * need_handoff_prepare implies this clk was already prepared by
+	 * __clk_init. now we have a proper user, so unset the flag in our
+	 * internal bookkeeping. See CLK_ENABLE_HAND_OFF flag in clk-provider.h
+	 * for details.
+	 */
+	if (core->need_handoff_prepare) {
+		core->need_handoff_prepare = false;
+		core->prepare_count--;
+	}
+
 	if (core->prepare_count)
 		return;
 
@@ -223,6 +236,19 @@ static void clk_disable_unused_subtree(struct clk_core *core)
 	hlist_for_each_entry(child, &core->children, child_node)
 		clk_disable_unused_subtree(child);
 
+	/*
+	 * setting CLK_ENABLE_HAND_OFF flag triggers this conditional
+	 *
+	 * need_handoff_enable implies this clk was already enabled by
+	 * __clk_init. now we have a proper user, so unset the flag in our
+	 * internal bookkeeping. See CLK_ENABLE_HAND_OFF flag in clk-provider.h
+	 * for details.
+	 */
+	if (core->need_handoff_enable) {
+		core->need_handoff_enable = false;
+		core->enable_count--;
+	}
+
 	flags = clk_enable_lock();
 
 	if (core->enable_count)
@@ -932,24 +958,10 @@ int clk_prepare(struct clk *clk)
 	if (!clk)
 		return 0;
 
-	/*
-	 * setting CLK_ENABLE_HAND_OFF flag triggers this conditional
-	 *
-	 * need_handoff_prepare implies this clk was already prepared by
-	 * __clk_init. now we have a proper user, so unset the flag in our
-	 * internal bookkeeping. See CLK_ENABLE_HAND_OFF flag in clk-provider.h
-	 * for details.
-	 */
-	if (clk->core->need_handoff_prepare) {
-		clk->core->need_handoff_prepare = false;
-		goto out;
-	}
-
 	clk_prepare_lock();
 	ret = clk_core_prepare(clk->core);
 	clk_prepare_unlock();
 
-out:
 	return ret;
 }
 EXPORT_SYMBOL_GPL(clk_prepare);
@@ -1061,24 +1073,10 @@ int clk_enable(struct clk *clk)
 	if (!clk)
 		return 0;
 
-	/*
-	 * setting CLK_ENABLE_HAND_OFF flag triggers this conditional
-	 *
-	 * need_handoff_enable implies this clk was already enabled by
-	 * __clk_init. now we have a proper user, so unset the flag in our
-	 * internal bookkeeping. See CLK_ENABLE_HAND_OFF flag in clk-provider.h
-	 * for details.
-	 */
-	if (clk->core->need_handoff_enable) {
-		clk->core->need_handoff_enable = false;
-		goto out;
-	}
-
 	flags = clk_enable_lock();
 	ret = clk_core_enable(clk->core);
 	clk_enable_unlock(flags);
 
-out:
 	return ret;
 }
 EXPORT_SYMBOL_GPL(clk_enable);
@@ -3189,14 +3187,22 @@ static int __clk_init(struct device *dev, struct clk *clk_user)
 	if (core->flags & CLK_ENABLE_HAND_OFF) {
 		unsigned long flags;
 
-		core->need_handoff_prepare = true;
-		core->need_handoff_enable = true;
-		ret = clk_core_prepare(core);
-		if (ret)
-			goto out;
-		flags = clk_enable_lock();
-		clk_core_enable(core);
-		clk_enable_unlock(flags);
+		/*
+		 * Few clocks might have hardware gating which would be required
+		 * to be ON before prepare/enabling the clocks. So check if the
+		 * clock has been turned ON earlier and we should
+		 * prepare/enable those clocks.
+		 */
+		if (clk_core_is_enabled(core)) {
+			core->need_handoff_prepare = true;
+			core->need_handoff_enable = true;
+			ret = clk_core_prepare(core);
+			if (ret)
+				goto out;
+			flags = clk_enable_lock();
+			clk_core_enable(core);
+			clk_enable_unlock(flags);
+		}
 	}
 
 	kref_init(&core->ref);