diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 19d735ab44db..cd2fb8384fbe 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -7313,9 +7313,10 @@ more_balance:
 out_balanced:
 	/*
 	 * We reach balance although we may have faced some affinity
-	 * constraints. Clear the imbalance flag if it was set.
+	 * constraints. Clear the imbalance flag only if other tasks got
+	 * a chance to move and fix the imbalance.
 	 */
-	if (sd_parent) {
+	if (sd_parent && !(env.flags & LBF_ALL_PINNED)) {
 		int *group_imbalance = &sd_parent->groups->sgc->imbalance;
 
 		if (*group_imbalance)