__enqueue_entity(cfs_rq, se);
}
-static void update_avg(u64 *avg, u64 sample)
-{
- s64 diff = sample - *avg;
- *avg += diff >> 3;
-}
-
-static void update_avg_stats(struct cfs_rq *cfs_rq, struct sched_entity *se)
-{
- if (!se->last_wakeup)
- return;
-
- update_avg(&se->avg_overlap, se->sum_exec_runtime - se->last_wakeup);
- se->last_wakeup = 0;
-}
-
static void
dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep)
{
update_stats_dequeue(cfs_rq, se);
if (sleep) {
- update_avg_stats(cfs_rq, se);
#ifdef CONFIG_SCHEDSTATS
if (entity_is_task(se)) {
struct task_struct *tsk = task_of(se);
hrtick_start(rq, delta, requeue);
}
}
-#else
+#else /* !CONFIG_SCHED_HRTICK */
static inline void
hrtick_start_fair(struct rq *rq, struct task_struct *p)
{
}
return cpu;
}
-#else
+#else /* !ARCH_HAS_SCHED_WAKE_IDLE*/
static inline int wake_idle(int cpu, struct task_struct *p)
{
return cpu;
static const struct sched_class fair_sched_class;
#ifdef CONFIG_FAIR_GROUP_SCHED
-static unsigned long effective_load(struct task_group *tg, int cpu,
- unsigned long wl, unsigned long wg)
+/*
+ * effective_load() calculates the load change as seen from the root_task_group
+ *
+ * Adding load to a group doesn't make a group heavier, but can cause movement
+ * of group shares between cpus. Assuming the shares were perfectly aligned one
+ * can calculate the shift in shares.
+ *
+ * The problem is that perfectly aligning the shares is rather expensive, hence
+ * we try to avoid doing that too often - see update_shares(), which ratelimits
+ * this change.
+ *
+ * We compensate this by not only taking the current delta into account, but
+ * also considering the delta between when the shares were last adjusted and
+ * now.
+ *
+ * We still saw a performance dip, some tracing learned us that between
+ * cgroup:/ and cgroup:/foo balancing the number of affine wakeups increased
+ * significantly. Therefore try to bias the error in direction of failing
+ * the affine wakeup.
+ *
+ */
+static long effective_load(struct task_group *tg, int cpu,
+ long wl, long wg)
{
struct sched_entity *se = tg->se[cpu];
+ long more_w;
+
+ if (!tg->parent)
+ return wl;
+
+ /*
+ * By not taking the decrease of shares on the other cpu into
+ * account our error leans towards reducing the affine wakeups.
+ */
+ if (!wl && sched_feat(ASYM_EFF_LOAD))
+ return wl;
+
+ /*
+ * Instead of using this increment, also add the difference
+ * between when the shares were last updated and now.
+ */
+ more_w = se->my_q->load.weight - se->my_q->rq_weight;
+ wl += more_w;
+ wg += more_w;
for_each_sched_entity(se) {
#define D(n) (likely(n) ? (n) : 1)
S = se->my_q->tg->shares;
s = se->my_q->shares;
- rw = se->my_q->load.weight;
+ rw = se->my_q->rq_weight;
a = S*(rw + wl);
b = S*rw + s*wg;
* a reasonable amount of time then attract this newly
* woken task:
*/
- if (sync && balanced && curr->sched_class == &fair_sched_class) {
+ if (sync && balanced) {
if (curr->se.avg_overlap < sysctl_sched_migration_cost &&
- p->se.avg_overlap < sysctl_sched_migration_cost)
+ p->se.avg_overlap < sysctl_sched_migration_cost)
return 1;
}
return;
}
- se->last_wakeup = se->sum_exec_runtime;
if (unlikely(se == pse))
return;
return 0;
}
-#endif
+#endif /* CONFIG_SMP */
/*
* scheduler tick hitting a task of our scheduling class: