mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-25 02:09:42 +00:00
![Vincent Guittot](/assets/img/avatar_default.png)
The current implementation of load tracking invariance scales the contribution with current frequency and uarch performance (only for utilization) of the CPU. One main result of this formula is that the figures are capped by current capacity of CPU. Another one is that the load_avg is not invariant because not scaled with uarch. The util_avg of a periodic task that runs r time slots every p time slots varies in the range : U * (1-y^r)/(1-y^p) * y^i < Utilization < U * (1-y^r)/(1-y^p) with U is the max util_avg value = SCHED_CAPACITY_SCALE At a lower capacity, the range becomes: U * C * (1-y^r')/(1-y^p) * y^i' < Utilization < U * C * (1-y^r')/(1-y^p) with C reflecting the compute capacity ratio between current capacity and max capacity. so C tries to compensate changes in (1-y^r') but it can't be accurate. Instead of scaling the contribution value of PELT algo, we should scale the running time. The PELT signal aims to track the amount of computation of tasks and/or rq so it seems more correct to scale the running time to reflect the effective amount of computation done since the last update. In order to be fully invariant, we need to apply the same amount of running time and idle time whatever the current capacity. Because running at lower capacity implies that the task will run longer, we have to ensure that the same amount of idle time will be applied when system becomes idle and no idle time has been "stolen". But reaching the maximum utilization value (SCHED_CAPACITY_SCALE) means that the task is seen as an always-running task whatever the capacity of the CPU (even at max compute capacity). In this case, we can discard this "stolen" idle times which becomes meaningless. In order to achieve this time scaling, a new clock_pelt is created per rq. The increase of this clock scales with current capacity when something is running on rq and synchronizes with clock_task when rq is idle. With this mechanism, we ensure the same running and idle time whatever the current capacity. This also enables to simplify the pelt algorithm by removing all references of uarch and frequency and applying the same contribution to utilization and loads. Furthermore, the scaling is done only once per update of clock (update_rq_clock_task()) instead of during each update of sched_entities and cfs/rt/dl_rq of the rq like the current implementation. This is interesting when cgroup are involved as shown in the results below: On a hikey (octo Arm64 platform). Performance cpufreq governor and only shallowest c-state to remove variance generated by those power features so we only track the impact of pelt algo. each test runs 16 times: ./perf bench sched pipe (higher is better) kernel tip/sched/core + patch ops/seconds ops/seconds diff cgroup root 59652(+/- 0.18%) 59876(+/- 0.24%) +0.38% level1 55608(+/- 0.27%) 55923(+/- 0.24%) +0.57% level2 52115(+/- 0.29%) 52564(+/- 0.22%) +0.86% hackbench -l 1000 (lower is better) kernel tip/sched/core + patch duration(sec) duration(sec) diff cgroup root 4.453(+/- 2.37%) 4.383(+/- 2.88%) -1.57% level1 4.859(+/- 8.50%) 4.830(+/- 7.07%) -0.60% level2 5.063(+/- 9.83%) 4.928(+/- 9.66%) -2.66% Then, the responsiveness of PELT is improved when CPU is not running at max capacity with this new algorithm. I have put below some examples of duration to reach some typical load values according to the capacity of the CPU with current implementation and with this patch. These values has been computed based on the geometric series and the half period value: Util (%) max capacity half capacity(mainline) half capacity(w/ patch) 972 (95%) 138ms not reachable 276ms 486 (47.5%) 30ms 138ms 60ms 256 (25%) 13ms 32ms 26ms On my hikey (octo Arm64 platform) with schedutil governor, the time to reach max OPP when starting from a null utilization, decreases from 223ms with current scale invariance down to 121ms with the new algorithm. Signed-off-by: Vincent Guittot <vincent.guittot@linaro.org> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Mike Galbraith <efault@gmx.de> Cc: Morten.Rasmussen@arm.com Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: bsegall@google.com Cc: dietmar.eggemann@arm.com Cc: patrick.bellasi@arm.com Cc: pjt@google.com Cc: pkondeti@codeaurora.org Cc: quentin.perret@arm.com Cc: rjw@rjwysocki.net Cc: srinivas.pandruvada@linux.intel.com Cc: thara.gopinath@linaro.org Link: https://lkml.kernel.org/r/1548257214-13745-3-git-send-email-vincent.guittot@linaro.org Signed-off-by: Ingo Molnar <mingo@kernel.org>
180 lines
5 KiB
C
180 lines
5 KiB
C
#ifdef CONFIG_SMP
|
|
#include "sched-pelt.h"
|
|
|
|
int __update_load_avg_blocked_se(u64 now, struct sched_entity *se);
|
|
int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se);
|
|
int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq);
|
|
int update_rt_rq_load_avg(u64 now, struct rq *rq, int running);
|
|
int update_dl_rq_load_avg(u64 now, struct rq *rq, int running);
|
|
|
|
#ifdef CONFIG_HAVE_SCHED_AVG_IRQ
|
|
int update_irq_load_avg(struct rq *rq, u64 running);
|
|
#else
|
|
static inline int
|
|
update_irq_load_avg(struct rq *rq, u64 running)
|
|
{
|
|
return 0;
|
|
}
|
|
#endif
|
|
|
|
/*
|
|
* When a task is dequeued, its estimated utilization should not be update if
|
|
* its util_avg has not been updated at least once.
|
|
* This flag is used to synchronize util_avg updates with util_est updates.
|
|
* We map this information into the LSB bit of the utilization saved at
|
|
* dequeue time (i.e. util_est.dequeued).
|
|
*/
|
|
#define UTIL_AVG_UNCHANGED 0x1
|
|
|
|
static inline void cfs_se_util_change(struct sched_avg *avg)
|
|
{
|
|
unsigned int enqueued;
|
|
|
|
if (!sched_feat(UTIL_EST))
|
|
return;
|
|
|
|
/* Avoid store if the flag has been already set */
|
|
enqueued = avg->util_est.enqueued;
|
|
if (!(enqueued & UTIL_AVG_UNCHANGED))
|
|
return;
|
|
|
|
/* Reset flag to report util_avg has been updated */
|
|
enqueued &= ~UTIL_AVG_UNCHANGED;
|
|
WRITE_ONCE(avg->util_est.enqueued, enqueued);
|
|
}
|
|
|
|
/*
|
|
* The clock_pelt scales the time to reflect the effective amount of
|
|
* computation done during the running delta time but then sync back to
|
|
* clock_task when rq is idle.
|
|
*
|
|
*
|
|
* absolute time | 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15|16
|
|
* @ max capacity ------******---------------******---------------
|
|
* @ half capacity ------************---------************---------
|
|
* clock pelt | 1| 2| 3| 4| 7| 8| 9| 10| 11|14|15|16
|
|
*
|
|
*/
|
|
static inline void update_rq_clock_pelt(struct rq *rq, s64 delta)
|
|
{
|
|
if (unlikely(is_idle_task(rq->curr))) {
|
|
/* The rq is idle, we can sync to clock_task */
|
|
rq->clock_pelt = rq_clock_task(rq);
|
|
return;
|
|
}
|
|
|
|
/*
|
|
* When a rq runs at a lower compute capacity, it will need
|
|
* more time to do the same amount of work than at max
|
|
* capacity. In order to be invariant, we scale the delta to
|
|
* reflect how much work has been really done.
|
|
* Running longer results in stealing idle time that will
|
|
* disturb the load signal compared to max capacity. This
|
|
* stolen idle time will be automatically reflected when the
|
|
* rq will be idle and the clock will be synced with
|
|
* rq_clock_task.
|
|
*/
|
|
|
|
/*
|
|
* Scale the elapsed time to reflect the real amount of
|
|
* computation
|
|
*/
|
|
delta = cap_scale(delta, arch_scale_cpu_capacity(NULL, cpu_of(rq)));
|
|
delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq)));
|
|
|
|
rq->clock_pelt += delta;
|
|
}
|
|
|
|
/*
|
|
* When rq becomes idle, we have to check if it has lost idle time
|
|
* because it was fully busy. A rq is fully used when the /Sum util_sum
|
|
* is greater or equal to:
|
|
* (LOAD_AVG_MAX - 1024 + rq->cfs.avg.period_contrib) << SCHED_CAPACITY_SHIFT;
|
|
* For optimization and computing rounding purpose, we don't take into account
|
|
* the position in the current window (period_contrib) and we use the higher
|
|
* bound of util_sum to decide.
|
|
*/
|
|
static inline void update_idle_rq_clock_pelt(struct rq *rq)
|
|
{
|
|
u32 divider = ((LOAD_AVG_MAX - 1024) << SCHED_CAPACITY_SHIFT) - LOAD_AVG_MAX;
|
|
u32 util_sum = rq->cfs.avg.util_sum;
|
|
util_sum += rq->avg_rt.util_sum;
|
|
util_sum += rq->avg_dl.util_sum;
|
|
|
|
/*
|
|
* Reflecting stolen time makes sense only if the idle
|
|
* phase would be present at max capacity. As soon as the
|
|
* utilization of a rq has reached the maximum value, it is
|
|
* considered as an always runnig rq without idle time to
|
|
* steal. This potential idle time is considered as lost in
|
|
* this case. We keep track of this lost idle time compare to
|
|
* rq's clock_task.
|
|
*/
|
|
if (util_sum >= divider)
|
|
rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt;
|
|
}
|
|
|
|
static inline u64 rq_clock_pelt(struct rq *rq)
|
|
{
|
|
lockdep_assert_held(&rq->lock);
|
|
assert_clock_updated(rq);
|
|
|
|
return rq->clock_pelt - rq->lost_idle_time;
|
|
}
|
|
|
|
#ifdef CONFIG_CFS_BANDWIDTH
|
|
/* rq->task_clock normalized against any time this cfs_rq has spent throttled */
|
|
static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
|
|
{
|
|
if (unlikely(cfs_rq->throttle_count))
|
|
return cfs_rq->throttled_clock_task - cfs_rq->throttled_clock_task_time;
|
|
|
|
return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time;
|
|
}
|
|
#else
|
|
static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
|
|
{
|
|
return rq_clock_pelt(rq_of(cfs_rq));
|
|
}
|
|
#endif
|
|
|
|
#else
|
|
|
|
static inline int
|
|
update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline int
|
|
update_rt_rq_load_avg(u64 now, struct rq *rq, int running)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline int
|
|
update_dl_rq_load_avg(u64 now, struct rq *rq, int running)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline int
|
|
update_irq_load_avg(struct rq *rq, u64 running)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline u64 rq_clock_pelt(struct rq *rq)
|
|
{
|
|
return rq_clock_task(rq);
|
|
}
|
|
|
|
static inline void
|
|
update_rq_clock_pelt(struct rq *rq, s64 delta) { }
|
|
|
|
static inline void
|
|
update_idle_rq_clock_pelt(struct rq *rq) { }
|
|
|
|
#endif
|
|
|
|
|