diff options
| author | Peter Zijlstra <peterz@infradead.org> | 2013-09-07 19:33:51 -0500 |
|---|---|---|
| committer | Mister Oyster <oysterized@gmail.com> | 2017-09-11 11:31:30 +0200 |
| commit | 0425711d7e7c58a232e5a421028cf8ea8d1b48b4 (patch) | |
| tree | a2060282e99e8a84e6f23878c6d23593fb76a1f9 /kernel | |
| parent | d354b465652dfabe1ca553f751f2b091d43595c0 (diff) | |
sched: Micro-optimize the smart wake-affine logic
Smart wake-affine is using node-size as the factor currently, but the overhead
of the mask operation is high.
Thus, this patch introduce the 'sd_llc_size' percpu variable, which will record
the highest cache-share domain size, and make it to be the new factor, in order
to reduce the overhead and make it more reasonable.
Tested-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Tested-by: Michael Wang <wangyun@linux.vnet.ibm.com>
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Acked-by: Michael Wang <wangyun@linux.vnet.ibm.com>
Cc: Mike Galbraith <efault@gmx.de>
Link: http://lkml.kernel.org/r/51D5008E.6030102@linux.vnet.ibm.com
[ Tidied up the changelog. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Change-Id: I8fb3ceac1e6944db078932172c99d544a4e304e4
Signed-off-by: Paul Reioux <reioux@gmail.com>
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/sched/core.c | 7 | ||||
| -rw-r--r-- | kernel/sched/fair.c | 2 | ||||
| -rw-r--r-- | kernel/sched/sched.h | 1 |
3 files changed, 8 insertions, 2 deletions
diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 8ea31ccc0..0fa11db49 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6254,18 +6254,23 @@ static void destroy_sched_domains(struct sched_domain *sd, int cpu) * two cpus are in the same cache domain, see cpus_share_cache(). */ DEFINE_PER_CPU(struct sched_domain *, sd_llc); +DEFINE_PER_CPU(int, sd_llc_size); DEFINE_PER_CPU(int, sd_llc_id); static void update_top_cache_domain(int cpu) { struct sched_domain *sd; int id = cpu; + int size = 1; sd = highest_flag_domain(cpu, SD_SHARE_PKG_RESOURCES); - if (sd) + if (sd) { id = cpumask_first(sched_domain_span(sd)); + size = cpumask_weight(sched_domain_span(sd)); + } rcu_assign_pointer(per_cpu(sd_llc, cpu), sd); + per_cpu(sd_llc_size, cpu) = size; per_cpu(sd_llc_id, cpu) = id; } diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 6a1dcc0bc..aa821712b 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -4756,7 +4756,7 @@ static inline unsigned long effective_load(struct task_group *tg, int cpu, static int wake_wide(struct task_struct *p) { - int factor = nr_cpus_node(cpu_to_node(smp_processor_id())); + int factor = this_cpu_read(sd_llc_size); /* * Yeah, it's the switching-frequency, could means many wakee or diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index f3d6533d1..4f9d15f0b 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -585,6 +585,7 @@ static inline struct sched_domain *highest_flag_domain(int cpu, int flag) } DECLARE_PER_CPU(struct sched_domain *, sd_llc); +DECLARE_PER_CPU(int, sd_llc_size); DECLARE_PER_CPU(int, sd_llc_id); struct sched_group_power { |