Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * Completely Fair Scheduling (CFS) Class (SCHED_NORMAL/SCHED_BATCH)
4 : *
5 : * Copyright (C) 2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
6 : *
7 : * Interactivity improvements by Mike Galbraith
8 : * (C) 2007 Mike Galbraith <efault@gmx.de>
9 : *
10 : * Various enhancements by Dmitry Adamushko.
11 : * (C) 2007 Dmitry Adamushko <dmitry.adamushko@gmail.com>
12 : *
13 : * Group scheduling enhancements by Srivatsa Vaddagiri
14 : * Copyright IBM Corporation, 2007
15 : * Author: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
16 : *
17 : * Scaled math optimizations by Thomas Gleixner
18 : * Copyright (C) 2007, Thomas Gleixner <tglx@linutronix.de>
19 : *
20 : * Adaptive scheduling granularity, math enhancements by Peter Zijlstra
21 : * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra
22 : */
23 : #include <linux/energy_model.h>
24 : #include <linux/mmap_lock.h>
25 : #include <linux/hugetlb_inline.h>
26 : #include <linux/jiffies.h>
27 : #include <linux/mm_api.h>
28 : #include <linux/highmem.h>
29 : #include <linux/spinlock_api.h>
30 : #include <linux/cpumask_api.h>
31 : #include <linux/lockdep_api.h>
32 : #include <linux/softirq.h>
33 : #include <linux/refcount_api.h>
34 : #include <linux/topology.h>
35 : #include <linux/sched/clock.h>
36 : #include <linux/sched/cond_resched.h>
37 : #include <linux/sched/cputime.h>
38 : #include <linux/sched/isolation.h>
39 :
40 : #include <linux/cpuidle.h>
41 : #include <linux/interrupt.h>
42 : #include <linux/mempolicy.h>
43 : #include <linux/mutex_api.h>
44 : #include <linux/profile.h>
45 : #include <linux/psi.h>
46 : #include <linux/ratelimit.h>
47 : #include <linux/task_work.h>
48 :
49 : #include <asm/switch_to.h>
50 :
51 : #include <linux/sched/cond_resched.h>
52 :
53 : #include "sched.h"
54 : #include "stats.h"
55 : #include "autogroup.h"
56 :
57 : /*
58 : * Targeted preemption latency for CPU-bound tasks:
59 : *
60 : * NOTE: this latency value is not the same as the concept of
61 : * 'timeslice length' - timeslices in CFS are of variable length
62 : * and have no persistent notion like in traditional, time-slice
63 : * based scheduling concepts.
64 : *
65 : * (to see the precise effective timeslice length of your workload,
66 : * run vmstat and monitor the context-switches (cs) field)
67 : *
68 : * (default: 6ms * (1 + ilog(ncpus)), units: nanoseconds)
69 : */
70 : unsigned int sysctl_sched_latency = 6000000ULL;
71 : static unsigned int normalized_sysctl_sched_latency = 6000000ULL;
72 :
73 : /*
74 : * The initial- and re-scaling of tunables is configurable
75 : *
76 : * Options are:
77 : *
78 : * SCHED_TUNABLESCALING_NONE - unscaled, always *1
79 : * SCHED_TUNABLESCALING_LOG - scaled logarithmical, *1+ilog(ncpus)
80 : * SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus
81 : *
82 : * (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus))
83 : */
84 : unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG;
85 :
86 : /*
87 : * Minimal preemption granularity for CPU-bound tasks:
88 : *
89 : * (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds)
90 : */
91 : unsigned int sysctl_sched_min_granularity = 750000ULL;
92 : static unsigned int normalized_sysctl_sched_min_granularity = 750000ULL;
93 :
94 : /*
95 : * Minimal preemption granularity for CPU-bound SCHED_IDLE tasks.
96 : * Applies only when SCHED_IDLE tasks compete with normal tasks.
97 : *
98 : * (default: 0.75 msec)
99 : */
100 : unsigned int sysctl_sched_idle_min_granularity = 750000ULL;
101 :
102 : /*
103 : * This value is kept at sysctl_sched_latency/sysctl_sched_min_granularity
104 : */
105 : static unsigned int sched_nr_latency = 8;
106 :
107 : /*
108 : * After fork, child runs first. If set to 0 (default) then
109 : * parent will (try to) run first.
110 : */
111 : unsigned int sysctl_sched_child_runs_first __read_mostly;
112 :
113 : /*
114 : * SCHED_OTHER wake-up granularity.
115 : *
116 : * This option delays the preemption effects of decoupled workloads
117 : * and reduces their over-scheduling. Synchronous workloads will still
118 : * have immediate wakeup/sleep latencies.
119 : *
120 : * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds)
121 : */
122 : unsigned int sysctl_sched_wakeup_granularity = 1000000UL;
123 : static unsigned int normalized_sysctl_sched_wakeup_granularity = 1000000UL;
124 :
125 : const_debug unsigned int sysctl_sched_migration_cost = 500000UL;
126 :
127 : int sched_thermal_decay_shift;
128 0 : static int __init setup_sched_thermal_decay_shift(char *str)
129 : {
130 0 : int _shift = 0;
131 :
132 0 : if (kstrtoint(str, 0, &_shift))
133 0 : pr_warn("Unable to set scheduler thermal pressure decay shift parameter\n");
134 :
135 0 : sched_thermal_decay_shift = clamp(_shift, 0, 10);
136 0 : return 1;
137 : }
138 : __setup("sched_thermal_decay_shift=", setup_sched_thermal_decay_shift);
139 :
140 : #ifdef CONFIG_SMP
141 : /*
142 : * For asym packing, by default the lower numbered CPU has higher priority.
143 : */
144 : int __weak arch_asym_cpu_priority(int cpu)
145 : {
146 : return -cpu;
147 : }
148 :
149 : /*
150 : * The margin used when comparing utilization with CPU capacity.
151 : *
152 : * (default: ~20%)
153 : */
154 : #define fits_capacity(cap, max) ((cap) * 1280 < (max) * 1024)
155 :
156 : /*
157 : * The margin used when comparing CPU capacities.
158 : * is 'cap1' noticeably greater than 'cap2'
159 : *
160 : * (default: ~5%)
161 : */
162 : #define capacity_greater(cap1, cap2) ((cap1) * 1024 > (cap2) * 1078)
163 : #endif
164 :
165 : #ifdef CONFIG_CFS_BANDWIDTH
166 : /*
167 : * Amount of runtime to allocate from global (tg) to local (per-cfs_rq) pool
168 : * each time a cfs_rq requests quota.
169 : *
170 : * Note: in the case that the slice exceeds the runtime remaining (either due
171 : * to consumption or the quota being specified to be smaller than the slice)
172 : * we will always only issue the remaining available time.
173 : *
174 : * (default: 5 msec, units: microseconds)
175 : */
176 : unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL;
177 : #endif
178 :
179 : static inline void update_load_add(struct load_weight *lw, unsigned long inc)
180 : {
181 727 : lw->weight += inc;
182 727 : lw->inv_weight = 0;
183 : }
184 :
185 : static inline void update_load_sub(struct load_weight *lw, unsigned long dec)
186 : {
187 618 : lw->weight -= dec;
188 618 : lw->inv_weight = 0;
189 : }
190 :
191 : static inline void update_load_set(struct load_weight *lw, unsigned long w)
192 : {
193 4 : lw->weight = w;
194 4 : lw->inv_weight = 0;
195 : }
196 :
197 : /*
198 : * Increase the granularity value when there are more CPUs,
199 : * because with more CPUs the 'effective latency' as visible
200 : * to users decreases. But the relationship is not linear,
201 : * so pick a second-best guess by going with the log2 of the
202 : * number of CPUs.
203 : *
204 : * This idea comes from the SD scheduler of Con Kolivas:
205 : */
206 : static unsigned int get_update_sysctl_factor(void)
207 : {
208 1 : unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8);
209 : unsigned int factor;
210 :
211 : switch (sysctl_sched_tunable_scaling) {
212 : case SCHED_TUNABLESCALING_NONE:
213 : factor = 1;
214 : break;
215 : case SCHED_TUNABLESCALING_LINEAR:
216 : factor = cpus;
217 : break;
218 : case SCHED_TUNABLESCALING_LOG:
219 : default:
220 : factor = 1 + ilog2(cpus);
221 : break;
222 : }
223 :
224 : return factor;
225 : }
226 :
227 : static void update_sysctl(void)
228 : {
229 1 : unsigned int factor = get_update_sysctl_factor();
230 :
231 : #define SET_SYSCTL(name) \
232 : (sysctl_##name = (factor) * normalized_sysctl_##name)
233 1 : SET_SYSCTL(sched_min_granularity);
234 1 : SET_SYSCTL(sched_latency);
235 1 : SET_SYSCTL(sched_wakeup_granularity);
236 : #undef SET_SYSCTL
237 : }
238 :
239 1 : void __init sched_init_granularity(void)
240 : {
241 : update_sysctl();
242 1 : }
243 :
244 : #define WMULT_CONST (~0U)
245 : #define WMULT_SHIFT 32
246 :
247 : static void __update_inv_weight(struct load_weight *lw)
248 : {
249 : unsigned long w;
250 :
251 110 : if (likely(lw->inv_weight))
252 : return;
253 :
254 110 : w = scale_load_down(lw->weight);
255 :
256 110 : if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST))
257 0 : lw->inv_weight = 1;
258 110 : else if (unlikely(!w))
259 0 : lw->inv_weight = WMULT_CONST;
260 : else
261 110 : lw->inv_weight = WMULT_CONST / w;
262 : }
263 :
264 : /*
265 : * delta_exec * weight / lw.weight
266 : * OR
267 : * (delta_exec * (weight * lw->inv_weight)) >> WMULT_SHIFT
268 : *
269 : * Either weight := NICE_0_LOAD and lw \e sched_prio_to_wmult[], in which case
270 : * we're guaranteed shift stays positive because inv_weight is guaranteed to
271 : * fit 32 bits, and NICE_0_LOAD gives another 10 bits; therefore shift >= 22.
272 : *
273 : * Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus
274 : * weight/lw.weight <= 1, and therefore our shift will also be positive.
275 : */
276 110 : static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw)
277 : {
278 110 : u64 fact = scale_load_down(weight);
279 110 : u32 fact_hi = (u32)(fact >> 32);
280 110 : int shift = WMULT_SHIFT;
281 : int fs;
282 :
283 110 : __update_inv_weight(lw);
284 :
285 110 : if (unlikely(fact_hi)) {
286 0 : fs = fls(fact_hi);
287 0 : shift -= fs;
288 0 : fact >>= fs;
289 : }
290 :
291 220 : fact = mul_u32_u32(fact, lw->inv_weight);
292 :
293 110 : fact_hi = (u32)(fact >> 32);
294 110 : if (fact_hi) {
295 0 : fs = fls(fact_hi);
296 0 : shift -= fs;
297 0 : fact >>= fs;
298 : }
299 :
300 220 : return mul_u64_u32_shr(delta_exec, fact, shift);
301 : }
302 :
303 :
304 : const struct sched_class fair_sched_class;
305 :
306 : /**************************************************************
307 : * CFS operations on generic schedulable entities:
308 : */
309 :
310 : #ifdef CONFIG_FAIR_GROUP_SCHED
311 :
312 : /* Walk up scheduling entities hierarchy */
313 : #define for_each_sched_entity(se) \
314 : for (; se; se = se->parent)
315 :
316 : static inline void cfs_rq_tg_path(struct cfs_rq *cfs_rq, char *path, int len)
317 : {
318 : if (!path)
319 : return;
320 :
321 : if (cfs_rq && task_group_is_autogroup(cfs_rq->tg))
322 : autogroup_path(cfs_rq->tg, path, len);
323 : else if (cfs_rq && cfs_rq->tg->css.cgroup)
324 : cgroup_path(cfs_rq->tg->css.cgroup, path, len);
325 : else
326 : strlcpy(path, "(null)", len);
327 : }
328 :
329 : static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
330 : {
331 : struct rq *rq = rq_of(cfs_rq);
332 : int cpu = cpu_of(rq);
333 :
334 : if (cfs_rq->on_list)
335 : return rq->tmp_alone_branch == &rq->leaf_cfs_rq_list;
336 :
337 : cfs_rq->on_list = 1;
338 :
339 : /*
340 : * Ensure we either appear before our parent (if already
341 : * enqueued) or force our parent to appear after us when it is
342 : * enqueued. The fact that we always enqueue bottom-up
343 : * reduces this to two cases and a special case for the root
344 : * cfs_rq. Furthermore, it also means that we will always reset
345 : * tmp_alone_branch either when the branch is connected
346 : * to a tree or when we reach the top of the tree
347 : */
348 : if (cfs_rq->tg->parent &&
349 : cfs_rq->tg->parent->cfs_rq[cpu]->on_list) {
350 : /*
351 : * If parent is already on the list, we add the child
352 : * just before. Thanks to circular linked property of
353 : * the list, this means to put the child at the tail
354 : * of the list that starts by parent.
355 : */
356 : list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list,
357 : &(cfs_rq->tg->parent->cfs_rq[cpu]->leaf_cfs_rq_list));
358 : /*
359 : * The branch is now connected to its tree so we can
360 : * reset tmp_alone_branch to the beginning of the
361 : * list.
362 : */
363 : rq->tmp_alone_branch = &rq->leaf_cfs_rq_list;
364 : return true;
365 : }
366 :
367 : if (!cfs_rq->tg->parent) {
368 : /*
369 : * cfs rq without parent should be put
370 : * at the tail of the list.
371 : */
372 : list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list,
373 : &rq->leaf_cfs_rq_list);
374 : /*
375 : * We have reach the top of a tree so we can reset
376 : * tmp_alone_branch to the beginning of the list.
377 : */
378 : rq->tmp_alone_branch = &rq->leaf_cfs_rq_list;
379 : return true;
380 : }
381 :
382 : /*
383 : * The parent has not already been added so we want to
384 : * make sure that it will be put after us.
385 : * tmp_alone_branch points to the begin of the branch
386 : * where we will add parent.
387 : */
388 : list_add_rcu(&cfs_rq->leaf_cfs_rq_list, rq->tmp_alone_branch);
389 : /*
390 : * update tmp_alone_branch to points to the new begin
391 : * of the branch
392 : */
393 : rq->tmp_alone_branch = &cfs_rq->leaf_cfs_rq_list;
394 : return false;
395 : }
396 :
397 : static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq)
398 : {
399 : if (cfs_rq->on_list) {
400 : struct rq *rq = rq_of(cfs_rq);
401 :
402 : /*
403 : * With cfs_rq being unthrottled/throttled during an enqueue,
404 : * it can happen the tmp_alone_branch points the a leaf that
405 : * we finally want to del. In this case, tmp_alone_branch moves
406 : * to the prev element but it will point to rq->leaf_cfs_rq_list
407 : * at the end of the enqueue.
408 : */
409 : if (rq->tmp_alone_branch == &cfs_rq->leaf_cfs_rq_list)
410 : rq->tmp_alone_branch = cfs_rq->leaf_cfs_rq_list.prev;
411 :
412 : list_del_rcu(&cfs_rq->leaf_cfs_rq_list);
413 : cfs_rq->on_list = 0;
414 : }
415 : }
416 :
417 : static inline void assert_list_leaf_cfs_rq(struct rq *rq)
418 : {
419 : SCHED_WARN_ON(rq->tmp_alone_branch != &rq->leaf_cfs_rq_list);
420 : }
421 :
422 : /* Iterate thr' all leaf cfs_rq's on a runqueue */
423 : #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \
424 : list_for_each_entry_safe(cfs_rq, pos, &rq->leaf_cfs_rq_list, \
425 : leaf_cfs_rq_list)
426 :
427 : /* Do the two (enqueued) entities belong to the same group ? */
428 : static inline struct cfs_rq *
429 : is_same_group(struct sched_entity *se, struct sched_entity *pse)
430 : {
431 : if (se->cfs_rq == pse->cfs_rq)
432 : return se->cfs_rq;
433 :
434 : return NULL;
435 : }
436 :
437 : static inline struct sched_entity *parent_entity(struct sched_entity *se)
438 : {
439 : return se->parent;
440 : }
441 :
442 : static void
443 : find_matching_se(struct sched_entity **se, struct sched_entity **pse)
444 : {
445 : int se_depth, pse_depth;
446 :
447 : /*
448 : * preemption test can be made between sibling entities who are in the
449 : * same cfs_rq i.e who have a common parent. Walk up the hierarchy of
450 : * both tasks until we find their ancestors who are siblings of common
451 : * parent.
452 : */
453 :
454 : /* First walk up until both entities are at same depth */
455 : se_depth = (*se)->depth;
456 : pse_depth = (*pse)->depth;
457 :
458 : while (se_depth > pse_depth) {
459 : se_depth--;
460 : *se = parent_entity(*se);
461 : }
462 :
463 : while (pse_depth > se_depth) {
464 : pse_depth--;
465 : *pse = parent_entity(*pse);
466 : }
467 :
468 : while (!is_same_group(*se, *pse)) {
469 : *se = parent_entity(*se);
470 : *pse = parent_entity(*pse);
471 : }
472 : }
473 :
474 : static int tg_is_idle(struct task_group *tg)
475 : {
476 : return tg->idle > 0;
477 : }
478 :
479 : static int cfs_rq_is_idle(struct cfs_rq *cfs_rq)
480 : {
481 : return cfs_rq->idle > 0;
482 : }
483 :
484 : static int se_is_idle(struct sched_entity *se)
485 : {
486 : if (entity_is_task(se))
487 : return task_has_idle_policy(task_of(se));
488 : return cfs_rq_is_idle(group_cfs_rq(se));
489 : }
490 :
491 : #else /* !CONFIG_FAIR_GROUP_SCHED */
492 :
493 : #define for_each_sched_entity(se) \
494 : for (; se; se = NULL)
495 :
496 : static inline void cfs_rq_tg_path(struct cfs_rq *cfs_rq, char *path, int len)
497 : {
498 0 : if (path)
499 0 : strlcpy(path, "(null)", len);
500 : }
501 :
502 : static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
503 : {
504 : return true;
505 : }
506 :
507 : static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq)
508 : {
509 : }
510 :
511 : static inline void assert_list_leaf_cfs_rq(struct rq *rq)
512 : {
513 : }
514 :
515 : #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \
516 : for (cfs_rq = &rq->cfs, pos = NULL; cfs_rq; cfs_rq = pos)
517 :
518 : static inline struct sched_entity *parent_entity(struct sched_entity *se)
519 : {
520 : return NULL;
521 : }
522 :
523 : static inline void
524 : find_matching_se(struct sched_entity **se, struct sched_entity **pse)
525 : {
526 : }
527 :
528 : static inline int tg_is_idle(struct task_group *tg)
529 : {
530 : return 0;
531 : }
532 :
533 : static int cfs_rq_is_idle(struct cfs_rq *cfs_rq)
534 : {
535 : return 0;
536 : }
537 :
538 : static int se_is_idle(struct sched_entity *se)
539 : {
540 : return 0;
541 : }
542 :
543 : #endif /* CONFIG_FAIR_GROUP_SCHED */
544 :
545 : static __always_inline
546 : void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec);
547 :
548 : /**************************************************************
549 : * Scheduling class tree data structure manipulation methods:
550 : */
551 :
552 : static inline u64 max_vruntime(u64 max_vruntime, u64 vruntime)
553 : {
554 1235 : s64 delta = (s64)(vruntime - max_vruntime);
555 1235 : if (delta > 0)
556 514 : max_vruntime = vruntime;
557 :
558 : return max_vruntime;
559 : }
560 :
561 : static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime)
562 : {
563 3 : s64 delta = (s64)(vruntime - min_vruntime);
564 3 : if (delta < 0)
565 3 : min_vruntime = vruntime;
566 :
567 : return min_vruntime;
568 : }
569 :
570 : static inline bool entity_before(struct sched_entity *a,
571 : struct sched_entity *b)
572 : {
573 218 : return (s64)(a->vruntime - b->vruntime) < 0;
574 : }
575 :
576 : #define __node_2_se(node) \
577 : rb_entry((node), struct sched_entity, run_node)
578 :
579 618 : static void update_min_vruntime(struct cfs_rq *cfs_rq)
580 : {
581 618 : struct sched_entity *curr = cfs_rq->curr;
582 618 : struct rb_node *leftmost = rb_first_cached(&cfs_rq->tasks_timeline);
583 :
584 618 : u64 vruntime = cfs_rq->min_vruntime;
585 :
586 618 : if (curr) {
587 618 : if (curr->on_rq)
588 3 : vruntime = curr->vruntime;
589 : else
590 : curr = NULL;
591 : }
592 :
593 618 : if (leftmost) { /* non-empty tree */
594 617 : struct sched_entity *se = __node_2_se(leftmost);
595 :
596 617 : if (!curr)
597 614 : vruntime = se->vruntime;
598 : else
599 3 : vruntime = min_vruntime(vruntime, se->vruntime);
600 : }
601 :
602 : /* ensure we never gain time by being placed backwards. */
603 1236 : cfs_rq->min_vruntime = max_vruntime(cfs_rq->min_vruntime, vruntime);
604 : #ifndef CONFIG_64BIT
605 : smp_wmb();
606 : cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime;
607 : #endif
608 618 : }
609 :
610 : static inline bool __entity_less(struct rb_node *a, const struct rb_node *b)
611 : {
612 218 : return entity_before(__node_2_se(a), __node_2_se(b));
613 : }
614 :
615 : /*
616 : * Enqueue an entity into the rb-tree:
617 : */
618 621 : static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
619 : {
620 1242 : rb_add_cached(&se->run_node, &cfs_rq->tasks_timeline, __entity_less);
621 621 : }
622 :
623 : static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
624 : {
625 620 : rb_erase_cached(&se->run_node, &cfs_rq->tasks_timeline);
626 : }
627 :
628 0 : struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq)
629 : {
630 617 : struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline);
631 :
632 617 : if (!left)
633 : return NULL;
634 :
635 617 : return __node_2_se(left);
636 : }
637 :
638 : static struct sched_entity *__pick_next_entity(struct sched_entity *se)
639 : {
640 0 : struct rb_node *next = rb_next(&se->run_node);
641 :
642 0 : if (!next)
643 : return NULL;
644 :
645 0 : return __node_2_se(next);
646 : }
647 :
648 : #ifdef CONFIG_SCHED_DEBUG
649 0 : struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq)
650 : {
651 0 : struct rb_node *last = rb_last(&cfs_rq->tasks_timeline.rb_root);
652 :
653 0 : if (!last)
654 : return NULL;
655 :
656 0 : return __node_2_se(last);
657 : }
658 :
659 : /**************************************************************
660 : * Scheduling class statistics methods:
661 : */
662 :
663 0 : int sched_update_scaling(void)
664 : {
665 0 : unsigned int factor = get_update_sysctl_factor();
666 :
667 0 : sched_nr_latency = DIV_ROUND_UP(sysctl_sched_latency,
668 : sysctl_sched_min_granularity);
669 :
670 : #define WRT_SYSCTL(name) \
671 : (normalized_sysctl_##name = sysctl_##name / (factor))
672 0 : WRT_SYSCTL(sched_min_granularity);
673 0 : WRT_SYSCTL(sched_latency);
674 0 : WRT_SYSCTL(sched_wakeup_granularity);
675 : #undef WRT_SYSCTL
676 :
677 0 : return 0;
678 : }
679 : #endif
680 :
681 : /*
682 : * delta /= w
683 : */
684 : static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se)
685 : {
686 313 : if (unlikely(se->load.weight != NICE_0_LOAD))
687 0 : delta = __calc_delta(delta, NICE_0_LOAD, &se->load);
688 :
689 : return delta;
690 : }
691 :
692 : /*
693 : * The idea is to set a period in which each task runs once.
694 : *
695 : * When there are too many tasks (sched_nr_latency) we have to stretch
696 : * this period because otherwise the slices get too small.
697 : *
698 : * p = (nr <= nl) ? l : l*nr/nl
699 : */
700 : static u64 __sched_period(unsigned long nr_running)
701 : {
702 110 : if (unlikely(nr_running > sched_nr_latency))
703 0 : return nr_running * sysctl_sched_min_granularity;
704 : else
705 110 : return sysctl_sched_latency;
706 : }
707 :
708 : static bool sched_idle_cfs_rq(struct cfs_rq *cfs_rq);
709 :
710 : /*
711 : * We calculate the wall-time slice from the period by taking a part
712 : * proportional to the weight.
713 : *
714 : * s = p*P[w/rw]
715 : */
716 110 : static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se)
717 : {
718 110 : unsigned int nr_running = cfs_rq->nr_running;
719 110 : struct sched_entity *init_se = se;
720 : unsigned int min_gran;
721 : u64 slice;
722 :
723 110 : if (sched_feat(ALT_PERIOD))
724 110 : nr_running = rq_of(cfs_rq)->cfs.h_nr_running;
725 :
726 110 : slice = __sched_period(nr_running + !se->on_rq);
727 :
728 110 : for_each_sched_entity(se) {
729 : struct load_weight *load;
730 : struct load_weight lw;
731 : struct cfs_rq *qcfs_rq;
732 :
733 220 : qcfs_rq = cfs_rq_of(se);
734 110 : load = &qcfs_rq->load;
735 :
736 110 : if (unlikely(!se->on_rq)) {
737 107 : lw = qcfs_rq->load;
738 :
739 214 : update_load_add(&lw, se->load.weight);
740 107 : load = &lw;
741 : }
742 110 : slice = __calc_delta(slice, se->load.weight, load);
743 : }
744 :
745 110 : if (sched_feat(BASE_SLICE)) {
746 110 : if (se_is_idle(init_se) && !sched_idle_cfs_rq(cfs_rq))
747 : min_gran = sysctl_sched_idle_min_granularity;
748 : else
749 110 : min_gran = sysctl_sched_min_granularity;
750 :
751 110 : slice = max_t(u64, slice, min_gran);
752 : }
753 :
754 110 : return slice;
755 : }
756 :
757 : /*
758 : * We calculate the vruntime slice of a to-be-inserted task.
759 : *
760 : * vs = s/w
761 : */
762 107 : static u64 sched_vslice(struct cfs_rq *cfs_rq, struct sched_entity *se)
763 : {
764 214 : return calc_delta_fair(sched_slice(cfs_rq, se), se);
765 : }
766 :
767 : #include "pelt.h"
768 : #ifdef CONFIG_SMP
769 :
770 : static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu);
771 : static unsigned long task_h_load(struct task_struct *p);
772 : static unsigned long capacity_of(int cpu);
773 :
774 : /* Give new sched_entity start runnable values to heavy its load in infant time */
775 : void init_entity_runnable_average(struct sched_entity *se)
776 : {
777 : struct sched_avg *sa = &se->avg;
778 :
779 : memset(sa, 0, sizeof(*sa));
780 :
781 : /*
782 : * Tasks are initialized with full load to be seen as heavy tasks until
783 : * they get a chance to stabilize to their real load level.
784 : * Group entities are initialized with zero load to reflect the fact that
785 : * nothing has been attached to the task group yet.
786 : */
787 : if (entity_is_task(se))
788 : sa->load_avg = scale_load_down(se->load.weight);
789 :
790 : /* when this task enqueue'ed, it will contribute to its cfs_rq's load_avg */
791 : }
792 :
793 : static void attach_entity_cfs_rq(struct sched_entity *se);
794 :
795 : /*
796 : * With new tasks being created, their initial util_avgs are extrapolated
797 : * based on the cfs_rq's current util_avg:
798 : *
799 : * util_avg = cfs_rq->util_avg / (cfs_rq->load_avg + 1) * se.load.weight
800 : *
801 : * However, in many cases, the above util_avg does not give a desired
802 : * value. Moreover, the sum of the util_avgs may be divergent, such
803 : * as when the series is a harmonic series.
804 : *
805 : * To solve this problem, we also cap the util_avg of successive tasks to
806 : * only 1/2 of the left utilization budget:
807 : *
808 : * util_avg_cap = (cpu_scale - cfs_rq->avg.util_avg) / 2^n
809 : *
810 : * where n denotes the nth task and cpu_scale the CPU capacity.
811 : *
812 : * For example, for a CPU with 1024 of capacity, a simplest series from
813 : * the beginning would be like:
814 : *
815 : * task util_avg: 512, 256, 128, 64, 32, 16, 8, ...
816 : * cfs_rq util_avg: 512, 768, 896, 960, 992, 1008, 1016, ...
817 : *
818 : * Finally, that extrapolated util_avg is clamped to the cap (util_avg_cap)
819 : * if util_avg > util_avg_cap.
820 : */
821 : void post_init_entity_util_avg(struct task_struct *p)
822 : {
823 : struct sched_entity *se = &p->se;
824 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
825 : struct sched_avg *sa = &se->avg;
826 : long cpu_scale = arch_scale_cpu_capacity(cpu_of(rq_of(cfs_rq)));
827 : long cap = (long)(cpu_scale - cfs_rq->avg.util_avg) / 2;
828 :
829 : if (cap > 0) {
830 : if (cfs_rq->avg.util_avg != 0) {
831 : sa->util_avg = cfs_rq->avg.util_avg * se->load.weight;
832 : sa->util_avg /= (cfs_rq->avg.load_avg + 1);
833 :
834 : if (sa->util_avg > cap)
835 : sa->util_avg = cap;
836 : } else {
837 : sa->util_avg = cap;
838 : }
839 : }
840 :
841 : sa->runnable_avg = sa->util_avg;
842 :
843 : if (p->sched_class != &fair_sched_class) {
844 : /*
845 : * For !fair tasks do:
846 : *
847 : update_cfs_rq_load_avg(now, cfs_rq);
848 : attach_entity_load_avg(cfs_rq, se);
849 : switched_from_fair(rq, p);
850 : *
851 : * such that the next switched_to_fair() has the
852 : * expected state.
853 : */
854 : se->avg.last_update_time = cfs_rq_clock_pelt(cfs_rq);
855 : return;
856 : }
857 :
858 : attach_entity_cfs_rq(se);
859 : }
860 :
861 : #else /* !CONFIG_SMP */
862 107 : void init_entity_runnable_average(struct sched_entity *se)
863 : {
864 107 : }
865 107 : void post_init_entity_util_avg(struct task_struct *p)
866 : {
867 107 : }
868 : static void update_tg_load_avg(struct cfs_rq *cfs_rq)
869 : {
870 : }
871 : #endif /* CONFIG_SMP */
872 :
873 : /*
874 : * Update the current task's runtime statistics.
875 : */
876 1863 : static void update_curr(struct cfs_rq *cfs_rq)
877 : {
878 1863 : struct sched_entity *curr = cfs_rq->curr;
879 3726 : u64 now = rq_clock_task(rq_of(cfs_rq));
880 : u64 delta_exec;
881 :
882 1863 : if (unlikely(!curr))
883 : return;
884 :
885 1857 : delta_exec = now - curr->exec_start;
886 1857 : if (unlikely((s64)delta_exec <= 0))
887 : return;
888 :
889 3 : curr->exec_start = now;
890 :
891 : if (schedstat_enabled()) {
892 : struct sched_statistics *stats;
893 :
894 : stats = __schedstats_from_se(curr);
895 : __schedstat_set(stats->exec_max,
896 : max(delta_exec, stats->exec_max));
897 : }
898 :
899 3 : curr->sum_exec_runtime += delta_exec;
900 : schedstat_add(cfs_rq->exec_clock, delta_exec);
901 :
902 3 : curr->vruntime += calc_delta_fair(delta_exec, curr);
903 3 : update_min_vruntime(cfs_rq);
904 :
905 : if (entity_is_task(curr)) {
906 3 : struct task_struct *curtask = task_of(curr);
907 :
908 3 : trace_sched_stat_runtime(curtask, delta_exec, curr->vruntime);
909 3 : cgroup_account_cputime(curtask, delta_exec);
910 : account_group_exec_runtime(curtask, delta_exec);
911 : }
912 :
913 : account_cfs_rq_runtime(cfs_rq, delta_exec);
914 : }
915 :
916 0 : static void update_curr_fair(struct rq *rq)
917 : {
918 0 : update_curr(cfs_rq_of(&rq->curr->se));
919 0 : }
920 :
921 : static inline void
922 : update_stats_wait_start_fair(struct cfs_rq *cfs_rq, struct sched_entity *se)
923 : {
924 : struct sched_statistics *stats;
925 1 : struct task_struct *p = NULL;
926 :
927 : if (!schedstat_enabled())
928 : return;
929 :
930 : stats = __schedstats_from_se(se);
931 :
932 : if (entity_is_task(se))
933 : p = task_of(se);
934 :
935 : __update_stats_wait_start(rq_of(cfs_rq), p, stats);
936 : }
937 :
938 : static inline void
939 : update_stats_wait_end_fair(struct cfs_rq *cfs_rq, struct sched_entity *se)
940 : {
941 : struct sched_statistics *stats;
942 620 : struct task_struct *p = NULL;
943 :
944 : if (!schedstat_enabled())
945 : return;
946 :
947 : stats = __schedstats_from_se(se);
948 :
949 : /*
950 : * When the sched_schedstat changes from 0 to 1, some sched se
951 : * maybe already in the runqueue, the se->statistics.wait_start
952 : * will be 0.So it will let the delta wrong. We need to avoid this
953 : * scenario.
954 : */
955 : if (unlikely(!schedstat_val(stats->wait_start)))
956 : return;
957 :
958 : if (entity_is_task(se))
959 : p = task_of(se);
960 :
961 : __update_stats_wait_end(rq_of(cfs_rq), p, stats);
962 : }
963 :
964 : static inline void
965 : update_stats_enqueue_sleeper_fair(struct cfs_rq *cfs_rq, struct sched_entity *se)
966 : {
967 : struct sched_statistics *stats;
968 : struct task_struct *tsk = NULL;
969 :
970 : if (!schedstat_enabled())
971 : return;
972 :
973 : stats = __schedstats_from_se(se);
974 :
975 : if (entity_is_task(se))
976 : tsk = task_of(se);
977 :
978 : __update_stats_enqueue_sleeper(rq_of(cfs_rq), tsk, stats);
979 : }
980 :
981 : /*
982 : * Task is being enqueued - update stats:
983 : */
984 : static inline void
985 : update_stats_enqueue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
986 : {
987 : if (!schedstat_enabled())
988 : return;
989 :
990 : /*
991 : * Are we enqueueing a waiting task? (for current tasks
992 : * a dequeue/enqueue event is a NOP)
993 : */
994 : if (se != cfs_rq->curr)
995 : update_stats_wait_start_fair(cfs_rq, se);
996 :
997 : if (flags & ENQUEUE_WAKEUP)
998 : update_stats_enqueue_sleeper_fair(cfs_rq, se);
999 : }
1000 :
1001 : static inline void
1002 : update_stats_dequeue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
1003 : {
1004 :
1005 : if (!schedstat_enabled())
1006 : return;
1007 :
1008 : /*
1009 : * Mark the end of the wait period if dequeueing a
1010 : * waiting task:
1011 : */
1012 : if (se != cfs_rq->curr)
1013 : update_stats_wait_end_fair(cfs_rq, se);
1014 :
1015 : if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) {
1016 : struct task_struct *tsk = task_of(se);
1017 : unsigned int state;
1018 :
1019 : /* XXX racy against TTWU */
1020 : state = READ_ONCE(tsk->__state);
1021 : if (state & TASK_INTERRUPTIBLE)
1022 : __schedstat_set(tsk->stats.sleep_start,
1023 : rq_clock(rq_of(cfs_rq)));
1024 : if (state & TASK_UNINTERRUPTIBLE)
1025 : __schedstat_set(tsk->stats.block_start,
1026 : rq_clock(rq_of(cfs_rq)));
1027 : }
1028 : }
1029 :
1030 : /*
1031 : * We are picking a new current task - update its stats:
1032 : */
1033 : static inline void
1034 : update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
1035 : {
1036 : /*
1037 : * We are starting a new run period:
1038 : */
1039 1240 : se->exec_start = rq_clock_task(rq_of(cfs_rq));
1040 : }
1041 :
1042 : /**************************************************
1043 : * Scheduling class queueing methods:
1044 : */
1045 :
1046 : #ifdef CONFIG_NUMA_BALANCING
1047 : /*
1048 : * Approximate time to scan a full NUMA task in ms. The task scan period is
1049 : * calculated based on the tasks virtual memory size and
1050 : * numa_balancing_scan_size.
1051 : */
1052 : unsigned int sysctl_numa_balancing_scan_period_min = 1000;
1053 : unsigned int sysctl_numa_balancing_scan_period_max = 60000;
1054 :
1055 : /* Portion of address space to scan in MB */
1056 : unsigned int sysctl_numa_balancing_scan_size = 256;
1057 :
1058 : /* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */
1059 : unsigned int sysctl_numa_balancing_scan_delay = 1000;
1060 :
1061 : struct numa_group {
1062 : refcount_t refcount;
1063 :
1064 : spinlock_t lock; /* nr_tasks, tasks */
1065 : int nr_tasks;
1066 : pid_t gid;
1067 : int active_nodes;
1068 :
1069 : struct rcu_head rcu;
1070 : unsigned long total_faults;
1071 : unsigned long max_faults_cpu;
1072 : /*
1073 : * faults[] array is split into two regions: faults_mem and faults_cpu.
1074 : *
1075 : * Faults_cpu is used to decide whether memory should move
1076 : * towards the CPU. As a consequence, these stats are weighted
1077 : * more by CPU use than by memory faults.
1078 : */
1079 : unsigned long faults[];
1080 : };
1081 :
1082 : /*
1083 : * For functions that can be called in multiple contexts that permit reading
1084 : * ->numa_group (see struct task_struct for locking rules).
1085 : */
1086 : static struct numa_group *deref_task_numa_group(struct task_struct *p)
1087 : {
1088 : return rcu_dereference_check(p->numa_group, p == current ||
1089 : (lockdep_is_held(__rq_lockp(task_rq(p))) && !READ_ONCE(p->on_cpu)));
1090 : }
1091 :
1092 : static struct numa_group *deref_curr_numa_group(struct task_struct *p)
1093 : {
1094 : return rcu_dereference_protected(p->numa_group, p == current);
1095 : }
1096 :
1097 : static inline unsigned long group_faults_priv(struct numa_group *ng);
1098 : static inline unsigned long group_faults_shared(struct numa_group *ng);
1099 :
1100 : static unsigned int task_nr_scan_windows(struct task_struct *p)
1101 : {
1102 : unsigned long rss = 0;
1103 : unsigned long nr_scan_pages;
1104 :
1105 : /*
1106 : * Calculations based on RSS as non-present and empty pages are skipped
1107 : * by the PTE scanner and NUMA hinting faults should be trapped based
1108 : * on resident pages
1109 : */
1110 : nr_scan_pages = sysctl_numa_balancing_scan_size << (20 - PAGE_SHIFT);
1111 : rss = get_mm_rss(p->mm);
1112 : if (!rss)
1113 : rss = nr_scan_pages;
1114 :
1115 : rss = round_up(rss, nr_scan_pages);
1116 : return rss / nr_scan_pages;
1117 : }
1118 :
1119 : /* For sanity's sake, never scan more PTEs than MAX_SCAN_WINDOW MB/sec. */
1120 : #define MAX_SCAN_WINDOW 2560
1121 :
1122 : static unsigned int task_scan_min(struct task_struct *p)
1123 : {
1124 : unsigned int scan_size = READ_ONCE(sysctl_numa_balancing_scan_size);
1125 : unsigned int scan, floor;
1126 : unsigned int windows = 1;
1127 :
1128 : if (scan_size < MAX_SCAN_WINDOW)
1129 : windows = MAX_SCAN_WINDOW / scan_size;
1130 : floor = 1000 / windows;
1131 :
1132 : scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p);
1133 : return max_t(unsigned int, floor, scan);
1134 : }
1135 :
1136 : static unsigned int task_scan_start(struct task_struct *p)
1137 : {
1138 : unsigned long smin = task_scan_min(p);
1139 : unsigned long period = smin;
1140 : struct numa_group *ng;
1141 :
1142 : /* Scale the maximum scan period with the amount of shared memory. */
1143 : rcu_read_lock();
1144 : ng = rcu_dereference(p->numa_group);
1145 : if (ng) {
1146 : unsigned long shared = group_faults_shared(ng);
1147 : unsigned long private = group_faults_priv(ng);
1148 :
1149 : period *= refcount_read(&ng->refcount);
1150 : period *= shared + 1;
1151 : period /= private + shared + 1;
1152 : }
1153 : rcu_read_unlock();
1154 :
1155 : return max(smin, period);
1156 : }
1157 :
1158 : static unsigned int task_scan_max(struct task_struct *p)
1159 : {
1160 : unsigned long smin = task_scan_min(p);
1161 : unsigned long smax;
1162 : struct numa_group *ng;
1163 :
1164 : /* Watch for min being lower than max due to floor calculations */
1165 : smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p);
1166 :
1167 : /* Scale the maximum scan period with the amount of shared memory. */
1168 : ng = deref_curr_numa_group(p);
1169 : if (ng) {
1170 : unsigned long shared = group_faults_shared(ng);
1171 : unsigned long private = group_faults_priv(ng);
1172 : unsigned long period = smax;
1173 :
1174 : period *= refcount_read(&ng->refcount);
1175 : period *= shared + 1;
1176 : period /= private + shared + 1;
1177 :
1178 : smax = max(smax, period);
1179 : }
1180 :
1181 : return max(smin, smax);
1182 : }
1183 :
1184 : static void account_numa_enqueue(struct rq *rq, struct task_struct *p)
1185 : {
1186 : rq->nr_numa_running += (p->numa_preferred_nid != NUMA_NO_NODE);
1187 : rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p));
1188 : }
1189 :
1190 : static void account_numa_dequeue(struct rq *rq, struct task_struct *p)
1191 : {
1192 : rq->nr_numa_running -= (p->numa_preferred_nid != NUMA_NO_NODE);
1193 : rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p));
1194 : }
1195 :
1196 : /* Shared or private faults. */
1197 : #define NR_NUMA_HINT_FAULT_TYPES 2
1198 :
1199 : /* Memory and CPU locality */
1200 : #define NR_NUMA_HINT_FAULT_STATS (NR_NUMA_HINT_FAULT_TYPES * 2)
1201 :
1202 : /* Averaged statistics, and temporary buffers. */
1203 : #define NR_NUMA_HINT_FAULT_BUCKETS (NR_NUMA_HINT_FAULT_STATS * 2)
1204 :
1205 : pid_t task_numa_group_id(struct task_struct *p)
1206 : {
1207 : struct numa_group *ng;
1208 : pid_t gid = 0;
1209 :
1210 : rcu_read_lock();
1211 : ng = rcu_dereference(p->numa_group);
1212 : if (ng)
1213 : gid = ng->gid;
1214 : rcu_read_unlock();
1215 :
1216 : return gid;
1217 : }
1218 :
1219 : /*
1220 : * The averaged statistics, shared & private, memory & CPU,
1221 : * occupy the first half of the array. The second half of the
1222 : * array is for current counters, which are averaged into the
1223 : * first set by task_numa_placement.
1224 : */
1225 : static inline int task_faults_idx(enum numa_faults_stats s, int nid, int priv)
1226 : {
1227 : return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv;
1228 : }
1229 :
1230 : static inline unsigned long task_faults(struct task_struct *p, int nid)
1231 : {
1232 : if (!p->numa_faults)
1233 : return 0;
1234 :
1235 : return p->numa_faults[task_faults_idx(NUMA_MEM, nid, 0)] +
1236 : p->numa_faults[task_faults_idx(NUMA_MEM, nid, 1)];
1237 : }
1238 :
1239 : static inline unsigned long group_faults(struct task_struct *p, int nid)
1240 : {
1241 : struct numa_group *ng = deref_task_numa_group(p);
1242 :
1243 : if (!ng)
1244 : return 0;
1245 :
1246 : return ng->faults[task_faults_idx(NUMA_MEM, nid, 0)] +
1247 : ng->faults[task_faults_idx(NUMA_MEM, nid, 1)];
1248 : }
1249 :
1250 : static inline unsigned long group_faults_cpu(struct numa_group *group, int nid)
1251 : {
1252 : return group->faults[task_faults_idx(NUMA_CPU, nid, 0)] +
1253 : group->faults[task_faults_idx(NUMA_CPU, nid, 1)];
1254 : }
1255 :
1256 : static inline unsigned long group_faults_priv(struct numa_group *ng)
1257 : {
1258 : unsigned long faults = 0;
1259 : int node;
1260 :
1261 : for_each_online_node(node) {
1262 : faults += ng->faults[task_faults_idx(NUMA_MEM, node, 1)];
1263 : }
1264 :
1265 : return faults;
1266 : }
1267 :
1268 : static inline unsigned long group_faults_shared(struct numa_group *ng)
1269 : {
1270 : unsigned long faults = 0;
1271 : int node;
1272 :
1273 : for_each_online_node(node) {
1274 : faults += ng->faults[task_faults_idx(NUMA_MEM, node, 0)];
1275 : }
1276 :
1277 : return faults;
1278 : }
1279 :
1280 : /*
1281 : * A node triggering more than 1/3 as many NUMA faults as the maximum is
1282 : * considered part of a numa group's pseudo-interleaving set. Migrations
1283 : * between these nodes are slowed down, to allow things to settle down.
1284 : */
1285 : #define ACTIVE_NODE_FRACTION 3
1286 :
1287 : static bool numa_is_active_node(int nid, struct numa_group *ng)
1288 : {
1289 : return group_faults_cpu(ng, nid) * ACTIVE_NODE_FRACTION > ng->max_faults_cpu;
1290 : }
1291 :
1292 : /* Handle placement on systems where not all nodes are directly connected. */
1293 : static unsigned long score_nearby_nodes(struct task_struct *p, int nid,
1294 : int lim_dist, bool task)
1295 : {
1296 : unsigned long score = 0;
1297 : int node, max_dist;
1298 :
1299 : /*
1300 : * All nodes are directly connected, and the same distance
1301 : * from each other. No need for fancy placement algorithms.
1302 : */
1303 : if (sched_numa_topology_type == NUMA_DIRECT)
1304 : return 0;
1305 :
1306 : /* sched_max_numa_distance may be changed in parallel. */
1307 : max_dist = READ_ONCE(sched_max_numa_distance);
1308 : /*
1309 : * This code is called for each node, introducing N^2 complexity,
1310 : * which should be ok given the number of nodes rarely exceeds 8.
1311 : */
1312 : for_each_online_node(node) {
1313 : unsigned long faults;
1314 : int dist = node_distance(nid, node);
1315 :
1316 : /*
1317 : * The furthest away nodes in the system are not interesting
1318 : * for placement; nid was already counted.
1319 : */
1320 : if (dist >= max_dist || node == nid)
1321 : continue;
1322 :
1323 : /*
1324 : * On systems with a backplane NUMA topology, compare groups
1325 : * of nodes, and move tasks towards the group with the most
1326 : * memory accesses. When comparing two nodes at distance
1327 : * "hoplimit", only nodes closer by than "hoplimit" are part
1328 : * of each group. Skip other nodes.
1329 : */
1330 : if (sched_numa_topology_type == NUMA_BACKPLANE && dist >= lim_dist)
1331 : continue;
1332 :
1333 : /* Add up the faults from nearby nodes. */
1334 : if (task)
1335 : faults = task_faults(p, node);
1336 : else
1337 : faults = group_faults(p, node);
1338 :
1339 : /*
1340 : * On systems with a glueless mesh NUMA topology, there are
1341 : * no fixed "groups of nodes". Instead, nodes that are not
1342 : * directly connected bounce traffic through intermediate
1343 : * nodes; a numa_group can occupy any set of nodes.
1344 : * The further away a node is, the less the faults count.
1345 : * This seems to result in good task placement.
1346 : */
1347 : if (sched_numa_topology_type == NUMA_GLUELESS_MESH) {
1348 : faults *= (max_dist - dist);
1349 : faults /= (max_dist - LOCAL_DISTANCE);
1350 : }
1351 :
1352 : score += faults;
1353 : }
1354 :
1355 : return score;
1356 : }
1357 :
1358 : /*
1359 : * These return the fraction of accesses done by a particular task, or
1360 : * task group, on a particular numa node. The group weight is given a
1361 : * larger multiplier, in order to group tasks together that are almost
1362 : * evenly spread out between numa nodes.
1363 : */
1364 : static inline unsigned long task_weight(struct task_struct *p, int nid,
1365 : int dist)
1366 : {
1367 : unsigned long faults, total_faults;
1368 :
1369 : if (!p->numa_faults)
1370 : return 0;
1371 :
1372 : total_faults = p->total_numa_faults;
1373 :
1374 : if (!total_faults)
1375 : return 0;
1376 :
1377 : faults = task_faults(p, nid);
1378 : faults += score_nearby_nodes(p, nid, dist, true);
1379 :
1380 : return 1000 * faults / total_faults;
1381 : }
1382 :
1383 : static inline unsigned long group_weight(struct task_struct *p, int nid,
1384 : int dist)
1385 : {
1386 : struct numa_group *ng = deref_task_numa_group(p);
1387 : unsigned long faults, total_faults;
1388 :
1389 : if (!ng)
1390 : return 0;
1391 :
1392 : total_faults = ng->total_faults;
1393 :
1394 : if (!total_faults)
1395 : return 0;
1396 :
1397 : faults = group_faults(p, nid);
1398 : faults += score_nearby_nodes(p, nid, dist, false);
1399 :
1400 : return 1000 * faults / total_faults;
1401 : }
1402 :
1403 : bool should_numa_migrate_memory(struct task_struct *p, struct page * page,
1404 : int src_nid, int dst_cpu)
1405 : {
1406 : struct numa_group *ng = deref_curr_numa_group(p);
1407 : int dst_nid = cpu_to_node(dst_cpu);
1408 : int last_cpupid, this_cpupid;
1409 :
1410 : this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid);
1411 : last_cpupid = page_cpupid_xchg_last(page, this_cpupid);
1412 :
1413 : /*
1414 : * Allow first faults or private faults to migrate immediately early in
1415 : * the lifetime of a task. The magic number 4 is based on waiting for
1416 : * two full passes of the "multi-stage node selection" test that is
1417 : * executed below.
1418 : */
1419 : if ((p->numa_preferred_nid == NUMA_NO_NODE || p->numa_scan_seq <= 4) &&
1420 : (cpupid_pid_unset(last_cpupid) || cpupid_match_pid(p, last_cpupid)))
1421 : return true;
1422 :
1423 : /*
1424 : * Multi-stage node selection is used in conjunction with a periodic
1425 : * migration fault to build a temporal task<->page relation. By using
1426 : * a two-stage filter we remove short/unlikely relations.
1427 : *
1428 : * Using P(p) ~ n_p / n_t as per frequentist probability, we can equate
1429 : * a task's usage of a particular page (n_p) per total usage of this
1430 : * page (n_t) (in a given time-span) to a probability.
1431 : *
1432 : * Our periodic faults will sample this probability and getting the
1433 : * same result twice in a row, given these samples are fully
1434 : * independent, is then given by P(n)^2, provided our sample period
1435 : * is sufficiently short compared to the usage pattern.
1436 : *
1437 : * This quadric squishes small probabilities, making it less likely we
1438 : * act on an unlikely task<->page relation.
1439 : */
1440 : if (!cpupid_pid_unset(last_cpupid) &&
1441 : cpupid_to_nid(last_cpupid) != dst_nid)
1442 : return false;
1443 :
1444 : /* Always allow migrate on private faults */
1445 : if (cpupid_match_pid(p, last_cpupid))
1446 : return true;
1447 :
1448 : /* A shared fault, but p->numa_group has not been set up yet. */
1449 : if (!ng)
1450 : return true;
1451 :
1452 : /*
1453 : * Destination node is much more heavily used than the source
1454 : * node? Allow migration.
1455 : */
1456 : if (group_faults_cpu(ng, dst_nid) > group_faults_cpu(ng, src_nid) *
1457 : ACTIVE_NODE_FRACTION)
1458 : return true;
1459 :
1460 : /*
1461 : * Distribute memory according to CPU & memory use on each node,
1462 : * with 3/4 hysteresis to avoid unnecessary memory migrations:
1463 : *
1464 : * faults_cpu(dst) 3 faults_cpu(src)
1465 : * --------------- * - > ---------------
1466 : * faults_mem(dst) 4 faults_mem(src)
1467 : */
1468 : return group_faults_cpu(ng, dst_nid) * group_faults(p, src_nid) * 3 >
1469 : group_faults_cpu(ng, src_nid) * group_faults(p, dst_nid) * 4;
1470 : }
1471 :
1472 : /*
1473 : * 'numa_type' describes the node at the moment of load balancing.
1474 : */
1475 : enum numa_type {
1476 : /* The node has spare capacity that can be used to run more tasks. */
1477 : node_has_spare = 0,
1478 : /*
1479 : * The node is fully used and the tasks don't compete for more CPU
1480 : * cycles. Nevertheless, some tasks might wait before running.
1481 : */
1482 : node_fully_busy,
1483 : /*
1484 : * The node is overloaded and can't provide expected CPU cycles to all
1485 : * tasks.
1486 : */
1487 : node_overloaded
1488 : };
1489 :
1490 : /* Cached statistics for all CPUs within a node */
1491 : struct numa_stats {
1492 : unsigned long load;
1493 : unsigned long runnable;
1494 : unsigned long util;
1495 : /* Total compute capacity of CPUs on a node */
1496 : unsigned long compute_capacity;
1497 : unsigned int nr_running;
1498 : unsigned int weight;
1499 : enum numa_type node_type;
1500 : int idle_cpu;
1501 : };
1502 :
1503 : static inline bool is_core_idle(int cpu)
1504 : {
1505 : #ifdef CONFIG_SCHED_SMT
1506 : int sibling;
1507 :
1508 : for_each_cpu(sibling, cpu_smt_mask(cpu)) {
1509 : if (cpu == sibling)
1510 : continue;
1511 :
1512 : if (!idle_cpu(sibling))
1513 : return false;
1514 : }
1515 : #endif
1516 :
1517 : return true;
1518 : }
1519 :
1520 : struct task_numa_env {
1521 : struct task_struct *p;
1522 :
1523 : int src_cpu, src_nid;
1524 : int dst_cpu, dst_nid;
1525 : int imb_numa_nr;
1526 :
1527 : struct numa_stats src_stats, dst_stats;
1528 :
1529 : int imbalance_pct;
1530 : int dist;
1531 :
1532 : struct task_struct *best_task;
1533 : long best_imp;
1534 : int best_cpu;
1535 : };
1536 :
1537 : static unsigned long cpu_load(struct rq *rq);
1538 : static unsigned long cpu_runnable(struct rq *rq);
1539 : static inline long adjust_numa_imbalance(int imbalance,
1540 : int dst_running, int imb_numa_nr);
1541 :
1542 : static inline enum
1543 : numa_type numa_classify(unsigned int imbalance_pct,
1544 : struct numa_stats *ns)
1545 : {
1546 : if ((ns->nr_running > ns->weight) &&
1547 : (((ns->compute_capacity * 100) < (ns->util * imbalance_pct)) ||
1548 : ((ns->compute_capacity * imbalance_pct) < (ns->runnable * 100))))
1549 : return node_overloaded;
1550 :
1551 : if ((ns->nr_running < ns->weight) ||
1552 : (((ns->compute_capacity * 100) > (ns->util * imbalance_pct)) &&
1553 : ((ns->compute_capacity * imbalance_pct) > (ns->runnable * 100))))
1554 : return node_has_spare;
1555 :
1556 : return node_fully_busy;
1557 : }
1558 :
1559 : #ifdef CONFIG_SCHED_SMT
1560 : /* Forward declarations of select_idle_sibling helpers */
1561 : static inline bool test_idle_cores(int cpu, bool def);
1562 : static inline int numa_idle_core(int idle_core, int cpu)
1563 : {
1564 : if (!static_branch_likely(&sched_smt_present) ||
1565 : idle_core >= 0 || !test_idle_cores(cpu, false))
1566 : return idle_core;
1567 :
1568 : /*
1569 : * Prefer cores instead of packing HT siblings
1570 : * and triggering future load balancing.
1571 : */
1572 : if (is_core_idle(cpu))
1573 : idle_core = cpu;
1574 :
1575 : return idle_core;
1576 : }
1577 : #else
1578 : static inline int numa_idle_core(int idle_core, int cpu)
1579 : {
1580 : return idle_core;
1581 : }
1582 : #endif
1583 :
1584 : /*
1585 : * Gather all necessary information to make NUMA balancing placement
1586 : * decisions that are compatible with standard load balancer. This
1587 : * borrows code and logic from update_sg_lb_stats but sharing a
1588 : * common implementation is impractical.
1589 : */
1590 : static void update_numa_stats(struct task_numa_env *env,
1591 : struct numa_stats *ns, int nid,
1592 : bool find_idle)
1593 : {
1594 : int cpu, idle_core = -1;
1595 :
1596 : memset(ns, 0, sizeof(*ns));
1597 : ns->idle_cpu = -1;
1598 :
1599 : rcu_read_lock();
1600 : for_each_cpu(cpu, cpumask_of_node(nid)) {
1601 : struct rq *rq = cpu_rq(cpu);
1602 :
1603 : ns->load += cpu_load(rq);
1604 : ns->runnable += cpu_runnable(rq);
1605 : ns->util += cpu_util_cfs(cpu);
1606 : ns->nr_running += rq->cfs.h_nr_running;
1607 : ns->compute_capacity += capacity_of(cpu);
1608 :
1609 : if (find_idle && !rq->nr_running && idle_cpu(cpu)) {
1610 : if (READ_ONCE(rq->numa_migrate_on) ||
1611 : !cpumask_test_cpu(cpu, env->p->cpus_ptr))
1612 : continue;
1613 :
1614 : if (ns->idle_cpu == -1)
1615 : ns->idle_cpu = cpu;
1616 :
1617 : idle_core = numa_idle_core(idle_core, cpu);
1618 : }
1619 : }
1620 : rcu_read_unlock();
1621 :
1622 : ns->weight = cpumask_weight(cpumask_of_node(nid));
1623 :
1624 : ns->node_type = numa_classify(env->imbalance_pct, ns);
1625 :
1626 : if (idle_core >= 0)
1627 : ns->idle_cpu = idle_core;
1628 : }
1629 :
1630 : static void task_numa_assign(struct task_numa_env *env,
1631 : struct task_struct *p, long imp)
1632 : {
1633 : struct rq *rq = cpu_rq(env->dst_cpu);
1634 :
1635 : /* Check if run-queue part of active NUMA balance. */
1636 : if (env->best_cpu != env->dst_cpu && xchg(&rq->numa_migrate_on, 1)) {
1637 : int cpu;
1638 : int start = env->dst_cpu;
1639 :
1640 : /* Find alternative idle CPU. */
1641 : for_each_cpu_wrap(cpu, cpumask_of_node(env->dst_nid), start) {
1642 : if (cpu == env->best_cpu || !idle_cpu(cpu) ||
1643 : !cpumask_test_cpu(cpu, env->p->cpus_ptr)) {
1644 : continue;
1645 : }
1646 :
1647 : env->dst_cpu = cpu;
1648 : rq = cpu_rq(env->dst_cpu);
1649 : if (!xchg(&rq->numa_migrate_on, 1))
1650 : goto assign;
1651 : }
1652 :
1653 : /* Failed to find an alternative idle CPU */
1654 : return;
1655 : }
1656 :
1657 : assign:
1658 : /*
1659 : * Clear previous best_cpu/rq numa-migrate flag, since task now
1660 : * found a better CPU to move/swap.
1661 : */
1662 : if (env->best_cpu != -1 && env->best_cpu != env->dst_cpu) {
1663 : rq = cpu_rq(env->best_cpu);
1664 : WRITE_ONCE(rq->numa_migrate_on, 0);
1665 : }
1666 :
1667 : if (env->best_task)
1668 : put_task_struct(env->best_task);
1669 : if (p)
1670 : get_task_struct(p);
1671 :
1672 : env->best_task = p;
1673 : env->best_imp = imp;
1674 : env->best_cpu = env->dst_cpu;
1675 : }
1676 :
1677 : static bool load_too_imbalanced(long src_load, long dst_load,
1678 : struct task_numa_env *env)
1679 : {
1680 : long imb, old_imb;
1681 : long orig_src_load, orig_dst_load;
1682 : long src_capacity, dst_capacity;
1683 :
1684 : /*
1685 : * The load is corrected for the CPU capacity available on each node.
1686 : *
1687 : * src_load dst_load
1688 : * ------------ vs ---------
1689 : * src_capacity dst_capacity
1690 : */
1691 : src_capacity = env->src_stats.compute_capacity;
1692 : dst_capacity = env->dst_stats.compute_capacity;
1693 :
1694 : imb = abs(dst_load * src_capacity - src_load * dst_capacity);
1695 :
1696 : orig_src_load = env->src_stats.load;
1697 : orig_dst_load = env->dst_stats.load;
1698 :
1699 : old_imb = abs(orig_dst_load * src_capacity - orig_src_load * dst_capacity);
1700 :
1701 : /* Would this change make things worse? */
1702 : return (imb > old_imb);
1703 : }
1704 :
1705 : /*
1706 : * Maximum NUMA importance can be 1998 (2*999);
1707 : * SMALLIMP @ 30 would be close to 1998/64.
1708 : * Used to deter task migration.
1709 : */
1710 : #define SMALLIMP 30
1711 :
1712 : /*
1713 : * This checks if the overall compute and NUMA accesses of the system would
1714 : * be improved if the source tasks was migrated to the target dst_cpu taking
1715 : * into account that it might be best if task running on the dst_cpu should
1716 : * be exchanged with the source task
1717 : */
1718 : static bool task_numa_compare(struct task_numa_env *env,
1719 : long taskimp, long groupimp, bool maymove)
1720 : {
1721 : struct numa_group *cur_ng, *p_ng = deref_curr_numa_group(env->p);
1722 : struct rq *dst_rq = cpu_rq(env->dst_cpu);
1723 : long imp = p_ng ? groupimp : taskimp;
1724 : struct task_struct *cur;
1725 : long src_load, dst_load;
1726 : int dist = env->dist;
1727 : long moveimp = imp;
1728 : long load;
1729 : bool stopsearch = false;
1730 :
1731 : if (READ_ONCE(dst_rq->numa_migrate_on))
1732 : return false;
1733 :
1734 : rcu_read_lock();
1735 : cur = rcu_dereference(dst_rq->curr);
1736 : if (cur && ((cur->flags & PF_EXITING) || is_idle_task(cur)))
1737 : cur = NULL;
1738 :
1739 : /*
1740 : * Because we have preemption enabled we can get migrated around and
1741 : * end try selecting ourselves (current == env->p) as a swap candidate.
1742 : */
1743 : if (cur == env->p) {
1744 : stopsearch = true;
1745 : goto unlock;
1746 : }
1747 :
1748 : if (!cur) {
1749 : if (maymove && moveimp >= env->best_imp)
1750 : goto assign;
1751 : else
1752 : goto unlock;
1753 : }
1754 :
1755 : /* Skip this swap candidate if cannot move to the source cpu. */
1756 : if (!cpumask_test_cpu(env->src_cpu, cur->cpus_ptr))
1757 : goto unlock;
1758 :
1759 : /*
1760 : * Skip this swap candidate if it is not moving to its preferred
1761 : * node and the best task is.
1762 : */
1763 : if (env->best_task &&
1764 : env->best_task->numa_preferred_nid == env->src_nid &&
1765 : cur->numa_preferred_nid != env->src_nid) {
1766 : goto unlock;
1767 : }
1768 :
1769 : /*
1770 : * "imp" is the fault differential for the source task between the
1771 : * source and destination node. Calculate the total differential for
1772 : * the source task and potential destination task. The more negative
1773 : * the value is, the more remote accesses that would be expected to
1774 : * be incurred if the tasks were swapped.
1775 : *
1776 : * If dst and source tasks are in the same NUMA group, or not
1777 : * in any group then look only at task weights.
1778 : */
1779 : cur_ng = rcu_dereference(cur->numa_group);
1780 : if (cur_ng == p_ng) {
1781 : imp = taskimp + task_weight(cur, env->src_nid, dist) -
1782 : task_weight(cur, env->dst_nid, dist);
1783 : /*
1784 : * Add some hysteresis to prevent swapping the
1785 : * tasks within a group over tiny differences.
1786 : */
1787 : if (cur_ng)
1788 : imp -= imp / 16;
1789 : } else {
1790 : /*
1791 : * Compare the group weights. If a task is all by itself
1792 : * (not part of a group), use the task weight instead.
1793 : */
1794 : if (cur_ng && p_ng)
1795 : imp += group_weight(cur, env->src_nid, dist) -
1796 : group_weight(cur, env->dst_nid, dist);
1797 : else
1798 : imp += task_weight(cur, env->src_nid, dist) -
1799 : task_weight(cur, env->dst_nid, dist);
1800 : }
1801 :
1802 : /* Discourage picking a task already on its preferred node */
1803 : if (cur->numa_preferred_nid == env->dst_nid)
1804 : imp -= imp / 16;
1805 :
1806 : /*
1807 : * Encourage picking a task that moves to its preferred node.
1808 : * This potentially makes imp larger than it's maximum of
1809 : * 1998 (see SMALLIMP and task_weight for why) but in this
1810 : * case, it does not matter.
1811 : */
1812 : if (cur->numa_preferred_nid == env->src_nid)
1813 : imp += imp / 8;
1814 :
1815 : if (maymove && moveimp > imp && moveimp > env->best_imp) {
1816 : imp = moveimp;
1817 : cur = NULL;
1818 : goto assign;
1819 : }
1820 :
1821 : /*
1822 : * Prefer swapping with a task moving to its preferred node over a
1823 : * task that is not.
1824 : */
1825 : if (env->best_task && cur->numa_preferred_nid == env->src_nid &&
1826 : env->best_task->numa_preferred_nid != env->src_nid) {
1827 : goto assign;
1828 : }
1829 :
1830 : /*
1831 : * If the NUMA importance is less than SMALLIMP,
1832 : * task migration might only result in ping pong
1833 : * of tasks and also hurt performance due to cache
1834 : * misses.
1835 : */
1836 : if (imp < SMALLIMP || imp <= env->best_imp + SMALLIMP / 2)
1837 : goto unlock;
1838 :
1839 : /*
1840 : * In the overloaded case, try and keep the load balanced.
1841 : */
1842 : load = task_h_load(env->p) - task_h_load(cur);
1843 : if (!load)
1844 : goto assign;
1845 :
1846 : dst_load = env->dst_stats.load + load;
1847 : src_load = env->src_stats.load - load;
1848 :
1849 : if (load_too_imbalanced(src_load, dst_load, env))
1850 : goto unlock;
1851 :
1852 : assign:
1853 : /* Evaluate an idle CPU for a task numa move. */
1854 : if (!cur) {
1855 : int cpu = env->dst_stats.idle_cpu;
1856 :
1857 : /* Nothing cached so current CPU went idle since the search. */
1858 : if (cpu < 0)
1859 : cpu = env->dst_cpu;
1860 :
1861 : /*
1862 : * If the CPU is no longer truly idle and the previous best CPU
1863 : * is, keep using it.
1864 : */
1865 : if (!idle_cpu(cpu) && env->best_cpu >= 0 &&
1866 : idle_cpu(env->best_cpu)) {
1867 : cpu = env->best_cpu;
1868 : }
1869 :
1870 : env->dst_cpu = cpu;
1871 : }
1872 :
1873 : task_numa_assign(env, cur, imp);
1874 :
1875 : /*
1876 : * If a move to idle is allowed because there is capacity or load
1877 : * balance improves then stop the search. While a better swap
1878 : * candidate may exist, a search is not free.
1879 : */
1880 : if (maymove && !cur && env->best_cpu >= 0 && idle_cpu(env->best_cpu))
1881 : stopsearch = true;
1882 :
1883 : /*
1884 : * If a swap candidate must be identified and the current best task
1885 : * moves its preferred node then stop the search.
1886 : */
1887 : if (!maymove && env->best_task &&
1888 : env->best_task->numa_preferred_nid == env->src_nid) {
1889 : stopsearch = true;
1890 : }
1891 : unlock:
1892 : rcu_read_unlock();
1893 :
1894 : return stopsearch;
1895 : }
1896 :
1897 : static void task_numa_find_cpu(struct task_numa_env *env,
1898 : long taskimp, long groupimp)
1899 : {
1900 : bool maymove = false;
1901 : int cpu;
1902 :
1903 : /*
1904 : * If dst node has spare capacity, then check if there is an
1905 : * imbalance that would be overruled by the load balancer.
1906 : */
1907 : if (env->dst_stats.node_type == node_has_spare) {
1908 : unsigned int imbalance;
1909 : int src_running, dst_running;
1910 :
1911 : /*
1912 : * Would movement cause an imbalance? Note that if src has
1913 : * more running tasks that the imbalance is ignored as the
1914 : * move improves the imbalance from the perspective of the
1915 : * CPU load balancer.
1916 : * */
1917 : src_running = env->src_stats.nr_running - 1;
1918 : dst_running = env->dst_stats.nr_running + 1;
1919 : imbalance = max(0, dst_running - src_running);
1920 : imbalance = adjust_numa_imbalance(imbalance, dst_running,
1921 : env->imb_numa_nr);
1922 :
1923 : /* Use idle CPU if there is no imbalance */
1924 : if (!imbalance) {
1925 : maymove = true;
1926 : if (env->dst_stats.idle_cpu >= 0) {
1927 : env->dst_cpu = env->dst_stats.idle_cpu;
1928 : task_numa_assign(env, NULL, 0);
1929 : return;
1930 : }
1931 : }
1932 : } else {
1933 : long src_load, dst_load, load;
1934 : /*
1935 : * If the improvement from just moving env->p direction is better
1936 : * than swapping tasks around, check if a move is possible.
1937 : */
1938 : load = task_h_load(env->p);
1939 : dst_load = env->dst_stats.load + load;
1940 : src_load = env->src_stats.load - load;
1941 : maymove = !load_too_imbalanced(src_load, dst_load, env);
1942 : }
1943 :
1944 : for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) {
1945 : /* Skip this CPU if the source task cannot migrate */
1946 : if (!cpumask_test_cpu(cpu, env->p->cpus_ptr))
1947 : continue;
1948 :
1949 : env->dst_cpu = cpu;
1950 : if (task_numa_compare(env, taskimp, groupimp, maymove))
1951 : break;
1952 : }
1953 : }
1954 :
1955 : static int task_numa_migrate(struct task_struct *p)
1956 : {
1957 : struct task_numa_env env = {
1958 : .p = p,
1959 :
1960 : .src_cpu = task_cpu(p),
1961 : .src_nid = task_node(p),
1962 :
1963 : .imbalance_pct = 112,
1964 :
1965 : .best_task = NULL,
1966 : .best_imp = 0,
1967 : .best_cpu = -1,
1968 : };
1969 : unsigned long taskweight, groupweight;
1970 : struct sched_domain *sd;
1971 : long taskimp, groupimp;
1972 : struct numa_group *ng;
1973 : struct rq *best_rq;
1974 : int nid, ret, dist;
1975 :
1976 : /*
1977 : * Pick the lowest SD_NUMA domain, as that would have the smallest
1978 : * imbalance and would be the first to start moving tasks about.
1979 : *
1980 : * And we want to avoid any moving of tasks about, as that would create
1981 : * random movement of tasks -- counter the numa conditions we're trying
1982 : * to satisfy here.
1983 : */
1984 : rcu_read_lock();
1985 : sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu));
1986 : if (sd) {
1987 : env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2;
1988 : env.imb_numa_nr = sd->imb_numa_nr;
1989 : }
1990 : rcu_read_unlock();
1991 :
1992 : /*
1993 : * Cpusets can break the scheduler domain tree into smaller
1994 : * balance domains, some of which do not cross NUMA boundaries.
1995 : * Tasks that are "trapped" in such domains cannot be migrated
1996 : * elsewhere, so there is no point in (re)trying.
1997 : */
1998 : if (unlikely(!sd)) {
1999 : sched_setnuma(p, task_node(p));
2000 : return -EINVAL;
2001 : }
2002 :
2003 : env.dst_nid = p->numa_preferred_nid;
2004 : dist = env.dist = node_distance(env.src_nid, env.dst_nid);
2005 : taskweight = task_weight(p, env.src_nid, dist);
2006 : groupweight = group_weight(p, env.src_nid, dist);
2007 : update_numa_stats(&env, &env.src_stats, env.src_nid, false);
2008 : taskimp = task_weight(p, env.dst_nid, dist) - taskweight;
2009 : groupimp = group_weight(p, env.dst_nid, dist) - groupweight;
2010 : update_numa_stats(&env, &env.dst_stats, env.dst_nid, true);
2011 :
2012 : /* Try to find a spot on the preferred nid. */
2013 : task_numa_find_cpu(&env, taskimp, groupimp);
2014 :
2015 : /*
2016 : * Look at other nodes in these cases:
2017 : * - there is no space available on the preferred_nid
2018 : * - the task is part of a numa_group that is interleaved across
2019 : * multiple NUMA nodes; in order to better consolidate the group,
2020 : * we need to check other locations.
2021 : */
2022 : ng = deref_curr_numa_group(p);
2023 : if (env.best_cpu == -1 || (ng && ng->active_nodes > 1)) {
2024 : for_each_node_state(nid, N_CPU) {
2025 : if (nid == env.src_nid || nid == p->numa_preferred_nid)
2026 : continue;
2027 :
2028 : dist = node_distance(env.src_nid, env.dst_nid);
2029 : if (sched_numa_topology_type == NUMA_BACKPLANE &&
2030 : dist != env.dist) {
2031 : taskweight = task_weight(p, env.src_nid, dist);
2032 : groupweight = group_weight(p, env.src_nid, dist);
2033 : }
2034 :
2035 : /* Only consider nodes where both task and groups benefit */
2036 : taskimp = task_weight(p, nid, dist) - taskweight;
2037 : groupimp = group_weight(p, nid, dist) - groupweight;
2038 : if (taskimp < 0 && groupimp < 0)
2039 : continue;
2040 :
2041 : env.dist = dist;
2042 : env.dst_nid = nid;
2043 : update_numa_stats(&env, &env.dst_stats, env.dst_nid, true);
2044 : task_numa_find_cpu(&env, taskimp, groupimp);
2045 : }
2046 : }
2047 :
2048 : /*
2049 : * If the task is part of a workload that spans multiple NUMA nodes,
2050 : * and is migrating into one of the workload's active nodes, remember
2051 : * this node as the task's preferred numa node, so the workload can
2052 : * settle down.
2053 : * A task that migrated to a second choice node will be better off
2054 : * trying for a better one later. Do not set the preferred node here.
2055 : */
2056 : if (ng) {
2057 : if (env.best_cpu == -1)
2058 : nid = env.src_nid;
2059 : else
2060 : nid = cpu_to_node(env.best_cpu);
2061 :
2062 : if (nid != p->numa_preferred_nid)
2063 : sched_setnuma(p, nid);
2064 : }
2065 :
2066 : /* No better CPU than the current one was found. */
2067 : if (env.best_cpu == -1) {
2068 : trace_sched_stick_numa(p, env.src_cpu, NULL, -1);
2069 : return -EAGAIN;
2070 : }
2071 :
2072 : best_rq = cpu_rq(env.best_cpu);
2073 : if (env.best_task == NULL) {
2074 : ret = migrate_task_to(p, env.best_cpu);
2075 : WRITE_ONCE(best_rq->numa_migrate_on, 0);
2076 : if (ret != 0)
2077 : trace_sched_stick_numa(p, env.src_cpu, NULL, env.best_cpu);
2078 : return ret;
2079 : }
2080 :
2081 : ret = migrate_swap(p, env.best_task, env.best_cpu, env.src_cpu);
2082 : WRITE_ONCE(best_rq->numa_migrate_on, 0);
2083 :
2084 : if (ret != 0)
2085 : trace_sched_stick_numa(p, env.src_cpu, env.best_task, env.best_cpu);
2086 : put_task_struct(env.best_task);
2087 : return ret;
2088 : }
2089 :
2090 : /* Attempt to migrate a task to a CPU on the preferred node. */
2091 : static void numa_migrate_preferred(struct task_struct *p)
2092 : {
2093 : unsigned long interval = HZ;
2094 :
2095 : /* This task has no NUMA fault statistics yet */
2096 : if (unlikely(p->numa_preferred_nid == NUMA_NO_NODE || !p->numa_faults))
2097 : return;
2098 :
2099 : /* Periodically retry migrating the task to the preferred node */
2100 : interval = min(interval, msecs_to_jiffies(p->numa_scan_period) / 16);
2101 : p->numa_migrate_retry = jiffies + interval;
2102 :
2103 : /* Success if task is already running on preferred CPU */
2104 : if (task_node(p) == p->numa_preferred_nid)
2105 : return;
2106 :
2107 : /* Otherwise, try migrate to a CPU on the preferred node */
2108 : task_numa_migrate(p);
2109 : }
2110 :
2111 : /*
2112 : * Find out how many nodes the workload is actively running on. Do this by
2113 : * tracking the nodes from which NUMA hinting faults are triggered. This can
2114 : * be different from the set of nodes where the workload's memory is currently
2115 : * located.
2116 : */
2117 : static void numa_group_count_active_nodes(struct numa_group *numa_group)
2118 : {
2119 : unsigned long faults, max_faults = 0;
2120 : int nid, active_nodes = 0;
2121 :
2122 : for_each_node_state(nid, N_CPU) {
2123 : faults = group_faults_cpu(numa_group, nid);
2124 : if (faults > max_faults)
2125 : max_faults = faults;
2126 : }
2127 :
2128 : for_each_node_state(nid, N_CPU) {
2129 : faults = group_faults_cpu(numa_group, nid);
2130 : if (faults * ACTIVE_NODE_FRACTION > max_faults)
2131 : active_nodes++;
2132 : }
2133 :
2134 : numa_group->max_faults_cpu = max_faults;
2135 : numa_group->active_nodes = active_nodes;
2136 : }
2137 :
2138 : /*
2139 : * When adapting the scan rate, the period is divided into NUMA_PERIOD_SLOTS
2140 : * increments. The more local the fault statistics are, the higher the scan
2141 : * period will be for the next scan window. If local/(local+remote) ratio is
2142 : * below NUMA_PERIOD_THRESHOLD (where range of ratio is 1..NUMA_PERIOD_SLOTS)
2143 : * the scan period will decrease. Aim for 70% local accesses.
2144 : */
2145 : #define NUMA_PERIOD_SLOTS 10
2146 : #define NUMA_PERIOD_THRESHOLD 7
2147 :
2148 : /*
2149 : * Increase the scan period (slow down scanning) if the majority of
2150 : * our memory is already on our local node, or if the majority of
2151 : * the page accesses are shared with other processes.
2152 : * Otherwise, decrease the scan period.
2153 : */
2154 : static void update_task_scan_period(struct task_struct *p,
2155 : unsigned long shared, unsigned long private)
2156 : {
2157 : unsigned int period_slot;
2158 : int lr_ratio, ps_ratio;
2159 : int diff;
2160 :
2161 : unsigned long remote = p->numa_faults_locality[0];
2162 : unsigned long local = p->numa_faults_locality[1];
2163 :
2164 : /*
2165 : * If there were no record hinting faults then either the task is
2166 : * completely idle or all activity is in areas that are not of interest
2167 : * to automatic numa balancing. Related to that, if there were failed
2168 : * migration then it implies we are migrating too quickly or the local
2169 : * node is overloaded. In either case, scan slower
2170 : */
2171 : if (local + shared == 0 || p->numa_faults_locality[2]) {
2172 : p->numa_scan_period = min(p->numa_scan_period_max,
2173 : p->numa_scan_period << 1);
2174 :
2175 : p->mm->numa_next_scan = jiffies +
2176 : msecs_to_jiffies(p->numa_scan_period);
2177 :
2178 : return;
2179 : }
2180 :
2181 : /*
2182 : * Prepare to scale scan period relative to the current period.
2183 : * == NUMA_PERIOD_THRESHOLD scan period stays the same
2184 : * < NUMA_PERIOD_THRESHOLD scan period decreases (scan faster)
2185 : * >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower)
2186 : */
2187 : period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS);
2188 : lr_ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote);
2189 : ps_ratio = (private * NUMA_PERIOD_SLOTS) / (private + shared);
2190 :
2191 : if (ps_ratio >= NUMA_PERIOD_THRESHOLD) {
2192 : /*
2193 : * Most memory accesses are local. There is no need to
2194 : * do fast NUMA scanning, since memory is already local.
2195 : */
2196 : int slot = ps_ratio - NUMA_PERIOD_THRESHOLD;
2197 : if (!slot)
2198 : slot = 1;
2199 : diff = slot * period_slot;
2200 : } else if (lr_ratio >= NUMA_PERIOD_THRESHOLD) {
2201 : /*
2202 : * Most memory accesses are shared with other tasks.
2203 : * There is no point in continuing fast NUMA scanning,
2204 : * since other tasks may just move the memory elsewhere.
2205 : */
2206 : int slot = lr_ratio - NUMA_PERIOD_THRESHOLD;
2207 : if (!slot)
2208 : slot = 1;
2209 : diff = slot * period_slot;
2210 : } else {
2211 : /*
2212 : * Private memory faults exceed (SLOTS-THRESHOLD)/SLOTS,
2213 : * yet they are not on the local NUMA node. Speed up
2214 : * NUMA scanning to get the memory moved over.
2215 : */
2216 : int ratio = max(lr_ratio, ps_ratio);
2217 : diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot;
2218 : }
2219 :
2220 : p->numa_scan_period = clamp(p->numa_scan_period + diff,
2221 : task_scan_min(p), task_scan_max(p));
2222 : memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality));
2223 : }
2224 :
2225 : /*
2226 : * Get the fraction of time the task has been running since the last
2227 : * NUMA placement cycle. The scheduler keeps similar statistics, but
2228 : * decays those on a 32ms period, which is orders of magnitude off
2229 : * from the dozens-of-seconds NUMA balancing period. Use the scheduler
2230 : * stats only if the task is so new there are no NUMA statistics yet.
2231 : */
2232 : static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period)
2233 : {
2234 : u64 runtime, delta, now;
2235 : /* Use the start of this time slice to avoid calculations. */
2236 : now = p->se.exec_start;
2237 : runtime = p->se.sum_exec_runtime;
2238 :
2239 : if (p->last_task_numa_placement) {
2240 : delta = runtime - p->last_sum_exec_runtime;
2241 : *period = now - p->last_task_numa_placement;
2242 :
2243 : /* Avoid time going backwards, prevent potential divide error: */
2244 : if (unlikely((s64)*period < 0))
2245 : *period = 0;
2246 : } else {
2247 : delta = p->se.avg.load_sum;
2248 : *period = LOAD_AVG_MAX;
2249 : }
2250 :
2251 : p->last_sum_exec_runtime = runtime;
2252 : p->last_task_numa_placement = now;
2253 :
2254 : return delta;
2255 : }
2256 :
2257 : /*
2258 : * Determine the preferred nid for a task in a numa_group. This needs to
2259 : * be done in a way that produces consistent results with group_weight,
2260 : * otherwise workloads might not converge.
2261 : */
2262 : static int preferred_group_nid(struct task_struct *p, int nid)
2263 : {
2264 : nodemask_t nodes;
2265 : int dist;
2266 :
2267 : /* Direct connections between all NUMA nodes. */
2268 : if (sched_numa_topology_type == NUMA_DIRECT)
2269 : return nid;
2270 :
2271 : /*
2272 : * On a system with glueless mesh NUMA topology, group_weight
2273 : * scores nodes according to the number of NUMA hinting faults on
2274 : * both the node itself, and on nearby nodes.
2275 : */
2276 : if (sched_numa_topology_type == NUMA_GLUELESS_MESH) {
2277 : unsigned long score, max_score = 0;
2278 : int node, max_node = nid;
2279 :
2280 : dist = sched_max_numa_distance;
2281 :
2282 : for_each_node_state(node, N_CPU) {
2283 : score = group_weight(p, node, dist);
2284 : if (score > max_score) {
2285 : max_score = score;
2286 : max_node = node;
2287 : }
2288 : }
2289 : return max_node;
2290 : }
2291 :
2292 : /*
2293 : * Finding the preferred nid in a system with NUMA backplane
2294 : * interconnect topology is more involved. The goal is to locate
2295 : * tasks from numa_groups near each other in the system, and
2296 : * untangle workloads from different sides of the system. This requires
2297 : * searching down the hierarchy of node groups, recursively searching
2298 : * inside the highest scoring group of nodes. The nodemask tricks
2299 : * keep the complexity of the search down.
2300 : */
2301 : nodes = node_states[N_CPU];
2302 : for (dist = sched_max_numa_distance; dist > LOCAL_DISTANCE; dist--) {
2303 : unsigned long max_faults = 0;
2304 : nodemask_t max_group = NODE_MASK_NONE;
2305 : int a, b;
2306 :
2307 : /* Are there nodes at this distance from each other? */
2308 : if (!find_numa_distance(dist))
2309 : continue;
2310 :
2311 : for_each_node_mask(a, nodes) {
2312 : unsigned long faults = 0;
2313 : nodemask_t this_group;
2314 : nodes_clear(this_group);
2315 :
2316 : /* Sum group's NUMA faults; includes a==b case. */
2317 : for_each_node_mask(b, nodes) {
2318 : if (node_distance(a, b) < dist) {
2319 : faults += group_faults(p, b);
2320 : node_set(b, this_group);
2321 : node_clear(b, nodes);
2322 : }
2323 : }
2324 :
2325 : /* Remember the top group. */
2326 : if (faults > max_faults) {
2327 : max_faults = faults;
2328 : max_group = this_group;
2329 : /*
2330 : * subtle: at the smallest distance there is
2331 : * just one node left in each "group", the
2332 : * winner is the preferred nid.
2333 : */
2334 : nid = a;
2335 : }
2336 : }
2337 : /* Next round, evaluate the nodes within max_group. */
2338 : if (!max_faults)
2339 : break;
2340 : nodes = max_group;
2341 : }
2342 : return nid;
2343 : }
2344 :
2345 : static void task_numa_placement(struct task_struct *p)
2346 : {
2347 : int seq, nid, max_nid = NUMA_NO_NODE;
2348 : unsigned long max_faults = 0;
2349 : unsigned long fault_types[2] = { 0, 0 };
2350 : unsigned long total_faults;
2351 : u64 runtime, period;
2352 : spinlock_t *group_lock = NULL;
2353 : struct numa_group *ng;
2354 :
2355 : /*
2356 : * The p->mm->numa_scan_seq field gets updated without
2357 : * exclusive access. Use READ_ONCE() here to ensure
2358 : * that the field is read in a single access:
2359 : */
2360 : seq = READ_ONCE(p->mm->numa_scan_seq);
2361 : if (p->numa_scan_seq == seq)
2362 : return;
2363 : p->numa_scan_seq = seq;
2364 : p->numa_scan_period_max = task_scan_max(p);
2365 :
2366 : total_faults = p->numa_faults_locality[0] +
2367 : p->numa_faults_locality[1];
2368 : runtime = numa_get_avg_runtime(p, &period);
2369 :
2370 : /* If the task is part of a group prevent parallel updates to group stats */
2371 : ng = deref_curr_numa_group(p);
2372 : if (ng) {
2373 : group_lock = &ng->lock;
2374 : spin_lock_irq(group_lock);
2375 : }
2376 :
2377 : /* Find the node with the highest number of faults */
2378 : for_each_online_node(nid) {
2379 : /* Keep track of the offsets in numa_faults array */
2380 : int mem_idx, membuf_idx, cpu_idx, cpubuf_idx;
2381 : unsigned long faults = 0, group_faults = 0;
2382 : int priv;
2383 :
2384 : for (priv = 0; priv < NR_NUMA_HINT_FAULT_TYPES; priv++) {
2385 : long diff, f_diff, f_weight;
2386 :
2387 : mem_idx = task_faults_idx(NUMA_MEM, nid, priv);
2388 : membuf_idx = task_faults_idx(NUMA_MEMBUF, nid, priv);
2389 : cpu_idx = task_faults_idx(NUMA_CPU, nid, priv);
2390 : cpubuf_idx = task_faults_idx(NUMA_CPUBUF, nid, priv);
2391 :
2392 : /* Decay existing window, copy faults since last scan */
2393 : diff = p->numa_faults[membuf_idx] - p->numa_faults[mem_idx] / 2;
2394 : fault_types[priv] += p->numa_faults[membuf_idx];
2395 : p->numa_faults[membuf_idx] = 0;
2396 :
2397 : /*
2398 : * Normalize the faults_from, so all tasks in a group
2399 : * count according to CPU use, instead of by the raw
2400 : * number of faults. Tasks with little runtime have
2401 : * little over-all impact on throughput, and thus their
2402 : * faults are less important.
2403 : */
2404 : f_weight = div64_u64(runtime << 16, period + 1);
2405 : f_weight = (f_weight * p->numa_faults[cpubuf_idx]) /
2406 : (total_faults + 1);
2407 : f_diff = f_weight - p->numa_faults[cpu_idx] / 2;
2408 : p->numa_faults[cpubuf_idx] = 0;
2409 :
2410 : p->numa_faults[mem_idx] += diff;
2411 : p->numa_faults[cpu_idx] += f_diff;
2412 : faults += p->numa_faults[mem_idx];
2413 : p->total_numa_faults += diff;
2414 : if (ng) {
2415 : /*
2416 : * safe because we can only change our own group
2417 : *
2418 : * mem_idx represents the offset for a given
2419 : * nid and priv in a specific region because it
2420 : * is at the beginning of the numa_faults array.
2421 : */
2422 : ng->faults[mem_idx] += diff;
2423 : ng->faults[cpu_idx] += f_diff;
2424 : ng->total_faults += diff;
2425 : group_faults += ng->faults[mem_idx];
2426 : }
2427 : }
2428 :
2429 : if (!ng) {
2430 : if (faults > max_faults) {
2431 : max_faults = faults;
2432 : max_nid = nid;
2433 : }
2434 : } else if (group_faults > max_faults) {
2435 : max_faults = group_faults;
2436 : max_nid = nid;
2437 : }
2438 : }
2439 :
2440 : /* Cannot migrate task to CPU-less node */
2441 : if (max_nid != NUMA_NO_NODE && !node_state(max_nid, N_CPU)) {
2442 : int near_nid = max_nid;
2443 : int distance, near_distance = INT_MAX;
2444 :
2445 : for_each_node_state(nid, N_CPU) {
2446 : distance = node_distance(max_nid, nid);
2447 : if (distance < near_distance) {
2448 : near_nid = nid;
2449 : near_distance = distance;
2450 : }
2451 : }
2452 : max_nid = near_nid;
2453 : }
2454 :
2455 : if (ng) {
2456 : numa_group_count_active_nodes(ng);
2457 : spin_unlock_irq(group_lock);
2458 : max_nid = preferred_group_nid(p, max_nid);
2459 : }
2460 :
2461 : if (max_faults) {
2462 : /* Set the new preferred node */
2463 : if (max_nid != p->numa_preferred_nid)
2464 : sched_setnuma(p, max_nid);
2465 : }
2466 :
2467 : update_task_scan_period(p, fault_types[0], fault_types[1]);
2468 : }
2469 :
2470 : static inline int get_numa_group(struct numa_group *grp)
2471 : {
2472 : return refcount_inc_not_zero(&grp->refcount);
2473 : }
2474 :
2475 : static inline void put_numa_group(struct numa_group *grp)
2476 : {
2477 : if (refcount_dec_and_test(&grp->refcount))
2478 : kfree_rcu(grp, rcu);
2479 : }
2480 :
2481 : static void task_numa_group(struct task_struct *p, int cpupid, int flags,
2482 : int *priv)
2483 : {
2484 : struct numa_group *grp, *my_grp;
2485 : struct task_struct *tsk;
2486 : bool join = false;
2487 : int cpu = cpupid_to_cpu(cpupid);
2488 : int i;
2489 :
2490 : if (unlikely(!deref_curr_numa_group(p))) {
2491 : unsigned int size = sizeof(struct numa_group) +
2492 : NR_NUMA_HINT_FAULT_STATS *
2493 : nr_node_ids * sizeof(unsigned long);
2494 :
2495 : grp = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
2496 : if (!grp)
2497 : return;
2498 :
2499 : refcount_set(&grp->refcount, 1);
2500 : grp->active_nodes = 1;
2501 : grp->max_faults_cpu = 0;
2502 : spin_lock_init(&grp->lock);
2503 : grp->gid = p->pid;
2504 :
2505 : for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++)
2506 : grp->faults[i] = p->numa_faults[i];
2507 :
2508 : grp->total_faults = p->total_numa_faults;
2509 :
2510 : grp->nr_tasks++;
2511 : rcu_assign_pointer(p->numa_group, grp);
2512 : }
2513 :
2514 : rcu_read_lock();
2515 : tsk = READ_ONCE(cpu_rq(cpu)->curr);
2516 :
2517 : if (!cpupid_match_pid(tsk, cpupid))
2518 : goto no_join;
2519 :
2520 : grp = rcu_dereference(tsk->numa_group);
2521 : if (!grp)
2522 : goto no_join;
2523 :
2524 : my_grp = deref_curr_numa_group(p);
2525 : if (grp == my_grp)
2526 : goto no_join;
2527 :
2528 : /*
2529 : * Only join the other group if its bigger; if we're the bigger group,
2530 : * the other task will join us.
2531 : */
2532 : if (my_grp->nr_tasks > grp->nr_tasks)
2533 : goto no_join;
2534 :
2535 : /*
2536 : * Tie-break on the grp address.
2537 : */
2538 : if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp)
2539 : goto no_join;
2540 :
2541 : /* Always join threads in the same process. */
2542 : if (tsk->mm == current->mm)
2543 : join = true;
2544 :
2545 : /* Simple filter to avoid false positives due to PID collisions */
2546 : if (flags & TNF_SHARED)
2547 : join = true;
2548 :
2549 : /* Update priv based on whether false sharing was detected */
2550 : *priv = !join;
2551 :
2552 : if (join && !get_numa_group(grp))
2553 : goto no_join;
2554 :
2555 : rcu_read_unlock();
2556 :
2557 : if (!join)
2558 : return;
2559 :
2560 : BUG_ON(irqs_disabled());
2561 : double_lock_irq(&my_grp->lock, &grp->lock);
2562 :
2563 : for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) {
2564 : my_grp->faults[i] -= p->numa_faults[i];
2565 : grp->faults[i] += p->numa_faults[i];
2566 : }
2567 : my_grp->total_faults -= p->total_numa_faults;
2568 : grp->total_faults += p->total_numa_faults;
2569 :
2570 : my_grp->nr_tasks--;
2571 : grp->nr_tasks++;
2572 :
2573 : spin_unlock(&my_grp->lock);
2574 : spin_unlock_irq(&grp->lock);
2575 :
2576 : rcu_assign_pointer(p->numa_group, grp);
2577 :
2578 : put_numa_group(my_grp);
2579 : return;
2580 :
2581 : no_join:
2582 : rcu_read_unlock();
2583 : return;
2584 : }
2585 :
2586 : /*
2587 : * Get rid of NUMA statistics associated with a task (either current or dead).
2588 : * If @final is set, the task is dead and has reached refcount zero, so we can
2589 : * safely free all relevant data structures. Otherwise, there might be
2590 : * concurrent reads from places like load balancing and procfs, and we should
2591 : * reset the data back to default state without freeing ->numa_faults.
2592 : */
2593 : void task_numa_free(struct task_struct *p, bool final)
2594 : {
2595 : /* safe: p either is current or is being freed by current */
2596 : struct numa_group *grp = rcu_dereference_raw(p->numa_group);
2597 : unsigned long *numa_faults = p->numa_faults;
2598 : unsigned long flags;
2599 : int i;
2600 :
2601 : if (!numa_faults)
2602 : return;
2603 :
2604 : if (grp) {
2605 : spin_lock_irqsave(&grp->lock, flags);
2606 : for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++)
2607 : grp->faults[i] -= p->numa_faults[i];
2608 : grp->total_faults -= p->total_numa_faults;
2609 :
2610 : grp->nr_tasks--;
2611 : spin_unlock_irqrestore(&grp->lock, flags);
2612 : RCU_INIT_POINTER(p->numa_group, NULL);
2613 : put_numa_group(grp);
2614 : }
2615 :
2616 : if (final) {
2617 : p->numa_faults = NULL;
2618 : kfree(numa_faults);
2619 : } else {
2620 : p->total_numa_faults = 0;
2621 : for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++)
2622 : numa_faults[i] = 0;
2623 : }
2624 : }
2625 :
2626 : /*
2627 : * Got a PROT_NONE fault for a page on @node.
2628 : */
2629 : void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags)
2630 : {
2631 : struct task_struct *p = current;
2632 : bool migrated = flags & TNF_MIGRATED;
2633 : int cpu_node = task_node(current);
2634 : int local = !!(flags & TNF_FAULT_LOCAL);
2635 : struct numa_group *ng;
2636 : int priv;
2637 :
2638 : if (!static_branch_likely(&sched_numa_balancing))
2639 : return;
2640 :
2641 : /* for example, ksmd faulting in a user's mm */
2642 : if (!p->mm)
2643 : return;
2644 :
2645 : /* Allocate buffer to track faults on a per-node basis */
2646 : if (unlikely(!p->numa_faults)) {
2647 : int size = sizeof(*p->numa_faults) *
2648 : NR_NUMA_HINT_FAULT_BUCKETS * nr_node_ids;
2649 :
2650 : p->numa_faults = kzalloc(size, GFP_KERNEL|__GFP_NOWARN);
2651 : if (!p->numa_faults)
2652 : return;
2653 :
2654 : p->total_numa_faults = 0;
2655 : memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality));
2656 : }
2657 :
2658 : /*
2659 : * First accesses are treated as private, otherwise consider accesses
2660 : * to be private if the accessing pid has not changed
2661 : */
2662 : if (unlikely(last_cpupid == (-1 & LAST_CPUPID_MASK))) {
2663 : priv = 1;
2664 : } else {
2665 : priv = cpupid_match_pid(p, last_cpupid);
2666 : if (!priv && !(flags & TNF_NO_GROUP))
2667 : task_numa_group(p, last_cpupid, flags, &priv);
2668 : }
2669 :
2670 : /*
2671 : * If a workload spans multiple NUMA nodes, a shared fault that
2672 : * occurs wholly within the set of nodes that the workload is
2673 : * actively using should be counted as local. This allows the
2674 : * scan rate to slow down when a workload has settled down.
2675 : */
2676 : ng = deref_curr_numa_group(p);
2677 : if (!priv && !local && ng && ng->active_nodes > 1 &&
2678 : numa_is_active_node(cpu_node, ng) &&
2679 : numa_is_active_node(mem_node, ng))
2680 : local = 1;
2681 :
2682 : /*
2683 : * Retry to migrate task to preferred node periodically, in case it
2684 : * previously failed, or the scheduler moved us.
2685 : */
2686 : if (time_after(jiffies, p->numa_migrate_retry)) {
2687 : task_numa_placement(p);
2688 : numa_migrate_preferred(p);
2689 : }
2690 :
2691 : if (migrated)
2692 : p->numa_pages_migrated += pages;
2693 : if (flags & TNF_MIGRATE_FAIL)
2694 : p->numa_faults_locality[2] += pages;
2695 :
2696 : p->numa_faults[task_faults_idx(NUMA_MEMBUF, mem_node, priv)] += pages;
2697 : p->numa_faults[task_faults_idx(NUMA_CPUBUF, cpu_node, priv)] += pages;
2698 : p->numa_faults_locality[local] += pages;
2699 : }
2700 :
2701 : static void reset_ptenuma_scan(struct task_struct *p)
2702 : {
2703 : /*
2704 : * We only did a read acquisition of the mmap sem, so
2705 : * p->mm->numa_scan_seq is written to without exclusive access
2706 : * and the update is not guaranteed to be atomic. That's not
2707 : * much of an issue though, since this is just used for
2708 : * statistical sampling. Use READ_ONCE/WRITE_ONCE, which are not
2709 : * expensive, to avoid any form of compiler optimizations:
2710 : */
2711 : WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1);
2712 : p->mm->numa_scan_offset = 0;
2713 : }
2714 :
2715 : /*
2716 : * The expensive part of numa migration is done from task_work context.
2717 : * Triggered from task_tick_numa().
2718 : */
2719 : static void task_numa_work(struct callback_head *work)
2720 : {
2721 : unsigned long migrate, next_scan, now = jiffies;
2722 : struct task_struct *p = current;
2723 : struct mm_struct *mm = p->mm;
2724 : u64 runtime = p->se.sum_exec_runtime;
2725 : struct vm_area_struct *vma;
2726 : unsigned long start, end;
2727 : unsigned long nr_pte_updates = 0;
2728 : long pages, virtpages;
2729 :
2730 : SCHED_WARN_ON(p != container_of(work, struct task_struct, numa_work));
2731 :
2732 : work->next = work;
2733 : /*
2734 : * Who cares about NUMA placement when they're dying.
2735 : *
2736 : * NOTE: make sure not to dereference p->mm before this check,
2737 : * exit_task_work() happens _after_ exit_mm() so we could be called
2738 : * without p->mm even though we still had it when we enqueued this
2739 : * work.
2740 : */
2741 : if (p->flags & PF_EXITING)
2742 : return;
2743 :
2744 : if (!mm->numa_next_scan) {
2745 : mm->numa_next_scan = now +
2746 : msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
2747 : }
2748 :
2749 : /*
2750 : * Enforce maximal scan/migration frequency..
2751 : */
2752 : migrate = mm->numa_next_scan;
2753 : if (time_before(now, migrate))
2754 : return;
2755 :
2756 : if (p->numa_scan_period == 0) {
2757 : p->numa_scan_period_max = task_scan_max(p);
2758 : p->numa_scan_period = task_scan_start(p);
2759 : }
2760 :
2761 : next_scan = now + msecs_to_jiffies(p->numa_scan_period);
2762 : if (cmpxchg(&mm->numa_next_scan, migrate, next_scan) != migrate)
2763 : return;
2764 :
2765 : /*
2766 : * Delay this task enough that another task of this mm will likely win
2767 : * the next time around.
2768 : */
2769 : p->node_stamp += 2 * TICK_NSEC;
2770 :
2771 : start = mm->numa_scan_offset;
2772 : pages = sysctl_numa_balancing_scan_size;
2773 : pages <<= 20 - PAGE_SHIFT; /* MB in pages */
2774 : virtpages = pages * 8; /* Scan up to this much virtual space */
2775 : if (!pages)
2776 : return;
2777 :
2778 :
2779 : if (!mmap_read_trylock(mm))
2780 : return;
2781 : vma = find_vma(mm, start);
2782 : if (!vma) {
2783 : reset_ptenuma_scan(p);
2784 : start = 0;
2785 : vma = mm->mmap;
2786 : }
2787 : for (; vma; vma = vma->vm_next) {
2788 : if (!vma_migratable(vma) || !vma_policy_mof(vma) ||
2789 : is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) {
2790 : continue;
2791 : }
2792 :
2793 : /*
2794 : * Shared library pages mapped by multiple processes are not
2795 : * migrated as it is expected they are cache replicated. Avoid
2796 : * hinting faults in read-only file-backed mappings or the vdso
2797 : * as migrating the pages will be of marginal benefit.
2798 : */
2799 : if (!vma->vm_mm ||
2800 : (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ)))
2801 : continue;
2802 :
2803 : /*
2804 : * Skip inaccessible VMAs to avoid any confusion between
2805 : * PROT_NONE and NUMA hinting ptes
2806 : */
2807 : if (!vma_is_accessible(vma))
2808 : continue;
2809 :
2810 : do {
2811 : start = max(start, vma->vm_start);
2812 : end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE);
2813 : end = min(end, vma->vm_end);
2814 : nr_pte_updates = change_prot_numa(vma, start, end);
2815 :
2816 : /*
2817 : * Try to scan sysctl_numa_balancing_size worth of
2818 : * hpages that have at least one present PTE that
2819 : * is not already pte-numa. If the VMA contains
2820 : * areas that are unused or already full of prot_numa
2821 : * PTEs, scan up to virtpages, to skip through those
2822 : * areas faster.
2823 : */
2824 : if (nr_pte_updates)
2825 : pages -= (end - start) >> PAGE_SHIFT;
2826 : virtpages -= (end - start) >> PAGE_SHIFT;
2827 :
2828 : start = end;
2829 : if (pages <= 0 || virtpages <= 0)
2830 : goto out;
2831 :
2832 : cond_resched();
2833 : } while (end != vma->vm_end);
2834 : }
2835 :
2836 : out:
2837 : /*
2838 : * It is possible to reach the end of the VMA list but the last few
2839 : * VMAs are not guaranteed to the vma_migratable. If they are not, we
2840 : * would find the !migratable VMA on the next scan but not reset the
2841 : * scanner to the start so check it now.
2842 : */
2843 : if (vma)
2844 : mm->numa_scan_offset = start;
2845 : else
2846 : reset_ptenuma_scan(p);
2847 : mmap_read_unlock(mm);
2848 :
2849 : /*
2850 : * Make sure tasks use at least 32x as much time to run other code
2851 : * than they used here, to limit NUMA PTE scanning overhead to 3% max.
2852 : * Usually update_task_scan_period slows down scanning enough; on an
2853 : * overloaded system we need to limit overhead on a per task basis.
2854 : */
2855 : if (unlikely(p->se.sum_exec_runtime != runtime)) {
2856 : u64 diff = p->se.sum_exec_runtime - runtime;
2857 : p->node_stamp += 32 * diff;
2858 : }
2859 : }
2860 :
2861 : void init_numa_balancing(unsigned long clone_flags, struct task_struct *p)
2862 : {
2863 : int mm_users = 0;
2864 : struct mm_struct *mm = p->mm;
2865 :
2866 : if (mm) {
2867 : mm_users = atomic_read(&mm->mm_users);
2868 : if (mm_users == 1) {
2869 : mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
2870 : mm->numa_scan_seq = 0;
2871 : }
2872 : }
2873 : p->node_stamp = 0;
2874 : p->numa_scan_seq = mm ? mm->numa_scan_seq : 0;
2875 : p->numa_scan_period = sysctl_numa_balancing_scan_delay;
2876 : /* Protect against double add, see task_tick_numa and task_numa_work */
2877 : p->numa_work.next = &p->numa_work;
2878 : p->numa_faults = NULL;
2879 : p->numa_pages_migrated = 0;
2880 : p->total_numa_faults = 0;
2881 : RCU_INIT_POINTER(p->numa_group, NULL);
2882 : p->last_task_numa_placement = 0;
2883 : p->last_sum_exec_runtime = 0;
2884 :
2885 : init_task_work(&p->numa_work, task_numa_work);
2886 :
2887 : /* New address space, reset the preferred nid */
2888 : if (!(clone_flags & CLONE_VM)) {
2889 : p->numa_preferred_nid = NUMA_NO_NODE;
2890 : return;
2891 : }
2892 :
2893 : /*
2894 : * New thread, keep existing numa_preferred_nid which should be copied
2895 : * already by arch_dup_task_struct but stagger when scans start.
2896 : */
2897 : if (mm) {
2898 : unsigned int delay;
2899 :
2900 : delay = min_t(unsigned int, task_scan_max(current),
2901 : current->numa_scan_period * mm_users * NSEC_PER_MSEC);
2902 : delay += 2 * TICK_NSEC;
2903 : p->node_stamp = delay;
2904 : }
2905 : }
2906 :
2907 : /*
2908 : * Drive the periodic memory faults..
2909 : */
2910 : static void task_tick_numa(struct rq *rq, struct task_struct *curr)
2911 : {
2912 : struct callback_head *work = &curr->numa_work;
2913 : u64 period, now;
2914 :
2915 : /*
2916 : * We don't care about NUMA placement if we don't have memory.
2917 : */
2918 : if ((curr->flags & (PF_EXITING | PF_KTHREAD)) || work->next != work)
2919 : return;
2920 :
2921 : /*
2922 : * Using runtime rather than walltime has the dual advantage that
2923 : * we (mostly) drive the selection from busy threads and that the
2924 : * task needs to have done some actual work before we bother with
2925 : * NUMA placement.
2926 : */
2927 : now = curr->se.sum_exec_runtime;
2928 : period = (u64)curr->numa_scan_period * NSEC_PER_MSEC;
2929 :
2930 : if (now > curr->node_stamp + period) {
2931 : if (!curr->node_stamp)
2932 : curr->numa_scan_period = task_scan_start(curr);
2933 : curr->node_stamp += period;
2934 :
2935 : if (!time_before(jiffies, curr->mm->numa_next_scan))
2936 : task_work_add(curr, work, TWA_RESUME);
2937 : }
2938 : }
2939 :
2940 : static void update_scan_period(struct task_struct *p, int new_cpu)
2941 : {
2942 : int src_nid = cpu_to_node(task_cpu(p));
2943 : int dst_nid = cpu_to_node(new_cpu);
2944 :
2945 : if (!static_branch_likely(&sched_numa_balancing))
2946 : return;
2947 :
2948 : if (!p->mm || !p->numa_faults || (p->flags & PF_EXITING))
2949 : return;
2950 :
2951 : if (src_nid == dst_nid)
2952 : return;
2953 :
2954 : /*
2955 : * Allow resets if faults have been trapped before one scan
2956 : * has completed. This is most likely due to a new task that
2957 : * is pulled cross-node due to wakeups or load balancing.
2958 : */
2959 : if (p->numa_scan_seq) {
2960 : /*
2961 : * Avoid scan adjustments if moving to the preferred
2962 : * node or if the task was not previously running on
2963 : * the preferred node.
2964 : */
2965 : if (dst_nid == p->numa_preferred_nid ||
2966 : (p->numa_preferred_nid != NUMA_NO_NODE &&
2967 : src_nid != p->numa_preferred_nid))
2968 : return;
2969 : }
2970 :
2971 : p->numa_scan_period = task_scan_start(p);
2972 : }
2973 :
2974 : #else
2975 : static void task_tick_numa(struct rq *rq, struct task_struct *curr)
2976 : {
2977 : }
2978 :
2979 : static inline void account_numa_enqueue(struct rq *rq, struct task_struct *p)
2980 : {
2981 : }
2982 :
2983 : static inline void account_numa_dequeue(struct rq *rq, struct task_struct *p)
2984 : {
2985 : }
2986 :
2987 : static inline void update_scan_period(struct task_struct *p, int new_cpu)
2988 : {
2989 : }
2990 :
2991 : #endif /* CONFIG_NUMA_BALANCING */
2992 :
2993 : static void
2994 : account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
2995 : {
2996 1240 : update_load_add(&cfs_rq->load, se->load.weight);
2997 : #ifdef CONFIG_SMP
2998 : if (entity_is_task(se)) {
2999 : struct rq *rq = rq_of(cfs_rq);
3000 :
3001 : account_numa_enqueue(rq, task_of(se));
3002 : list_add(&se->group_node, &rq->cfs_tasks);
3003 : }
3004 : #endif
3005 620 : cfs_rq->nr_running++;
3006 620 : if (se_is_idle(se))
3007 : cfs_rq->idle_nr_running++;
3008 : }
3009 :
3010 : static void
3011 : account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
3012 : {
3013 1236 : update_load_sub(&cfs_rq->load, se->load.weight);
3014 : #ifdef CONFIG_SMP
3015 : if (entity_is_task(se)) {
3016 : account_numa_dequeue(rq_of(cfs_rq), task_of(se));
3017 : list_del_init(&se->group_node);
3018 : }
3019 : #endif
3020 618 : cfs_rq->nr_running--;
3021 618 : if (se_is_idle(se))
3022 : cfs_rq->idle_nr_running--;
3023 : }
3024 :
3025 : /*
3026 : * Signed add and clamp on underflow.
3027 : *
3028 : * Explicitly do a load-store to ensure the intermediate value never hits
3029 : * memory. This allows lockless observations without ever seeing the negative
3030 : * values.
3031 : */
3032 : #define add_positive(_ptr, _val) do { \
3033 : typeof(_ptr) ptr = (_ptr); \
3034 : typeof(_val) val = (_val); \
3035 : typeof(*ptr) res, var = READ_ONCE(*ptr); \
3036 : \
3037 : res = var + val; \
3038 : \
3039 : if (val < 0 && res > var) \
3040 : res = 0; \
3041 : \
3042 : WRITE_ONCE(*ptr, res); \
3043 : } while (0)
3044 :
3045 : /*
3046 : * Unsigned subtract and clamp on underflow.
3047 : *
3048 : * Explicitly do a load-store to ensure the intermediate value never hits
3049 : * memory. This allows lockless observations without ever seeing the negative
3050 : * values.
3051 : */
3052 : #define sub_positive(_ptr, _val) do { \
3053 : typeof(_ptr) ptr = (_ptr); \
3054 : typeof(*ptr) val = (_val); \
3055 : typeof(*ptr) res, var = READ_ONCE(*ptr); \
3056 : res = var - val; \
3057 : if (res > var) \
3058 : res = 0; \
3059 : WRITE_ONCE(*ptr, res); \
3060 : } while (0)
3061 :
3062 : /*
3063 : * Remove and clamp on negative, from a local variable.
3064 : *
3065 : * A variant of sub_positive(), which does not use explicit load-store
3066 : * and is thus optimized for local variable updates.
3067 : */
3068 : #define lsub_positive(_ptr, _val) do { \
3069 : typeof(_ptr) ptr = (_ptr); \
3070 : *ptr -= min_t(typeof(*ptr), *ptr, _val); \
3071 : } while (0)
3072 :
3073 : #ifdef CONFIG_SMP
3074 : static inline void
3075 : enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
3076 : {
3077 : cfs_rq->avg.load_avg += se->avg.load_avg;
3078 : cfs_rq->avg.load_sum += se_weight(se) * se->avg.load_sum;
3079 : }
3080 :
3081 : static inline void
3082 : dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
3083 : {
3084 : sub_positive(&cfs_rq->avg.load_avg, se->avg.load_avg);
3085 : sub_positive(&cfs_rq->avg.load_sum, se_weight(se) * se->avg.load_sum);
3086 : /* See update_cfs_rq_load_avg() */
3087 : cfs_rq->avg.load_sum = max_t(u32, cfs_rq->avg.load_sum,
3088 : cfs_rq->avg.load_avg * PELT_MIN_DIVIDER);
3089 : }
3090 : #else
3091 : static inline void
3092 : enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { }
3093 : static inline void
3094 : dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { }
3095 : #endif
3096 :
3097 4 : static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se,
3098 : unsigned long weight)
3099 : {
3100 4 : if (se->on_rq) {
3101 : /* commit outstanding execution time */
3102 0 : if (cfs_rq->curr == se)
3103 0 : update_curr(cfs_rq);
3104 0 : update_load_sub(&cfs_rq->load, se->load.weight);
3105 : }
3106 4 : dequeue_load_avg(cfs_rq, se);
3107 :
3108 8 : update_load_set(&se->load, weight);
3109 :
3110 : #ifdef CONFIG_SMP
3111 : do {
3112 : u32 divider = get_pelt_divider(&se->avg);
3113 :
3114 : se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider);
3115 : } while (0);
3116 : #endif
3117 :
3118 4 : enqueue_load_avg(cfs_rq, se);
3119 4 : if (se->on_rq)
3120 0 : update_load_add(&cfs_rq->load, se->load.weight);
3121 :
3122 4 : }
3123 :
3124 4 : void reweight_task(struct task_struct *p, int prio)
3125 : {
3126 4 : struct sched_entity *se = &p->se;
3127 8 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
3128 4 : struct load_weight *load = &se->load;
3129 4 : unsigned long weight = scale_load(sched_prio_to_weight[prio]);
3130 :
3131 4 : reweight_entity(cfs_rq, se, weight);
3132 4 : load->inv_weight = sched_prio_to_wmult[prio];
3133 4 : }
3134 :
3135 : #ifdef CONFIG_FAIR_GROUP_SCHED
3136 : #ifdef CONFIG_SMP
3137 : /*
3138 : * All this does is approximate the hierarchical proportion which includes that
3139 : * global sum we all love to hate.
3140 : *
3141 : * That is, the weight of a group entity, is the proportional share of the
3142 : * group weight based on the group runqueue weights. That is:
3143 : *
3144 : * tg->weight * grq->load.weight
3145 : * ge->load.weight = ----------------------------- (1)
3146 : * \Sum grq->load.weight
3147 : *
3148 : * Now, because computing that sum is prohibitively expensive to compute (been
3149 : * there, done that) we approximate it with this average stuff. The average
3150 : * moves slower and therefore the approximation is cheaper and more stable.
3151 : *
3152 : * So instead of the above, we substitute:
3153 : *
3154 : * grq->load.weight -> grq->avg.load_avg (2)
3155 : *
3156 : * which yields the following:
3157 : *
3158 : * tg->weight * grq->avg.load_avg
3159 : * ge->load.weight = ------------------------------ (3)
3160 : * tg->load_avg
3161 : *
3162 : * Where: tg->load_avg ~= \Sum grq->avg.load_avg
3163 : *
3164 : * That is shares_avg, and it is right (given the approximation (2)).
3165 : *
3166 : * The problem with it is that because the average is slow -- it was designed
3167 : * to be exactly that of course -- this leads to transients in boundary
3168 : * conditions. In specific, the case where the group was idle and we start the
3169 : * one task. It takes time for our CPU's grq->avg.load_avg to build up,
3170 : * yielding bad latency etc..
3171 : *
3172 : * Now, in that special case (1) reduces to:
3173 : *
3174 : * tg->weight * grq->load.weight
3175 : * ge->load.weight = ----------------------------- = tg->weight (4)
3176 : * grp->load.weight
3177 : *
3178 : * That is, the sum collapses because all other CPUs are idle; the UP scenario.
3179 : *
3180 : * So what we do is modify our approximation (3) to approach (4) in the (near)
3181 : * UP case, like:
3182 : *
3183 : * ge->load.weight =
3184 : *
3185 : * tg->weight * grq->load.weight
3186 : * --------------------------------------------------- (5)
3187 : * tg->load_avg - grq->avg.load_avg + grq->load.weight
3188 : *
3189 : * But because grq->load.weight can drop to 0, resulting in a divide by zero,
3190 : * we need to use grq->avg.load_avg as its lower bound, which then gives:
3191 : *
3192 : *
3193 : * tg->weight * grq->load.weight
3194 : * ge->load.weight = ----------------------------- (6)
3195 : * tg_load_avg'
3196 : *
3197 : * Where:
3198 : *
3199 : * tg_load_avg' = tg->load_avg - grq->avg.load_avg +
3200 : * max(grq->load.weight, grq->avg.load_avg)
3201 : *
3202 : * And that is shares_weight and is icky. In the (near) UP case it approaches
3203 : * (4) while in the normal case it approaches (3). It consistently
3204 : * overestimates the ge->load.weight and therefore:
3205 : *
3206 : * \Sum ge->load.weight >= tg->weight
3207 : *
3208 : * hence icky!
3209 : */
3210 : static long calc_group_shares(struct cfs_rq *cfs_rq)
3211 : {
3212 : long tg_weight, tg_shares, load, shares;
3213 : struct task_group *tg = cfs_rq->tg;
3214 :
3215 : tg_shares = READ_ONCE(tg->shares);
3216 :
3217 : load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg);
3218 :
3219 : tg_weight = atomic_long_read(&tg->load_avg);
3220 :
3221 : /* Ensure tg_weight >= load */
3222 : tg_weight -= cfs_rq->tg_load_avg_contrib;
3223 : tg_weight += load;
3224 :
3225 : shares = (tg_shares * load);
3226 : if (tg_weight)
3227 : shares /= tg_weight;
3228 :
3229 : /*
3230 : * MIN_SHARES has to be unscaled here to support per-CPU partitioning
3231 : * of a group with small tg->shares value. It is a floor value which is
3232 : * assigned as a minimum load.weight to the sched_entity representing
3233 : * the group on a CPU.
3234 : *
3235 : * E.g. on 64-bit for a group with tg->shares of scale_load(15)=15*1024
3236 : * on an 8-core system with 8 tasks each runnable on one CPU shares has
3237 : * to be 15*1024*1/8=1920 instead of scale_load(MIN_SHARES)=2*1024. In
3238 : * case no task is runnable on a CPU MIN_SHARES=2 should be returned
3239 : * instead of 0.
3240 : */
3241 : return clamp_t(long, shares, MIN_SHARES, tg_shares);
3242 : }
3243 : #endif /* CONFIG_SMP */
3244 :
3245 : static inline int throttled_hierarchy(struct cfs_rq *cfs_rq);
3246 :
3247 : /*
3248 : * Recomputes the group entity based on the current state of its group
3249 : * runqueue.
3250 : */
3251 : static void update_cfs_group(struct sched_entity *se)
3252 : {
3253 : struct cfs_rq *gcfs_rq = group_cfs_rq(se);
3254 : long shares;
3255 :
3256 : if (!gcfs_rq)
3257 : return;
3258 :
3259 : if (throttled_hierarchy(gcfs_rq))
3260 : return;
3261 :
3262 : #ifndef CONFIG_SMP
3263 : shares = READ_ONCE(gcfs_rq->tg->shares);
3264 :
3265 : if (likely(se->load.weight == shares))
3266 : return;
3267 : #else
3268 : shares = calc_group_shares(gcfs_rq);
3269 : #endif
3270 :
3271 : reweight_entity(cfs_rq_of(se), se, shares);
3272 : }
3273 :
3274 : #else /* CONFIG_FAIR_GROUP_SCHED */
3275 : static inline void update_cfs_group(struct sched_entity *se)
3276 : {
3277 : }
3278 : #endif /* CONFIG_FAIR_GROUP_SCHED */
3279 :
3280 : static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq, int flags)
3281 : {
3282 1241 : struct rq *rq = rq_of(cfs_rq);
3283 :
3284 : if (&rq->cfs == cfs_rq) {
3285 : /*
3286 : * There are a few boundary cases this might miss but it should
3287 : * get called often enough that that should (hopefully) not be
3288 : * a real problem.
3289 : *
3290 : * It will not get called when we go idle, because the idle
3291 : * thread is a different class (!fair), nor will the utilization
3292 : * number include things like RT tasks.
3293 : *
3294 : * As is, the util number is not freq-invariant (we'd have to
3295 : * implement arch_scale_freq_capacity() for that).
3296 : *
3297 : * See cpu_util_cfs().
3298 : */
3299 : cpufreq_update_util(rq, flags);
3300 : }
3301 : }
3302 :
3303 : #ifdef CONFIG_SMP
3304 : #ifdef CONFIG_FAIR_GROUP_SCHED
3305 : /*
3306 : * Because list_add_leaf_cfs_rq always places a child cfs_rq on the list
3307 : * immediately before a parent cfs_rq, and cfs_rqs are removed from the list
3308 : * bottom-up, we only have to test whether the cfs_rq before us on the list
3309 : * is our child.
3310 : * If cfs_rq is not on the list, test whether a child needs its to be added to
3311 : * connect a branch to the tree * (see list_add_leaf_cfs_rq() for details).
3312 : */
3313 : static inline bool child_cfs_rq_on_list(struct cfs_rq *cfs_rq)
3314 : {
3315 : struct cfs_rq *prev_cfs_rq;
3316 : struct list_head *prev;
3317 :
3318 : if (cfs_rq->on_list) {
3319 : prev = cfs_rq->leaf_cfs_rq_list.prev;
3320 : } else {
3321 : struct rq *rq = rq_of(cfs_rq);
3322 :
3323 : prev = rq->tmp_alone_branch;
3324 : }
3325 :
3326 : prev_cfs_rq = container_of(prev, struct cfs_rq, leaf_cfs_rq_list);
3327 :
3328 : return (prev_cfs_rq->tg->parent == cfs_rq->tg);
3329 : }
3330 :
3331 : static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq)
3332 : {
3333 : if (cfs_rq->load.weight)
3334 : return false;
3335 :
3336 : if (cfs_rq->avg.load_sum)
3337 : return false;
3338 :
3339 : if (cfs_rq->avg.util_sum)
3340 : return false;
3341 :
3342 : if (cfs_rq->avg.runnable_sum)
3343 : return false;
3344 :
3345 : if (child_cfs_rq_on_list(cfs_rq))
3346 : return false;
3347 :
3348 : /*
3349 : * _avg must be null when _sum are null because _avg = _sum / divider
3350 : * Make sure that rounding and/or propagation of PELT values never
3351 : * break this.
3352 : */
3353 : SCHED_WARN_ON(cfs_rq->avg.load_avg ||
3354 : cfs_rq->avg.util_avg ||
3355 : cfs_rq->avg.runnable_avg);
3356 :
3357 : return true;
3358 : }
3359 :
3360 : /**
3361 : * update_tg_load_avg - update the tg's load avg
3362 : * @cfs_rq: the cfs_rq whose avg changed
3363 : *
3364 : * This function 'ensures': tg->load_avg := \Sum tg->cfs_rq[]->avg.load.
3365 : * However, because tg->load_avg is a global value there are performance
3366 : * considerations.
3367 : *
3368 : * In order to avoid having to look at the other cfs_rq's, we use a
3369 : * differential update where we store the last value we propagated. This in
3370 : * turn allows skipping updates if the differential is 'small'.
3371 : *
3372 : * Updating tg's load_avg is necessary before update_cfs_share().
3373 : */
3374 : static inline void update_tg_load_avg(struct cfs_rq *cfs_rq)
3375 : {
3376 : long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib;
3377 :
3378 : /*
3379 : * No need to update load_avg for root_task_group as it is not used.
3380 : */
3381 : if (cfs_rq->tg == &root_task_group)
3382 : return;
3383 :
3384 : if (abs(delta) > cfs_rq->tg_load_avg_contrib / 64) {
3385 : atomic_long_add(delta, &cfs_rq->tg->load_avg);
3386 : cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg;
3387 : }
3388 : }
3389 :
3390 : /*
3391 : * Called within set_task_rq() right before setting a task's CPU. The
3392 : * caller only guarantees p->pi_lock is held; no other assumptions,
3393 : * including the state of rq->lock, should be made.
3394 : */
3395 : void set_task_rq_fair(struct sched_entity *se,
3396 : struct cfs_rq *prev, struct cfs_rq *next)
3397 : {
3398 : u64 p_last_update_time;
3399 : u64 n_last_update_time;
3400 :
3401 : if (!sched_feat(ATTACH_AGE_LOAD))
3402 : return;
3403 :
3404 : /*
3405 : * We are supposed to update the task to "current" time, then its up to
3406 : * date and ready to go to new CPU/cfs_rq. But we have difficulty in
3407 : * getting what current time is, so simply throw away the out-of-date
3408 : * time. This will result in the wakee task is less decayed, but giving
3409 : * the wakee more load sounds not bad.
3410 : */
3411 : if (!(se->avg.last_update_time && prev))
3412 : return;
3413 :
3414 : #ifndef CONFIG_64BIT
3415 : {
3416 : u64 p_last_update_time_copy;
3417 : u64 n_last_update_time_copy;
3418 :
3419 : do {
3420 : p_last_update_time_copy = prev->load_last_update_time_copy;
3421 : n_last_update_time_copy = next->load_last_update_time_copy;
3422 :
3423 : smp_rmb();
3424 :
3425 : p_last_update_time = prev->avg.last_update_time;
3426 : n_last_update_time = next->avg.last_update_time;
3427 :
3428 : } while (p_last_update_time != p_last_update_time_copy ||
3429 : n_last_update_time != n_last_update_time_copy);
3430 : }
3431 : #else
3432 : p_last_update_time = prev->avg.last_update_time;
3433 : n_last_update_time = next->avg.last_update_time;
3434 : #endif
3435 : __update_load_avg_blocked_se(p_last_update_time, se);
3436 : se->avg.last_update_time = n_last_update_time;
3437 : }
3438 :
3439 : /*
3440 : * When on migration a sched_entity joins/leaves the PELT hierarchy, we need to
3441 : * propagate its contribution. The key to this propagation is the invariant
3442 : * that for each group:
3443 : *
3444 : * ge->avg == grq->avg (1)
3445 : *
3446 : * _IFF_ we look at the pure running and runnable sums. Because they
3447 : * represent the very same entity, just at different points in the hierarchy.
3448 : *
3449 : * Per the above update_tg_cfs_util() and update_tg_cfs_runnable() are trivial
3450 : * and simply copies the running/runnable sum over (but still wrong, because
3451 : * the group entity and group rq do not have their PELT windows aligned).
3452 : *
3453 : * However, update_tg_cfs_load() is more complex. So we have:
3454 : *
3455 : * ge->avg.load_avg = ge->load.weight * ge->avg.runnable_avg (2)
3456 : *
3457 : * And since, like util, the runnable part should be directly transferable,
3458 : * the following would _appear_ to be the straight forward approach:
3459 : *
3460 : * grq->avg.load_avg = grq->load.weight * grq->avg.runnable_avg (3)
3461 : *
3462 : * And per (1) we have:
3463 : *
3464 : * ge->avg.runnable_avg == grq->avg.runnable_avg
3465 : *
3466 : * Which gives:
3467 : *
3468 : * ge->load.weight * grq->avg.load_avg
3469 : * ge->avg.load_avg = ----------------------------------- (4)
3470 : * grq->load.weight
3471 : *
3472 : * Except that is wrong!
3473 : *
3474 : * Because while for entities historical weight is not important and we
3475 : * really only care about our future and therefore can consider a pure
3476 : * runnable sum, runqueues can NOT do this.
3477 : *
3478 : * We specifically want runqueues to have a load_avg that includes
3479 : * historical weights. Those represent the blocked load, the load we expect
3480 : * to (shortly) return to us. This only works by keeping the weights as
3481 : * integral part of the sum. We therefore cannot decompose as per (3).
3482 : *
3483 : * Another reason this doesn't work is that runnable isn't a 0-sum entity.
3484 : * Imagine a rq with 2 tasks that each are runnable 2/3 of the time. Then the
3485 : * rq itself is runnable anywhere between 2/3 and 1 depending on how the
3486 : * runnable section of these tasks overlap (or not). If they were to perfectly
3487 : * align the rq as a whole would be runnable 2/3 of the time. If however we
3488 : * always have at least 1 runnable task, the rq as a whole is always runnable.
3489 : *
3490 : * So we'll have to approximate.. :/
3491 : *
3492 : * Given the constraint:
3493 : *
3494 : * ge->avg.running_sum <= ge->avg.runnable_sum <= LOAD_AVG_MAX
3495 : *
3496 : * We can construct a rule that adds runnable to a rq by assuming minimal
3497 : * overlap.
3498 : *
3499 : * On removal, we'll assume each task is equally runnable; which yields:
3500 : *
3501 : * grq->avg.runnable_sum = grq->avg.load_sum / grq->load.weight
3502 : *
3503 : * XXX: only do this for the part of runnable > running ?
3504 : *
3505 : */
3506 : static inline void
3507 : update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq)
3508 : {
3509 : long delta_sum, delta_avg = gcfs_rq->avg.util_avg - se->avg.util_avg;
3510 : u32 new_sum, divider;
3511 :
3512 : /* Nothing to update */
3513 : if (!delta_avg)
3514 : return;
3515 :
3516 : /*
3517 : * cfs_rq->avg.period_contrib can be used for both cfs_rq and se.
3518 : * See ___update_load_avg() for details.
3519 : */
3520 : divider = get_pelt_divider(&cfs_rq->avg);
3521 :
3522 :
3523 : /* Set new sched_entity's utilization */
3524 : se->avg.util_avg = gcfs_rq->avg.util_avg;
3525 : new_sum = se->avg.util_avg * divider;
3526 : delta_sum = (long)new_sum - (long)se->avg.util_sum;
3527 : se->avg.util_sum = new_sum;
3528 :
3529 : /* Update parent cfs_rq utilization */
3530 : add_positive(&cfs_rq->avg.util_avg, delta_avg);
3531 : add_positive(&cfs_rq->avg.util_sum, delta_sum);
3532 :
3533 : /* See update_cfs_rq_load_avg() */
3534 : cfs_rq->avg.util_sum = max_t(u32, cfs_rq->avg.util_sum,
3535 : cfs_rq->avg.util_avg * PELT_MIN_DIVIDER);
3536 : }
3537 :
3538 : static inline void
3539 : update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq)
3540 : {
3541 : long delta_sum, delta_avg = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg;
3542 : u32 new_sum, divider;
3543 :
3544 : /* Nothing to update */
3545 : if (!delta_avg)
3546 : return;
3547 :
3548 : /*
3549 : * cfs_rq->avg.period_contrib can be used for both cfs_rq and se.
3550 : * See ___update_load_avg() for details.
3551 : */
3552 : divider = get_pelt_divider(&cfs_rq->avg);
3553 :
3554 : /* Set new sched_entity's runnable */
3555 : se->avg.runnable_avg = gcfs_rq->avg.runnable_avg;
3556 : new_sum = se->avg.runnable_avg * divider;
3557 : delta_sum = (long)new_sum - (long)se->avg.runnable_sum;
3558 : se->avg.runnable_sum = new_sum;
3559 :
3560 : /* Update parent cfs_rq runnable */
3561 : add_positive(&cfs_rq->avg.runnable_avg, delta_avg);
3562 : add_positive(&cfs_rq->avg.runnable_sum, delta_sum);
3563 : /* See update_cfs_rq_load_avg() */
3564 : cfs_rq->avg.runnable_sum = max_t(u32, cfs_rq->avg.runnable_sum,
3565 : cfs_rq->avg.runnable_avg * PELT_MIN_DIVIDER);
3566 : }
3567 :
3568 : static inline void
3569 : update_tg_cfs_load(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq)
3570 : {
3571 : long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum;
3572 : unsigned long load_avg;
3573 : u64 load_sum = 0;
3574 : s64 delta_sum;
3575 : u32 divider;
3576 :
3577 : if (!runnable_sum)
3578 : return;
3579 :
3580 : gcfs_rq->prop_runnable_sum = 0;
3581 :
3582 : /*
3583 : * cfs_rq->avg.period_contrib can be used for both cfs_rq and se.
3584 : * See ___update_load_avg() for details.
3585 : */
3586 : divider = get_pelt_divider(&cfs_rq->avg);
3587 :
3588 : if (runnable_sum >= 0) {
3589 : /*
3590 : * Add runnable; clip at LOAD_AVG_MAX. Reflects that until
3591 : * the CPU is saturated running == runnable.
3592 : */
3593 : runnable_sum += se->avg.load_sum;
3594 : runnable_sum = min_t(long, runnable_sum, divider);
3595 : } else {
3596 : /*
3597 : * Estimate the new unweighted runnable_sum of the gcfs_rq by
3598 : * assuming all tasks are equally runnable.
3599 : */
3600 : if (scale_load_down(gcfs_rq->load.weight)) {
3601 : load_sum = div_u64(gcfs_rq->avg.load_sum,
3602 : scale_load_down(gcfs_rq->load.weight));
3603 : }
3604 :
3605 : /* But make sure to not inflate se's runnable */
3606 : runnable_sum = min(se->avg.load_sum, load_sum);
3607 : }
3608 :
3609 : /*
3610 : * runnable_sum can't be lower than running_sum
3611 : * Rescale running sum to be in the same range as runnable sum
3612 : * running_sum is in [0 : LOAD_AVG_MAX << SCHED_CAPACITY_SHIFT]
3613 : * runnable_sum is in [0 : LOAD_AVG_MAX]
3614 : */
3615 : running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT;
3616 : runnable_sum = max(runnable_sum, running_sum);
3617 :
3618 : load_sum = se_weight(se) * runnable_sum;
3619 : load_avg = div_u64(load_sum, divider);
3620 :
3621 : delta_avg = load_avg - se->avg.load_avg;
3622 : if (!delta_avg)
3623 : return;
3624 :
3625 : delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum;
3626 :
3627 : se->avg.load_sum = runnable_sum;
3628 : se->avg.load_avg = load_avg;
3629 : add_positive(&cfs_rq->avg.load_avg, delta_avg);
3630 : add_positive(&cfs_rq->avg.load_sum, delta_sum);
3631 : /* See update_cfs_rq_load_avg() */
3632 : cfs_rq->avg.load_sum = max_t(u32, cfs_rq->avg.load_sum,
3633 : cfs_rq->avg.load_avg * PELT_MIN_DIVIDER);
3634 : }
3635 :
3636 : static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum)
3637 : {
3638 : cfs_rq->propagate = 1;
3639 : cfs_rq->prop_runnable_sum += runnable_sum;
3640 : }
3641 :
3642 : /* Update task and its cfs_rq load average */
3643 : static inline int propagate_entity_load_avg(struct sched_entity *se)
3644 : {
3645 : struct cfs_rq *cfs_rq, *gcfs_rq;
3646 :
3647 : if (entity_is_task(se))
3648 : return 0;
3649 :
3650 : gcfs_rq = group_cfs_rq(se);
3651 : if (!gcfs_rq->propagate)
3652 : return 0;
3653 :
3654 : gcfs_rq->propagate = 0;
3655 :
3656 : cfs_rq = cfs_rq_of(se);
3657 :
3658 : add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum);
3659 :
3660 : update_tg_cfs_util(cfs_rq, se, gcfs_rq);
3661 : update_tg_cfs_runnable(cfs_rq, se, gcfs_rq);
3662 : update_tg_cfs_load(cfs_rq, se, gcfs_rq);
3663 :
3664 : trace_pelt_cfs_tp(cfs_rq);
3665 : trace_pelt_se_tp(se);
3666 :
3667 : return 1;
3668 : }
3669 :
3670 : /*
3671 : * Check if we need to update the load and the utilization of a blocked
3672 : * group_entity:
3673 : */
3674 : static inline bool skip_blocked_update(struct sched_entity *se)
3675 : {
3676 : struct cfs_rq *gcfs_rq = group_cfs_rq(se);
3677 :
3678 : /*
3679 : * If sched_entity still have not zero load or utilization, we have to
3680 : * decay it:
3681 : */
3682 : if (se->avg.load_avg || se->avg.util_avg)
3683 : return false;
3684 :
3685 : /*
3686 : * If there is a pending propagation, we have to update the load and
3687 : * the utilization of the sched_entity:
3688 : */
3689 : if (gcfs_rq->propagate)
3690 : return false;
3691 :
3692 : /*
3693 : * Otherwise, the load and the utilization of the sched_entity is
3694 : * already zero and there is no pending propagation, so it will be a
3695 : * waste of time to try to decay it:
3696 : */
3697 : return true;
3698 : }
3699 :
3700 : #else /* CONFIG_FAIR_GROUP_SCHED */
3701 :
3702 : static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) {}
3703 :
3704 : static inline int propagate_entity_load_avg(struct sched_entity *se)
3705 : {
3706 : return 0;
3707 : }
3708 :
3709 : static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) {}
3710 :
3711 : #endif /* CONFIG_FAIR_GROUP_SCHED */
3712 :
3713 : /**
3714 : * update_cfs_rq_load_avg - update the cfs_rq's load/util averages
3715 : * @now: current time, as per cfs_rq_clock_pelt()
3716 : * @cfs_rq: cfs_rq to update
3717 : *
3718 : * The cfs_rq avg is the direct sum of all its entities (blocked and runnable)
3719 : * avg. The immediate corollary is that all (fair) tasks must be attached, see
3720 : * post_init_entity_util_avg().
3721 : *
3722 : * cfs_rq->avg is used for task_h_load() and update_cfs_share() for example.
3723 : *
3724 : * Return: true if the load decayed or we removed load.
3725 : *
3726 : * Since both these conditions indicate a changed cfs_rq->avg.load we should
3727 : * call update_tg_load_avg() when this function returns true.
3728 : */
3729 : static inline int
3730 : update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq)
3731 : {
3732 : unsigned long removed_load = 0, removed_util = 0, removed_runnable = 0;
3733 : struct sched_avg *sa = &cfs_rq->avg;
3734 : int decayed = 0;
3735 :
3736 : if (cfs_rq->removed.nr) {
3737 : unsigned long r;
3738 : u32 divider = get_pelt_divider(&cfs_rq->avg);
3739 :
3740 : raw_spin_lock(&cfs_rq->removed.lock);
3741 : swap(cfs_rq->removed.util_avg, removed_util);
3742 : swap(cfs_rq->removed.load_avg, removed_load);
3743 : swap(cfs_rq->removed.runnable_avg, removed_runnable);
3744 : cfs_rq->removed.nr = 0;
3745 : raw_spin_unlock(&cfs_rq->removed.lock);
3746 :
3747 : r = removed_load;
3748 : sub_positive(&sa->load_avg, r);
3749 : sub_positive(&sa->load_sum, r * divider);
3750 : /* See sa->util_sum below */
3751 : sa->load_sum = max_t(u32, sa->load_sum, sa->load_avg * PELT_MIN_DIVIDER);
3752 :
3753 : r = removed_util;
3754 : sub_positive(&sa->util_avg, r);
3755 : sub_positive(&sa->util_sum, r * divider);
3756 : /*
3757 : * Because of rounding, se->util_sum might ends up being +1 more than
3758 : * cfs->util_sum. Although this is not a problem by itself, detaching
3759 : * a lot of tasks with the rounding problem between 2 updates of
3760 : * util_avg (~1ms) can make cfs->util_sum becoming null whereas
3761 : * cfs_util_avg is not.
3762 : * Check that util_sum is still above its lower bound for the new
3763 : * util_avg. Given that period_contrib might have moved since the last
3764 : * sync, we are only sure that util_sum must be above or equal to
3765 : * util_avg * minimum possible divider
3766 : */
3767 : sa->util_sum = max_t(u32, sa->util_sum, sa->util_avg * PELT_MIN_DIVIDER);
3768 :
3769 : r = removed_runnable;
3770 : sub_positive(&sa->runnable_avg, r);
3771 : sub_positive(&sa->runnable_sum, r * divider);
3772 : /* See sa->util_sum above */
3773 : sa->runnable_sum = max_t(u32, sa->runnable_sum,
3774 : sa->runnable_avg * PELT_MIN_DIVIDER);
3775 :
3776 : /*
3777 : * removed_runnable is the unweighted version of removed_load so we
3778 : * can use it to estimate removed_load_sum.
3779 : */
3780 : add_tg_cfs_propagate(cfs_rq,
3781 : -(long)(removed_runnable * divider) >> SCHED_CAPACITY_SHIFT);
3782 :
3783 : decayed = 1;
3784 : }
3785 :
3786 : decayed |= __update_load_avg_cfs_rq(now, cfs_rq);
3787 :
3788 : #ifndef CONFIG_64BIT
3789 : smp_wmb();
3790 : cfs_rq->load_last_update_time_copy = sa->last_update_time;
3791 : #endif
3792 :
3793 : return decayed;
3794 : }
3795 :
3796 : /**
3797 : * attach_entity_load_avg - attach this entity to its cfs_rq load avg
3798 : * @cfs_rq: cfs_rq to attach to
3799 : * @se: sched_entity to attach
3800 : *
3801 : * Must call update_cfs_rq_load_avg() before this, since we rely on
3802 : * cfs_rq->avg.last_update_time being current.
3803 : */
3804 : static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
3805 : {
3806 : /*
3807 : * cfs_rq->avg.period_contrib can be used for both cfs_rq and se.
3808 : * See ___update_load_avg() for details.
3809 : */
3810 : u32 divider = get_pelt_divider(&cfs_rq->avg);
3811 :
3812 : /*
3813 : * When we attach the @se to the @cfs_rq, we must align the decay
3814 : * window because without that, really weird and wonderful things can
3815 : * happen.
3816 : *
3817 : * XXX illustrate
3818 : */
3819 : se->avg.last_update_time = cfs_rq->avg.last_update_time;
3820 : se->avg.period_contrib = cfs_rq->avg.period_contrib;
3821 :
3822 : /*
3823 : * Hell(o) Nasty stuff.. we need to recompute _sum based on the new
3824 : * period_contrib. This isn't strictly correct, but since we're
3825 : * entirely outside of the PELT hierarchy, nobody cares if we truncate
3826 : * _sum a little.
3827 : */
3828 : se->avg.util_sum = se->avg.util_avg * divider;
3829 :
3830 : se->avg.runnable_sum = se->avg.runnable_avg * divider;
3831 :
3832 : se->avg.load_sum = se->avg.load_avg * divider;
3833 : if (se_weight(se) < se->avg.load_sum)
3834 : se->avg.load_sum = div_u64(se->avg.load_sum, se_weight(se));
3835 : else
3836 : se->avg.load_sum = 1;
3837 :
3838 : enqueue_load_avg(cfs_rq, se);
3839 : cfs_rq->avg.util_avg += se->avg.util_avg;
3840 : cfs_rq->avg.util_sum += se->avg.util_sum;
3841 : cfs_rq->avg.runnable_avg += se->avg.runnable_avg;
3842 : cfs_rq->avg.runnable_sum += se->avg.runnable_sum;
3843 :
3844 : add_tg_cfs_propagate(cfs_rq, se->avg.load_sum);
3845 :
3846 : cfs_rq_util_change(cfs_rq, 0);
3847 :
3848 : trace_pelt_cfs_tp(cfs_rq);
3849 : }
3850 :
3851 : /**
3852 : * detach_entity_load_avg - detach this entity from its cfs_rq load avg
3853 : * @cfs_rq: cfs_rq to detach from
3854 : * @se: sched_entity to detach
3855 : *
3856 : * Must call update_cfs_rq_load_avg() before this, since we rely on
3857 : * cfs_rq->avg.last_update_time being current.
3858 : */
3859 : static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
3860 : {
3861 : dequeue_load_avg(cfs_rq, se);
3862 : sub_positive(&cfs_rq->avg.util_avg, se->avg.util_avg);
3863 : sub_positive(&cfs_rq->avg.util_sum, se->avg.util_sum);
3864 : /* See update_cfs_rq_load_avg() */
3865 : cfs_rq->avg.util_sum = max_t(u32, cfs_rq->avg.util_sum,
3866 : cfs_rq->avg.util_avg * PELT_MIN_DIVIDER);
3867 :
3868 : sub_positive(&cfs_rq->avg.runnable_avg, se->avg.runnable_avg);
3869 : sub_positive(&cfs_rq->avg.runnable_sum, se->avg.runnable_sum);
3870 : /* See update_cfs_rq_load_avg() */
3871 : cfs_rq->avg.runnable_sum = max_t(u32, cfs_rq->avg.runnable_sum,
3872 : cfs_rq->avg.runnable_avg * PELT_MIN_DIVIDER);
3873 :
3874 : add_tg_cfs_propagate(cfs_rq, -se->avg.load_sum);
3875 :
3876 : cfs_rq_util_change(cfs_rq, 0);
3877 :
3878 : trace_pelt_cfs_tp(cfs_rq);
3879 : }
3880 :
3881 : /*
3882 : * Optional action to be done while updating the load average
3883 : */
3884 : #define UPDATE_TG 0x1
3885 : #define SKIP_AGE_LOAD 0x2
3886 : #define DO_ATTACH 0x4
3887 :
3888 : /* Update task and its cfs_rq load average */
3889 : static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
3890 : {
3891 : u64 now = cfs_rq_clock_pelt(cfs_rq);
3892 : int decayed;
3893 :
3894 : /*
3895 : * Track task load average for carrying it to new CPU after migrated, and
3896 : * track group sched_entity load average for task_h_load calc in migration
3897 : */
3898 : if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD))
3899 : __update_load_avg_se(now, cfs_rq, se);
3900 :
3901 : decayed = update_cfs_rq_load_avg(now, cfs_rq);
3902 : decayed |= propagate_entity_load_avg(se);
3903 :
3904 : if (!se->avg.last_update_time && (flags & DO_ATTACH)) {
3905 :
3906 : /*
3907 : * DO_ATTACH means we're here from enqueue_entity().
3908 : * !last_update_time means we've passed through
3909 : * migrate_task_rq_fair() indicating we migrated.
3910 : *
3911 : * IOW we're enqueueing a task on a new CPU.
3912 : */
3913 : attach_entity_load_avg(cfs_rq, se);
3914 : update_tg_load_avg(cfs_rq);
3915 :
3916 : } else if (decayed) {
3917 : cfs_rq_util_change(cfs_rq, 0);
3918 :
3919 : if (flags & UPDATE_TG)
3920 : update_tg_load_avg(cfs_rq);
3921 : }
3922 : }
3923 :
3924 : #ifndef CONFIG_64BIT
3925 : static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq)
3926 : {
3927 : u64 last_update_time_copy;
3928 : u64 last_update_time;
3929 :
3930 : do {
3931 : last_update_time_copy = cfs_rq->load_last_update_time_copy;
3932 : smp_rmb();
3933 : last_update_time = cfs_rq->avg.last_update_time;
3934 : } while (last_update_time != last_update_time_copy);
3935 :
3936 : return last_update_time;
3937 : }
3938 : #else
3939 : static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq)
3940 : {
3941 : return cfs_rq->avg.last_update_time;
3942 : }
3943 : #endif
3944 :
3945 : /*
3946 : * Synchronize entity load avg of dequeued entity without locking
3947 : * the previous rq.
3948 : */
3949 : static void sync_entity_load_avg(struct sched_entity *se)
3950 : {
3951 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
3952 : u64 last_update_time;
3953 :
3954 : last_update_time = cfs_rq_last_update_time(cfs_rq);
3955 : __update_load_avg_blocked_se(last_update_time, se);
3956 : }
3957 :
3958 : /*
3959 : * Task first catches up with cfs_rq, and then subtract
3960 : * itself from the cfs_rq (task must be off the queue now).
3961 : */
3962 : static void remove_entity_load_avg(struct sched_entity *se)
3963 : {
3964 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
3965 : unsigned long flags;
3966 :
3967 : /*
3968 : * tasks cannot exit without having gone through wake_up_new_task() ->
3969 : * post_init_entity_util_avg() which will have added things to the
3970 : * cfs_rq, so we can remove unconditionally.
3971 : */
3972 :
3973 : sync_entity_load_avg(se);
3974 :
3975 : raw_spin_lock_irqsave(&cfs_rq->removed.lock, flags);
3976 : ++cfs_rq->removed.nr;
3977 : cfs_rq->removed.util_avg += se->avg.util_avg;
3978 : cfs_rq->removed.load_avg += se->avg.load_avg;
3979 : cfs_rq->removed.runnable_avg += se->avg.runnable_avg;
3980 : raw_spin_unlock_irqrestore(&cfs_rq->removed.lock, flags);
3981 : }
3982 :
3983 : static inline unsigned long cfs_rq_runnable_avg(struct cfs_rq *cfs_rq)
3984 : {
3985 : return cfs_rq->avg.runnable_avg;
3986 : }
3987 :
3988 : static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq)
3989 : {
3990 : return cfs_rq->avg.load_avg;
3991 : }
3992 :
3993 : static int newidle_balance(struct rq *this_rq, struct rq_flags *rf);
3994 :
3995 : static inline unsigned long task_util(struct task_struct *p)
3996 : {
3997 : return READ_ONCE(p->se.avg.util_avg);
3998 : }
3999 :
4000 : static inline unsigned long _task_util_est(struct task_struct *p)
4001 : {
4002 : struct util_est ue = READ_ONCE(p->se.avg.util_est);
4003 :
4004 : return max(ue.ewma, (ue.enqueued & ~UTIL_AVG_UNCHANGED));
4005 : }
4006 :
4007 : static inline unsigned long task_util_est(struct task_struct *p)
4008 : {
4009 : return max(task_util(p), _task_util_est(p));
4010 : }
4011 :
4012 : #ifdef CONFIG_UCLAMP_TASK
4013 : static inline unsigned long uclamp_task_util(struct task_struct *p)
4014 : {
4015 : return clamp(task_util_est(p),
4016 : uclamp_eff_value(p, UCLAMP_MIN),
4017 : uclamp_eff_value(p, UCLAMP_MAX));
4018 : }
4019 : #else
4020 : static inline unsigned long uclamp_task_util(struct task_struct *p)
4021 : {
4022 : return task_util_est(p);
4023 : }
4024 : #endif
4025 :
4026 : static inline void util_est_enqueue(struct cfs_rq *cfs_rq,
4027 : struct task_struct *p)
4028 : {
4029 : unsigned int enqueued;
4030 :
4031 : if (!sched_feat(UTIL_EST))
4032 : return;
4033 :
4034 : /* Update root cfs_rq's estimated utilization */
4035 : enqueued = cfs_rq->avg.util_est.enqueued;
4036 : enqueued += _task_util_est(p);
4037 : WRITE_ONCE(cfs_rq->avg.util_est.enqueued, enqueued);
4038 :
4039 : trace_sched_util_est_cfs_tp(cfs_rq);
4040 : }
4041 :
4042 : static inline void util_est_dequeue(struct cfs_rq *cfs_rq,
4043 : struct task_struct *p)
4044 : {
4045 : unsigned int enqueued;
4046 :
4047 : if (!sched_feat(UTIL_EST))
4048 : return;
4049 :
4050 : /* Update root cfs_rq's estimated utilization */
4051 : enqueued = cfs_rq->avg.util_est.enqueued;
4052 : enqueued -= min_t(unsigned int, enqueued, _task_util_est(p));
4053 : WRITE_ONCE(cfs_rq->avg.util_est.enqueued, enqueued);
4054 :
4055 : trace_sched_util_est_cfs_tp(cfs_rq);
4056 : }
4057 :
4058 : #define UTIL_EST_MARGIN (SCHED_CAPACITY_SCALE / 100)
4059 :
4060 : /*
4061 : * Check if a (signed) value is within a specified (unsigned) margin,
4062 : * based on the observation that:
4063 : *
4064 : * abs(x) < y := (unsigned)(x + y - 1) < (2 * y - 1)
4065 : *
4066 : * NOTE: this only works when value + margin < INT_MAX.
4067 : */
4068 : static inline bool within_margin(int value, int margin)
4069 : {
4070 : return ((unsigned int)(value + margin - 1) < (2 * margin - 1));
4071 : }
4072 :
4073 : static inline void util_est_update(struct cfs_rq *cfs_rq,
4074 : struct task_struct *p,
4075 : bool task_sleep)
4076 : {
4077 : long last_ewma_diff, last_enqueued_diff;
4078 : struct util_est ue;
4079 :
4080 : if (!sched_feat(UTIL_EST))
4081 : return;
4082 :
4083 : /*
4084 : * Skip update of task's estimated utilization when the task has not
4085 : * yet completed an activation, e.g. being migrated.
4086 : */
4087 : if (!task_sleep)
4088 : return;
4089 :
4090 : /*
4091 : * If the PELT values haven't changed since enqueue time,
4092 : * skip the util_est update.
4093 : */
4094 : ue = p->se.avg.util_est;
4095 : if (ue.enqueued & UTIL_AVG_UNCHANGED)
4096 : return;
4097 :
4098 : last_enqueued_diff = ue.enqueued;
4099 :
4100 : /*
4101 : * Reset EWMA on utilization increases, the moving average is used only
4102 : * to smooth utilization decreases.
4103 : */
4104 : ue.enqueued = task_util(p);
4105 : if (sched_feat(UTIL_EST_FASTUP)) {
4106 : if (ue.ewma < ue.enqueued) {
4107 : ue.ewma = ue.enqueued;
4108 : goto done;
4109 : }
4110 : }
4111 :
4112 : /*
4113 : * Skip update of task's estimated utilization when its members are
4114 : * already ~1% close to its last activation value.
4115 : */
4116 : last_ewma_diff = ue.enqueued - ue.ewma;
4117 : last_enqueued_diff -= ue.enqueued;
4118 : if (within_margin(last_ewma_diff, UTIL_EST_MARGIN)) {
4119 : if (!within_margin(last_enqueued_diff, UTIL_EST_MARGIN))
4120 : goto done;
4121 :
4122 : return;
4123 : }
4124 :
4125 : /*
4126 : * To avoid overestimation of actual task utilization, skip updates if
4127 : * we cannot grant there is idle time in this CPU.
4128 : */
4129 : if (task_util(p) > capacity_orig_of(cpu_of(rq_of(cfs_rq))))
4130 : return;
4131 :
4132 : /*
4133 : * Update Task's estimated utilization
4134 : *
4135 : * When *p completes an activation we can consolidate another sample
4136 : * of the task size. This is done by storing the current PELT value
4137 : * as ue.enqueued and by using this value to update the Exponential
4138 : * Weighted Moving Average (EWMA):
4139 : *
4140 : * ewma(t) = w * task_util(p) + (1-w) * ewma(t-1)
4141 : * = w * task_util(p) + ewma(t-1) - w * ewma(t-1)
4142 : * = w * (task_util(p) - ewma(t-1)) + ewma(t-1)
4143 : * = w * ( last_ewma_diff ) + ewma(t-1)
4144 : * = w * (last_ewma_diff + ewma(t-1) / w)
4145 : *
4146 : * Where 'w' is the weight of new samples, which is configured to be
4147 : * 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT)
4148 : */
4149 : ue.ewma <<= UTIL_EST_WEIGHT_SHIFT;
4150 : ue.ewma += last_ewma_diff;
4151 : ue.ewma >>= UTIL_EST_WEIGHT_SHIFT;
4152 : done:
4153 : ue.enqueued |= UTIL_AVG_UNCHANGED;
4154 : WRITE_ONCE(p->se.avg.util_est, ue);
4155 :
4156 : trace_sched_util_est_se_tp(&p->se);
4157 : }
4158 :
4159 : static inline int task_fits_capacity(struct task_struct *p,
4160 : unsigned long capacity)
4161 : {
4162 : return fits_capacity(uclamp_task_util(p), capacity);
4163 : }
4164 :
4165 : static inline void update_misfit_status(struct task_struct *p, struct rq *rq)
4166 : {
4167 : if (!static_branch_unlikely(&sched_asym_cpucapacity))
4168 : return;
4169 :
4170 : if (!p || p->nr_cpus_allowed == 1) {
4171 : rq->misfit_task_load = 0;
4172 : return;
4173 : }
4174 :
4175 : if (task_fits_capacity(p, capacity_of(cpu_of(rq)))) {
4176 : rq->misfit_task_load = 0;
4177 : return;
4178 : }
4179 :
4180 : /*
4181 : * Make sure that misfit_task_load will not be null even if
4182 : * task_h_load() returns 0.
4183 : */
4184 : rq->misfit_task_load = max_t(unsigned long, task_h_load(p), 1);
4185 : }
4186 :
4187 : #else /* CONFIG_SMP */
4188 :
4189 : static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq)
4190 : {
4191 : return true;
4192 : }
4193 :
4194 : #define UPDATE_TG 0x0
4195 : #define SKIP_AGE_LOAD 0x0
4196 : #define DO_ATTACH 0x0
4197 :
4198 : static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int not_used1)
4199 : {
4200 1241 : cfs_rq_util_change(cfs_rq, 0);
4201 : }
4202 :
4203 : static inline void remove_entity_load_avg(struct sched_entity *se) {}
4204 :
4205 : static inline void
4206 : attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {}
4207 : static inline void
4208 : detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {}
4209 :
4210 : static inline int newidle_balance(struct rq *rq, struct rq_flags *rf)
4211 : {
4212 : return 0;
4213 : }
4214 :
4215 : static inline void
4216 : util_est_enqueue(struct cfs_rq *cfs_rq, struct task_struct *p) {}
4217 :
4218 : static inline void
4219 : util_est_dequeue(struct cfs_rq *cfs_rq, struct task_struct *p) {}
4220 :
4221 : static inline void
4222 : util_est_update(struct cfs_rq *cfs_rq, struct task_struct *p,
4223 : bool task_sleep) {}
4224 : static inline void update_misfit_status(struct task_struct *p, struct rq *rq) {}
4225 :
4226 : #endif /* CONFIG_SMP */
4227 :
4228 : static void check_spread(struct cfs_rq *cfs_rq, struct sched_entity *se)
4229 : {
4230 : #ifdef CONFIG_SCHED_DEBUG
4231 1239 : s64 d = se->vruntime - cfs_rq->min_vruntime;
4232 :
4233 : if (d < 0)
4234 : d = -d;
4235 :
4236 : if (d > 3*sysctl_sched_latency)
4237 : schedstat_inc(cfs_rq->nr_spread_over);
4238 : #endif
4239 : }
4240 :
4241 : static void
4242 617 : place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
4243 : {
4244 617 : u64 vruntime = cfs_rq->min_vruntime;
4245 :
4246 : /*
4247 : * The 'current' period is already promised to the current tasks,
4248 : * however the extra weight of the new task will slow them down a
4249 : * little, place the new task so that it fits in the slot that
4250 : * stays open at the end.
4251 : */
4252 617 : if (initial && sched_feat(START_DEBIT))
4253 107 : vruntime += sched_vslice(cfs_rq, se);
4254 :
4255 : /* sleeps up to a single latency don't count. */
4256 617 : if (!initial) {
4257 : unsigned long thresh;
4258 :
4259 510 : if (se_is_idle(se))
4260 : thresh = sysctl_sched_min_granularity;
4261 : else
4262 510 : thresh = sysctl_sched_latency;
4263 :
4264 : /*
4265 : * Halve their sleep time's effect, to allow
4266 : * for a gentler effect of sleepers:
4267 : */
4268 510 : if (sched_feat(GENTLE_FAIR_SLEEPERS))
4269 510 : thresh >>= 1;
4270 :
4271 510 : vruntime -= thresh;
4272 : }
4273 :
4274 : /* ensure we never gain time by being placed backwards. */
4275 1234 : se->vruntime = max_vruntime(se->vruntime, vruntime);
4276 617 : }
4277 :
4278 : static void check_enqueue_throttle(struct cfs_rq *cfs_rq);
4279 :
4280 : static inline bool cfs_bandwidth_used(void);
4281 :
4282 : /*
4283 : * MIGRATION
4284 : *
4285 : * dequeue
4286 : * update_curr()
4287 : * update_min_vruntime()
4288 : * vruntime -= min_vruntime
4289 : *
4290 : * enqueue
4291 : * update_curr()
4292 : * update_min_vruntime()
4293 : * vruntime += min_vruntime
4294 : *
4295 : * this way the vruntime transition between RQs is done when both
4296 : * min_vruntime are up-to-date.
4297 : *
4298 : * WAKEUP (remote)
4299 : *
4300 : * ->migrate_task_rq_fair() (p->state == TASK_WAKING)
4301 : * vruntime -= min_vruntime
4302 : *
4303 : * enqueue
4304 : * update_curr()
4305 : * update_min_vruntime()
4306 : * vruntime += min_vruntime
4307 : *
4308 : * this way we don't have the most up-to-date min_vruntime on the originating
4309 : * CPU and an up-to-date min_vruntime on the destination CPU.
4310 : */
4311 :
4312 : static void
4313 620 : enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
4314 : {
4315 620 : bool renorm = !(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_MIGRATED);
4316 620 : bool curr = cfs_rq->curr == se;
4317 :
4318 : /*
4319 : * If we're the current task, we must renormalise before calling
4320 : * update_curr().
4321 : */
4322 620 : if (renorm && curr)
4323 0 : se->vruntime += cfs_rq->min_vruntime;
4324 :
4325 620 : update_curr(cfs_rq);
4326 :
4327 : /*
4328 : * Otherwise, renormalise after, such that we're placed at the current
4329 : * moment in time, instead of some random moment in the past. Being
4330 : * placed in the past could significantly boost this task to the
4331 : * fairness detriment of existing tasks.
4332 : */
4333 620 : if (renorm && !curr)
4334 110 : se->vruntime += cfs_rq->min_vruntime;
4335 :
4336 : /*
4337 : * When enqueuing a sched_entity, we must:
4338 : * - Update loads to have both entity and cfs_rq synced with now.
4339 : * - Add its load to cfs_rq->runnable_avg
4340 : * - For group_entity, update its weight to reflect the new share of
4341 : * its group cfs_rq
4342 : * - Add its new weight to cfs_rq->load.weight
4343 : */
4344 620 : update_load_avg(cfs_rq, se, UPDATE_TG | DO_ATTACH);
4345 620 : se_update_runnable(se);
4346 620 : update_cfs_group(se);
4347 1240 : account_entity_enqueue(cfs_rq, se);
4348 :
4349 620 : if (flags & ENQUEUE_WAKEUP)
4350 510 : place_entity(cfs_rq, se, 0);
4351 :
4352 : check_schedstat_required();
4353 620 : update_stats_enqueue_fair(cfs_rq, se, flags);
4354 620 : check_spread(cfs_rq, se);
4355 620 : if (!curr)
4356 620 : __enqueue_entity(cfs_rq, se);
4357 620 : se->on_rq = 1;
4358 :
4359 : /*
4360 : * When bandwidth control is enabled, cfs might have been removed
4361 : * because of a parent been throttled but cfs->nr_running > 1. Try to
4362 : * add it unconditionally.
4363 : */
4364 : if (cfs_rq->nr_running == 1 || cfs_bandwidth_used())
4365 : list_add_leaf_cfs_rq(cfs_rq);
4366 :
4367 : if (cfs_rq->nr_running == 1)
4368 : check_enqueue_throttle(cfs_rq);
4369 620 : }
4370 :
4371 : static void __clear_buddies_last(struct sched_entity *se)
4372 : {
4373 0 : for_each_sched_entity(se) {
4374 0 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
4375 0 : if (cfs_rq->last != se)
4376 : break;
4377 :
4378 0 : cfs_rq->last = NULL;
4379 : }
4380 : }
4381 :
4382 : static void __clear_buddies_next(struct sched_entity *se)
4383 : {
4384 200 : for_each_sched_entity(se) {
4385 400 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
4386 200 : if (cfs_rq->next != se)
4387 : break;
4388 :
4389 200 : cfs_rq->next = NULL;
4390 : }
4391 : }
4392 :
4393 : static void __clear_buddies_skip(struct sched_entity *se)
4394 : {
4395 0 : for_each_sched_entity(se) {
4396 0 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
4397 0 : if (cfs_rq->skip != se)
4398 : break;
4399 :
4400 0 : cfs_rq->skip = NULL;
4401 : }
4402 : }
4403 :
4404 1241 : static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se)
4405 : {
4406 1241 : if (cfs_rq->last == se)
4407 : __clear_buddies_last(se);
4408 :
4409 1241 : if (cfs_rq->next == se)
4410 : __clear_buddies_next(se);
4411 :
4412 1241 : if (cfs_rq->skip == se)
4413 : __clear_buddies_skip(se);
4414 1241 : }
4415 :
4416 : static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);
4417 :
4418 : static void
4419 618 : dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
4420 : {
4421 : /*
4422 : * Update run-time statistics of the 'current'.
4423 : */
4424 618 : update_curr(cfs_rq);
4425 :
4426 : /*
4427 : * When dequeuing a sched_entity, we must:
4428 : * - Update loads to have both entity and cfs_rq synced with now.
4429 : * - Subtract its load from the cfs_rq->runnable_avg.
4430 : * - Subtract its previous weight from cfs_rq->load.weight.
4431 : * - For group entity, update its weight to reflect the new share
4432 : * of its group cfs_rq.
4433 : */
4434 618 : update_load_avg(cfs_rq, se, UPDATE_TG);
4435 618 : se_update_runnable(se);
4436 :
4437 618 : update_stats_dequeue_fair(cfs_rq, se, flags);
4438 :
4439 618 : clear_buddies(cfs_rq, se);
4440 :
4441 618 : if (se != cfs_rq->curr)
4442 : __dequeue_entity(cfs_rq, se);
4443 618 : se->on_rq = 0;
4444 1236 : account_entity_dequeue(cfs_rq, se);
4445 :
4446 : /*
4447 : * Normalize after update_curr(); which will also have moved
4448 : * min_vruntime if @se is the one holding it back. But before doing
4449 : * update_min_vruntime() again, which will discount @se's position and
4450 : * can move min_vruntime forward still more.
4451 : */
4452 618 : if (!(flags & DEQUEUE_SLEEP))
4453 3 : se->vruntime -= cfs_rq->min_vruntime;
4454 :
4455 : /* return excess runtime on last dequeue */
4456 618 : return_cfs_rq_runtime(cfs_rq);
4457 :
4458 618 : update_cfs_group(se);
4459 :
4460 : /*
4461 : * Now advance min_vruntime if @se was the entity holding it back,
4462 : * except when: DEQUEUE_SAVE && !DEQUEUE_MOVE, in this case we'll be
4463 : * put back on, and if we advance min_vruntime, we'll be placed back
4464 : * further than we started -- ie. we'll be penalized.
4465 : */
4466 618 : if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) != DEQUEUE_SAVE)
4467 615 : update_min_vruntime(cfs_rq);
4468 618 : }
4469 :
4470 : /*
4471 : * Preempt the current task with a newly woken task if needed:
4472 : */
4473 : static void
4474 3 : check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr)
4475 : {
4476 : unsigned long ideal_runtime, delta_exec;
4477 : struct sched_entity *se;
4478 : s64 delta;
4479 :
4480 3 : ideal_runtime = sched_slice(cfs_rq, curr);
4481 3 : delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime;
4482 3 : if (delta_exec > ideal_runtime) {
4483 3 : resched_curr(rq_of(cfs_rq));
4484 : /*
4485 : * The current task ran long enough, ensure it doesn't get
4486 : * re-elected due to buddy favours.
4487 : */
4488 3 : clear_buddies(cfs_rq, curr);
4489 3 : return;
4490 : }
4491 :
4492 : /*
4493 : * Ensure that a task that missed wakeup preemption by a
4494 : * narrow margin doesn't have to wait for a full slice.
4495 : * This also mitigates buddy induced latencies under load.
4496 : */
4497 0 : if (delta_exec < sysctl_sched_min_granularity)
4498 : return;
4499 :
4500 0 : se = __pick_first_entity(cfs_rq);
4501 0 : delta = curr->vruntime - se->vruntime;
4502 :
4503 0 : if (delta < 0)
4504 : return;
4505 :
4506 0 : if (delta > ideal_runtime)
4507 0 : resched_curr(rq_of(cfs_rq));
4508 : }
4509 :
4510 : static void
4511 620 : set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
4512 : {
4513 620 : clear_buddies(cfs_rq, se);
4514 :
4515 : /* 'current' is not kept within the tree. */
4516 620 : if (se->on_rq) {
4517 : /*
4518 : * Any task has to be enqueued before it get to execute on
4519 : * a CPU. So account for the time it spent waiting on the
4520 : * runqueue.
4521 : */
4522 1240 : update_stats_wait_end_fair(cfs_rq, se);
4523 : __dequeue_entity(cfs_rq, se);
4524 : update_load_avg(cfs_rq, se, UPDATE_TG);
4525 : }
4526 :
4527 1240 : update_stats_curr_start(cfs_rq, se);
4528 620 : cfs_rq->curr = se;
4529 :
4530 : /*
4531 : * Track our maximum slice length, if the CPU's load is at
4532 : * least twice that of our own weight (i.e. dont track it
4533 : * when there are only lesser-weight tasks around):
4534 : */
4535 : if (schedstat_enabled() &&
4536 : rq_of(cfs_rq)->cfs.load.weight >= 2*se->load.weight) {
4537 : struct sched_statistics *stats;
4538 :
4539 : stats = __schedstats_from_se(se);
4540 : __schedstat_set(stats->slice_max,
4541 : max((u64)stats->slice_max,
4542 : se->sum_exec_runtime - se->prev_sum_exec_runtime));
4543 : }
4544 :
4545 620 : se->prev_sum_exec_runtime = se->sum_exec_runtime;
4546 620 : }
4547 :
4548 : static int
4549 : wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se);
4550 :
4551 : /*
4552 : * Pick the next process, keeping these things in mind, in this order:
4553 : * 1) keep things fair between processes/task groups
4554 : * 2) pick the "next" process, since someone really wants that to run
4555 : * 3) pick the "last" process, for cache locality
4556 : * 4) do not run the "skip" process, if something else is available
4557 : */
4558 : static struct sched_entity *
4559 617 : pick_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *curr)
4560 : {
4561 617 : struct sched_entity *left = __pick_first_entity(cfs_rq);
4562 : struct sched_entity *se;
4563 :
4564 : /*
4565 : * If curr is set we have to see if its left of the leftmost entity
4566 : * still in the tree, provided there was anything in the tree at all.
4567 : */
4568 617 : if (!left || (curr && entity_before(curr, left)))
4569 : left = curr;
4570 :
4571 617 : se = left; /* ideally we run the leftmost entity */
4572 :
4573 : /*
4574 : * Avoid running the skip buddy, if running something else can
4575 : * be done without getting too unfair.
4576 : */
4577 617 : if (cfs_rq->skip && cfs_rq->skip == se) {
4578 : struct sched_entity *second;
4579 :
4580 0 : if (se == curr) {
4581 : second = __pick_first_entity(cfs_rq);
4582 : } else {
4583 0 : second = __pick_next_entity(se);
4584 0 : if (!second || (curr && entity_before(curr, second)))
4585 : second = curr;
4586 : }
4587 :
4588 0 : if (second && wakeup_preempt_entity(second, left) < 1)
4589 0 : se = second;
4590 : }
4591 :
4592 617 : if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, left) < 1) {
4593 : /*
4594 : * Someone really wants this to run. If it's not unfair, run it.
4595 : */
4596 200 : se = cfs_rq->next;
4597 417 : } else if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1) {
4598 : /*
4599 : * Prefer last buddy, try to return the CPU to a preempted task.
4600 : */
4601 0 : se = cfs_rq->last;
4602 : }
4603 :
4604 617 : return se;
4605 : }
4606 :
4607 : static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq);
4608 :
4609 619 : static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev)
4610 : {
4611 : /*
4612 : * If still on the runqueue then deactivate_task()
4613 : * was not called and update_curr() has to be done:
4614 : */
4615 619 : if (prev->on_rq)
4616 1 : update_curr(cfs_rq);
4617 :
4618 : /* throttle cfs_rqs exceeding runtime */
4619 619 : check_cfs_rq_runtime(cfs_rq);
4620 :
4621 619 : check_spread(cfs_rq, prev);
4622 :
4623 619 : if (prev->on_rq) {
4624 1 : update_stats_wait_start_fair(cfs_rq, prev);
4625 : /* Put 'current' back into the tree. */
4626 1 : __enqueue_entity(cfs_rq, prev);
4627 : /* in !on_rq case, update occurred at dequeue */
4628 1 : update_load_avg(cfs_rq, prev, 0);
4629 : }
4630 619 : cfs_rq->curr = NULL;
4631 619 : }
4632 :
4633 : static void
4634 3 : entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
4635 : {
4636 : /*
4637 : * Update run-time statistics of the 'current'.
4638 : */
4639 3 : update_curr(cfs_rq);
4640 :
4641 : /*
4642 : * Ensure that runnable average is periodically updated.
4643 : */
4644 3 : update_load_avg(cfs_rq, curr, UPDATE_TG);
4645 3 : update_cfs_group(curr);
4646 :
4647 : #ifdef CONFIG_SCHED_HRTICK
4648 : /*
4649 : * queued ticks are scheduled to match the slice, so don't bother
4650 : * validating it and just reschedule.
4651 : */
4652 : if (queued) {
4653 : resched_curr(rq_of(cfs_rq));
4654 : return;
4655 : }
4656 : /*
4657 : * don't let the period tick interfere with the hrtick preemption
4658 : */
4659 : if (!sched_feat(DOUBLE_TICK) &&
4660 : hrtimer_active(&rq_of(cfs_rq)->hrtick_timer))
4661 : return;
4662 : #endif
4663 :
4664 3 : if (cfs_rq->nr_running > 1)
4665 3 : check_preempt_tick(cfs_rq, curr);
4666 3 : }
4667 :
4668 :
4669 : /**************************************************
4670 : * CFS bandwidth control machinery
4671 : */
4672 :
4673 : #ifdef CONFIG_CFS_BANDWIDTH
4674 :
4675 : #ifdef CONFIG_JUMP_LABEL
4676 : static struct static_key __cfs_bandwidth_used;
4677 :
4678 : static inline bool cfs_bandwidth_used(void)
4679 : {
4680 : return static_key_false(&__cfs_bandwidth_used);
4681 : }
4682 :
4683 : void cfs_bandwidth_usage_inc(void)
4684 : {
4685 : static_key_slow_inc_cpuslocked(&__cfs_bandwidth_used);
4686 : }
4687 :
4688 : void cfs_bandwidth_usage_dec(void)
4689 : {
4690 : static_key_slow_dec_cpuslocked(&__cfs_bandwidth_used);
4691 : }
4692 : #else /* CONFIG_JUMP_LABEL */
4693 : static bool cfs_bandwidth_used(void)
4694 : {
4695 : return true;
4696 : }
4697 :
4698 : void cfs_bandwidth_usage_inc(void) {}
4699 : void cfs_bandwidth_usage_dec(void) {}
4700 : #endif /* CONFIG_JUMP_LABEL */
4701 :
4702 : /*
4703 : * default period for cfs group bandwidth.
4704 : * default: 0.1s, units: nanoseconds
4705 : */
4706 : static inline u64 default_cfs_period(void)
4707 : {
4708 : return 100000000ULL;
4709 : }
4710 :
4711 : static inline u64 sched_cfs_bandwidth_slice(void)
4712 : {
4713 : return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC;
4714 : }
4715 :
4716 : /*
4717 : * Replenish runtime according to assigned quota. We use sched_clock_cpu
4718 : * directly instead of rq->clock to avoid adding additional synchronization
4719 : * around rq->lock.
4720 : *
4721 : * requires cfs_b->lock
4722 : */
4723 : void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)
4724 : {
4725 : s64 runtime;
4726 :
4727 : if (unlikely(cfs_b->quota == RUNTIME_INF))
4728 : return;
4729 :
4730 : cfs_b->runtime += cfs_b->quota;
4731 : runtime = cfs_b->runtime_snap - cfs_b->runtime;
4732 : if (runtime > 0) {
4733 : cfs_b->burst_time += runtime;
4734 : cfs_b->nr_burst++;
4735 : }
4736 :
4737 : cfs_b->runtime = min(cfs_b->runtime, cfs_b->quota + cfs_b->burst);
4738 : cfs_b->runtime_snap = cfs_b->runtime;
4739 : }
4740 :
4741 : static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
4742 : {
4743 : return &tg->cfs_bandwidth;
4744 : }
4745 :
4746 : /* returns 0 on failure to allocate runtime */
4747 : static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b,
4748 : struct cfs_rq *cfs_rq, u64 target_runtime)
4749 : {
4750 : u64 min_amount, amount = 0;
4751 :
4752 : lockdep_assert_held(&cfs_b->lock);
4753 :
4754 : /* note: this is a positive sum as runtime_remaining <= 0 */
4755 : min_amount = target_runtime - cfs_rq->runtime_remaining;
4756 :
4757 : if (cfs_b->quota == RUNTIME_INF)
4758 : amount = min_amount;
4759 : else {
4760 : start_cfs_bandwidth(cfs_b);
4761 :
4762 : if (cfs_b->runtime > 0) {
4763 : amount = min(cfs_b->runtime, min_amount);
4764 : cfs_b->runtime -= amount;
4765 : cfs_b->idle = 0;
4766 : }
4767 : }
4768 :
4769 : cfs_rq->runtime_remaining += amount;
4770 :
4771 : return cfs_rq->runtime_remaining > 0;
4772 : }
4773 :
4774 : /* returns 0 on failure to allocate runtime */
4775 : static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
4776 : {
4777 : struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
4778 : int ret;
4779 :
4780 : raw_spin_lock(&cfs_b->lock);
4781 : ret = __assign_cfs_rq_runtime(cfs_b, cfs_rq, sched_cfs_bandwidth_slice());
4782 : raw_spin_unlock(&cfs_b->lock);
4783 :
4784 : return ret;
4785 : }
4786 :
4787 : static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
4788 : {
4789 : /* dock delta_exec before expiring quota (as it could span periods) */
4790 : cfs_rq->runtime_remaining -= delta_exec;
4791 :
4792 : if (likely(cfs_rq->runtime_remaining > 0))
4793 : return;
4794 :
4795 : if (cfs_rq->throttled)
4796 : return;
4797 : /*
4798 : * if we're unable to extend our runtime we resched so that the active
4799 : * hierarchy can be throttled
4800 : */
4801 : if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr))
4802 : resched_curr(rq_of(cfs_rq));
4803 : }
4804 :
4805 : static __always_inline
4806 : void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
4807 : {
4808 : if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled)
4809 : return;
4810 :
4811 : __account_cfs_rq_runtime(cfs_rq, delta_exec);
4812 : }
4813 :
4814 : static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
4815 : {
4816 : return cfs_bandwidth_used() && cfs_rq->throttled;
4817 : }
4818 :
4819 : /* check whether cfs_rq, or any parent, is throttled */
4820 : static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
4821 : {
4822 : return cfs_bandwidth_used() && cfs_rq->throttle_count;
4823 : }
4824 :
4825 : /*
4826 : * Ensure that neither of the group entities corresponding to src_cpu or
4827 : * dest_cpu are members of a throttled hierarchy when performing group
4828 : * load-balance operations.
4829 : */
4830 : static inline int throttled_lb_pair(struct task_group *tg,
4831 : int src_cpu, int dest_cpu)
4832 : {
4833 : struct cfs_rq *src_cfs_rq, *dest_cfs_rq;
4834 :
4835 : src_cfs_rq = tg->cfs_rq[src_cpu];
4836 : dest_cfs_rq = tg->cfs_rq[dest_cpu];
4837 :
4838 : return throttled_hierarchy(src_cfs_rq) ||
4839 : throttled_hierarchy(dest_cfs_rq);
4840 : }
4841 :
4842 : static int tg_unthrottle_up(struct task_group *tg, void *data)
4843 : {
4844 : struct rq *rq = data;
4845 : struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
4846 :
4847 : cfs_rq->throttle_count--;
4848 : if (!cfs_rq->throttle_count) {
4849 : cfs_rq->throttled_clock_task_time += rq_clock_task(rq) -
4850 : cfs_rq->throttled_clock_task;
4851 :
4852 : /* Add cfs_rq with load or one or more already running entities to the list */
4853 : if (!cfs_rq_is_decayed(cfs_rq) || cfs_rq->nr_running)
4854 : list_add_leaf_cfs_rq(cfs_rq);
4855 : }
4856 :
4857 : return 0;
4858 : }
4859 :
4860 : static int tg_throttle_down(struct task_group *tg, void *data)
4861 : {
4862 : struct rq *rq = data;
4863 : struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
4864 :
4865 : /* group is entering throttled state, stop time */
4866 : if (!cfs_rq->throttle_count) {
4867 : cfs_rq->throttled_clock_task = rq_clock_task(rq);
4868 : list_del_leaf_cfs_rq(cfs_rq);
4869 : }
4870 : cfs_rq->throttle_count++;
4871 :
4872 : return 0;
4873 : }
4874 :
4875 : static bool throttle_cfs_rq(struct cfs_rq *cfs_rq)
4876 : {
4877 : struct rq *rq = rq_of(cfs_rq);
4878 : struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
4879 : struct sched_entity *se;
4880 : long task_delta, idle_task_delta, dequeue = 1;
4881 :
4882 : raw_spin_lock(&cfs_b->lock);
4883 : /* This will start the period timer if necessary */
4884 : if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, 1)) {
4885 : /*
4886 : * We have raced with bandwidth becoming available, and if we
4887 : * actually throttled the timer might not unthrottle us for an
4888 : * entire period. We additionally needed to make sure that any
4889 : * subsequent check_cfs_rq_runtime calls agree not to throttle
4890 : * us, as we may commit to do cfs put_prev+pick_next, so we ask
4891 : * for 1ns of runtime rather than just check cfs_b.
4892 : */
4893 : dequeue = 0;
4894 : } else {
4895 : list_add_tail_rcu(&cfs_rq->throttled_list,
4896 : &cfs_b->throttled_cfs_rq);
4897 : }
4898 : raw_spin_unlock(&cfs_b->lock);
4899 :
4900 : if (!dequeue)
4901 : return false; /* Throttle no longer required. */
4902 :
4903 : se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))];
4904 :
4905 : /* freeze hierarchy runnable averages while throttled */
4906 : rcu_read_lock();
4907 : walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq);
4908 : rcu_read_unlock();
4909 :
4910 : task_delta = cfs_rq->h_nr_running;
4911 : idle_task_delta = cfs_rq->idle_h_nr_running;
4912 : for_each_sched_entity(se) {
4913 : struct cfs_rq *qcfs_rq = cfs_rq_of(se);
4914 : /* throttled entity or throttle-on-deactivate */
4915 : if (!se->on_rq)
4916 : goto done;
4917 :
4918 : dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP);
4919 :
4920 : if (cfs_rq_is_idle(group_cfs_rq(se)))
4921 : idle_task_delta = cfs_rq->h_nr_running;
4922 :
4923 : qcfs_rq->h_nr_running -= task_delta;
4924 : qcfs_rq->idle_h_nr_running -= idle_task_delta;
4925 :
4926 : if (qcfs_rq->load.weight) {
4927 : /* Avoid re-evaluating load for this entity: */
4928 : se = parent_entity(se);
4929 : break;
4930 : }
4931 : }
4932 :
4933 : for_each_sched_entity(se) {
4934 : struct cfs_rq *qcfs_rq = cfs_rq_of(se);
4935 : /* throttled entity or throttle-on-deactivate */
4936 : if (!se->on_rq)
4937 : goto done;
4938 :
4939 : update_load_avg(qcfs_rq, se, 0);
4940 : se_update_runnable(se);
4941 :
4942 : if (cfs_rq_is_idle(group_cfs_rq(se)))
4943 : idle_task_delta = cfs_rq->h_nr_running;
4944 :
4945 : qcfs_rq->h_nr_running -= task_delta;
4946 : qcfs_rq->idle_h_nr_running -= idle_task_delta;
4947 : }
4948 :
4949 : /* At this point se is NULL and we are at root level*/
4950 : sub_nr_running(rq, task_delta);
4951 :
4952 : done:
4953 : /*
4954 : * Note: distribution will already see us throttled via the
4955 : * throttled-list. rq->lock protects completion.
4956 : */
4957 : cfs_rq->throttled = 1;
4958 : cfs_rq->throttled_clock = rq_clock(rq);
4959 : return true;
4960 : }
4961 :
4962 : void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
4963 : {
4964 : struct rq *rq = rq_of(cfs_rq);
4965 : struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
4966 : struct sched_entity *se;
4967 : long task_delta, idle_task_delta;
4968 :
4969 : se = cfs_rq->tg->se[cpu_of(rq)];
4970 :
4971 : cfs_rq->throttled = 0;
4972 :
4973 : update_rq_clock(rq);
4974 :
4975 : raw_spin_lock(&cfs_b->lock);
4976 : cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock;
4977 : list_del_rcu(&cfs_rq->throttled_list);
4978 : raw_spin_unlock(&cfs_b->lock);
4979 :
4980 : /* update hierarchical throttle state */
4981 : walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq);
4982 :
4983 : /* Nothing to run but something to decay (on_list)? Complete the branch */
4984 : if (!cfs_rq->load.weight) {
4985 : if (cfs_rq->on_list)
4986 : goto unthrottle_throttle;
4987 : return;
4988 : }
4989 :
4990 : task_delta = cfs_rq->h_nr_running;
4991 : idle_task_delta = cfs_rq->idle_h_nr_running;
4992 : for_each_sched_entity(se) {
4993 : struct cfs_rq *qcfs_rq = cfs_rq_of(se);
4994 :
4995 : if (se->on_rq)
4996 : break;
4997 : enqueue_entity(qcfs_rq, se, ENQUEUE_WAKEUP);
4998 :
4999 : if (cfs_rq_is_idle(group_cfs_rq(se)))
5000 : idle_task_delta = cfs_rq->h_nr_running;
5001 :
5002 : qcfs_rq->h_nr_running += task_delta;
5003 : qcfs_rq->idle_h_nr_running += idle_task_delta;
5004 :
5005 : /* end evaluation on encountering a throttled cfs_rq */
5006 : if (cfs_rq_throttled(qcfs_rq))
5007 : goto unthrottle_throttle;
5008 : }
5009 :
5010 : for_each_sched_entity(se) {
5011 : struct cfs_rq *qcfs_rq = cfs_rq_of(se);
5012 :
5013 : update_load_avg(qcfs_rq, se, UPDATE_TG);
5014 : se_update_runnable(se);
5015 :
5016 : if (cfs_rq_is_idle(group_cfs_rq(se)))
5017 : idle_task_delta = cfs_rq->h_nr_running;
5018 :
5019 : qcfs_rq->h_nr_running += task_delta;
5020 : qcfs_rq->idle_h_nr_running += idle_task_delta;
5021 :
5022 : /* end evaluation on encountering a throttled cfs_rq */
5023 : if (cfs_rq_throttled(qcfs_rq))
5024 : goto unthrottle_throttle;
5025 :
5026 : /*
5027 : * One parent has been throttled and cfs_rq removed from the
5028 : * list. Add it back to not break the leaf list.
5029 : */
5030 : if (throttled_hierarchy(qcfs_rq))
5031 : list_add_leaf_cfs_rq(qcfs_rq);
5032 : }
5033 :
5034 : /* At this point se is NULL and we are at root level*/
5035 : add_nr_running(rq, task_delta);
5036 :
5037 : unthrottle_throttle:
5038 : /*
5039 : * The cfs_rq_throttled() breaks in the above iteration can result in
5040 : * incomplete leaf list maintenance, resulting in triggering the
5041 : * assertion below.
5042 : */
5043 : for_each_sched_entity(se) {
5044 : struct cfs_rq *qcfs_rq = cfs_rq_of(se);
5045 :
5046 : if (list_add_leaf_cfs_rq(qcfs_rq))
5047 : break;
5048 : }
5049 :
5050 : assert_list_leaf_cfs_rq(rq);
5051 :
5052 : /* Determine whether we need to wake up potentially idle CPU: */
5053 : if (rq->curr == rq->idle && rq->cfs.nr_running)
5054 : resched_curr(rq);
5055 : }
5056 :
5057 : static void distribute_cfs_runtime(struct cfs_bandwidth *cfs_b)
5058 : {
5059 : struct cfs_rq *cfs_rq;
5060 : u64 runtime, remaining = 1;
5061 :
5062 : rcu_read_lock();
5063 : list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq,
5064 : throttled_list) {
5065 : struct rq *rq = rq_of(cfs_rq);
5066 : struct rq_flags rf;
5067 :
5068 : rq_lock_irqsave(rq, &rf);
5069 : if (!cfs_rq_throttled(cfs_rq))
5070 : goto next;
5071 :
5072 : /* By the above check, this should never be true */
5073 : SCHED_WARN_ON(cfs_rq->runtime_remaining > 0);
5074 :
5075 : raw_spin_lock(&cfs_b->lock);
5076 : runtime = -cfs_rq->runtime_remaining + 1;
5077 : if (runtime > cfs_b->runtime)
5078 : runtime = cfs_b->runtime;
5079 : cfs_b->runtime -= runtime;
5080 : remaining = cfs_b->runtime;
5081 : raw_spin_unlock(&cfs_b->lock);
5082 :
5083 : cfs_rq->runtime_remaining += runtime;
5084 :
5085 : /* we check whether we're throttled above */
5086 : if (cfs_rq->runtime_remaining > 0)
5087 : unthrottle_cfs_rq(cfs_rq);
5088 :
5089 : next:
5090 : rq_unlock_irqrestore(rq, &rf);
5091 :
5092 : if (!remaining)
5093 : break;
5094 : }
5095 : rcu_read_unlock();
5096 : }
5097 :
5098 : /*
5099 : * Responsible for refilling a task_group's bandwidth and unthrottling its
5100 : * cfs_rqs as appropriate. If there has been no activity within the last
5101 : * period the timer is deactivated until scheduling resumes; cfs_b->idle is
5102 : * used to track this state.
5103 : */
5104 : static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags)
5105 : {
5106 : int throttled;
5107 :
5108 : /* no need to continue the timer with no bandwidth constraint */
5109 : if (cfs_b->quota == RUNTIME_INF)
5110 : goto out_deactivate;
5111 :
5112 : throttled = !list_empty(&cfs_b->throttled_cfs_rq);
5113 : cfs_b->nr_periods += overrun;
5114 :
5115 : /* Refill extra burst quota even if cfs_b->idle */
5116 : __refill_cfs_bandwidth_runtime(cfs_b);
5117 :
5118 : /*
5119 : * idle depends on !throttled (for the case of a large deficit), and if
5120 : * we're going inactive then everything else can be deferred
5121 : */
5122 : if (cfs_b->idle && !throttled)
5123 : goto out_deactivate;
5124 :
5125 : if (!throttled) {
5126 : /* mark as potentially idle for the upcoming period */
5127 : cfs_b->idle = 1;
5128 : return 0;
5129 : }
5130 :
5131 : /* account preceding periods in which throttling occurred */
5132 : cfs_b->nr_throttled += overrun;
5133 :
5134 : /*
5135 : * This check is repeated as we release cfs_b->lock while we unthrottle.
5136 : */
5137 : while (throttled && cfs_b->runtime > 0) {
5138 : raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
5139 : /* we can't nest cfs_b->lock while distributing bandwidth */
5140 : distribute_cfs_runtime(cfs_b);
5141 : raw_spin_lock_irqsave(&cfs_b->lock, flags);
5142 :
5143 : throttled = !list_empty(&cfs_b->throttled_cfs_rq);
5144 : }
5145 :
5146 : /*
5147 : * While we are ensured activity in the period following an
5148 : * unthrottle, this also covers the case in which the new bandwidth is
5149 : * insufficient to cover the existing bandwidth deficit. (Forcing the
5150 : * timer to remain active while there are any throttled entities.)
5151 : */
5152 : cfs_b->idle = 0;
5153 :
5154 : return 0;
5155 :
5156 : out_deactivate:
5157 : return 1;
5158 : }
5159 :
5160 : /* a cfs_rq won't donate quota below this amount */
5161 : static const u64 min_cfs_rq_runtime = 1 * NSEC_PER_MSEC;
5162 : /* minimum remaining period time to redistribute slack quota */
5163 : static const u64 min_bandwidth_expiration = 2 * NSEC_PER_MSEC;
5164 : /* how long we wait to gather additional slack before distributing */
5165 : static const u64 cfs_bandwidth_slack_period = 5 * NSEC_PER_MSEC;
5166 :
5167 : /*
5168 : * Are we near the end of the current quota period?
5169 : *
5170 : * Requires cfs_b->lock for hrtimer_expires_remaining to be safe against the
5171 : * hrtimer base being cleared by hrtimer_start. In the case of
5172 : * migrate_hrtimers, base is never cleared, so we are fine.
5173 : */
5174 : static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire)
5175 : {
5176 : struct hrtimer *refresh_timer = &cfs_b->period_timer;
5177 : s64 remaining;
5178 :
5179 : /* if the call-back is running a quota refresh is already occurring */
5180 : if (hrtimer_callback_running(refresh_timer))
5181 : return 1;
5182 :
5183 : /* is a quota refresh about to occur? */
5184 : remaining = ktime_to_ns(hrtimer_expires_remaining(refresh_timer));
5185 : if (remaining < (s64)min_expire)
5186 : return 1;
5187 :
5188 : return 0;
5189 : }
5190 :
5191 : static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)
5192 : {
5193 : u64 min_left = cfs_bandwidth_slack_period + min_bandwidth_expiration;
5194 :
5195 : /* if there's a quota refresh soon don't bother with slack */
5196 : if (runtime_refresh_within(cfs_b, min_left))
5197 : return;
5198 :
5199 : /* don't push forwards an existing deferred unthrottle */
5200 : if (cfs_b->slack_started)
5201 : return;
5202 : cfs_b->slack_started = true;
5203 :
5204 : hrtimer_start(&cfs_b->slack_timer,
5205 : ns_to_ktime(cfs_bandwidth_slack_period),
5206 : HRTIMER_MODE_REL);
5207 : }
5208 :
5209 : /* we know any runtime found here is valid as update_curr() precedes return */
5210 : static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
5211 : {
5212 : struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
5213 : s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime;
5214 :
5215 : if (slack_runtime <= 0)
5216 : return;
5217 :
5218 : raw_spin_lock(&cfs_b->lock);
5219 : if (cfs_b->quota != RUNTIME_INF) {
5220 : cfs_b->runtime += slack_runtime;
5221 :
5222 : /* we are under rq->lock, defer unthrottling using a timer */
5223 : if (cfs_b->runtime > sched_cfs_bandwidth_slice() &&
5224 : !list_empty(&cfs_b->throttled_cfs_rq))
5225 : start_cfs_slack_bandwidth(cfs_b);
5226 : }
5227 : raw_spin_unlock(&cfs_b->lock);
5228 :
5229 : /* even if it's not valid for return we don't want to try again */
5230 : cfs_rq->runtime_remaining -= slack_runtime;
5231 : }
5232 :
5233 : static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
5234 : {
5235 : if (!cfs_bandwidth_used())
5236 : return;
5237 :
5238 : if (!cfs_rq->runtime_enabled || cfs_rq->nr_running)
5239 : return;
5240 :
5241 : __return_cfs_rq_runtime(cfs_rq);
5242 : }
5243 :
5244 : /*
5245 : * This is done with a timer (instead of inline with bandwidth return) since
5246 : * it's necessary to juggle rq->locks to unthrottle their respective cfs_rqs.
5247 : */
5248 : static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)
5249 : {
5250 : u64 runtime = 0, slice = sched_cfs_bandwidth_slice();
5251 : unsigned long flags;
5252 :
5253 : /* confirm we're still not at a refresh boundary */
5254 : raw_spin_lock_irqsave(&cfs_b->lock, flags);
5255 : cfs_b->slack_started = false;
5256 :
5257 : if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) {
5258 : raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
5259 : return;
5260 : }
5261 :
5262 : if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice)
5263 : runtime = cfs_b->runtime;
5264 :
5265 : raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
5266 :
5267 : if (!runtime)
5268 : return;
5269 :
5270 : distribute_cfs_runtime(cfs_b);
5271 : }
5272 :
5273 : /*
5274 : * When a group wakes up we want to make sure that its quota is not already
5275 : * expired/exceeded, otherwise it may be allowed to steal additional ticks of
5276 : * runtime as update_curr() throttling can not trigger until it's on-rq.
5277 : */
5278 : static void check_enqueue_throttle(struct cfs_rq *cfs_rq)
5279 : {
5280 : if (!cfs_bandwidth_used())
5281 : return;
5282 :
5283 : /* an active group must be handled by the update_curr()->put() path */
5284 : if (!cfs_rq->runtime_enabled || cfs_rq->curr)
5285 : return;
5286 :
5287 : /* ensure the group is not already throttled */
5288 : if (cfs_rq_throttled(cfs_rq))
5289 : return;
5290 :
5291 : /* update runtime allocation */
5292 : account_cfs_rq_runtime(cfs_rq, 0);
5293 : if (cfs_rq->runtime_remaining <= 0)
5294 : throttle_cfs_rq(cfs_rq);
5295 : }
5296 :
5297 : static void sync_throttle(struct task_group *tg, int cpu)
5298 : {
5299 : struct cfs_rq *pcfs_rq, *cfs_rq;
5300 :
5301 : if (!cfs_bandwidth_used())
5302 : return;
5303 :
5304 : if (!tg->parent)
5305 : return;
5306 :
5307 : cfs_rq = tg->cfs_rq[cpu];
5308 : pcfs_rq = tg->parent->cfs_rq[cpu];
5309 :
5310 : cfs_rq->throttle_count = pcfs_rq->throttle_count;
5311 : cfs_rq->throttled_clock_task = rq_clock_task(cpu_rq(cpu));
5312 : }
5313 :
5314 : /* conditionally throttle active cfs_rq's from put_prev_entity() */
5315 : static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq)
5316 : {
5317 : if (!cfs_bandwidth_used())
5318 : return false;
5319 :
5320 : if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0))
5321 : return false;
5322 :
5323 : /*
5324 : * it's possible for a throttled entity to be forced into a running
5325 : * state (e.g. set_curr_task), in this case we're finished.
5326 : */
5327 : if (cfs_rq_throttled(cfs_rq))
5328 : return true;
5329 :
5330 : return throttle_cfs_rq(cfs_rq);
5331 : }
5332 :
5333 : static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer)
5334 : {
5335 : struct cfs_bandwidth *cfs_b =
5336 : container_of(timer, struct cfs_bandwidth, slack_timer);
5337 :
5338 : do_sched_cfs_slack_timer(cfs_b);
5339 :
5340 : return HRTIMER_NORESTART;
5341 : }
5342 :
5343 : extern const u64 max_cfs_quota_period;
5344 :
5345 : static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer)
5346 : {
5347 : struct cfs_bandwidth *cfs_b =
5348 : container_of(timer, struct cfs_bandwidth, period_timer);
5349 : unsigned long flags;
5350 : int overrun;
5351 : int idle = 0;
5352 : int count = 0;
5353 :
5354 : raw_spin_lock_irqsave(&cfs_b->lock, flags);
5355 : for (;;) {
5356 : overrun = hrtimer_forward_now(timer, cfs_b->period);
5357 : if (!overrun)
5358 : break;
5359 :
5360 : idle = do_sched_cfs_period_timer(cfs_b, overrun, flags);
5361 :
5362 : if (++count > 3) {
5363 : u64 new, old = ktime_to_ns(cfs_b->period);
5364 :
5365 : /*
5366 : * Grow period by a factor of 2 to avoid losing precision.
5367 : * Precision loss in the quota/period ratio can cause __cfs_schedulable
5368 : * to fail.
5369 : */
5370 : new = old * 2;
5371 : if (new < max_cfs_quota_period) {
5372 : cfs_b->period = ns_to_ktime(new);
5373 : cfs_b->quota *= 2;
5374 : cfs_b->burst *= 2;
5375 :
5376 : pr_warn_ratelimited(
5377 : "cfs_period_timer[cpu%d]: period too short, scaling up (new cfs_period_us = %lld, cfs_quota_us = %lld)\n",
5378 : smp_processor_id(),
5379 : div_u64(new, NSEC_PER_USEC),
5380 : div_u64(cfs_b->quota, NSEC_PER_USEC));
5381 : } else {
5382 : pr_warn_ratelimited(
5383 : "cfs_period_timer[cpu%d]: period too short, but cannot scale up without losing precision (cfs_period_us = %lld, cfs_quota_us = %lld)\n",
5384 : smp_processor_id(),
5385 : div_u64(old, NSEC_PER_USEC),
5386 : div_u64(cfs_b->quota, NSEC_PER_USEC));
5387 : }
5388 :
5389 : /* reset count so we don't come right back in here */
5390 : count = 0;
5391 : }
5392 : }
5393 : if (idle)
5394 : cfs_b->period_active = 0;
5395 : raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
5396 :
5397 : return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
5398 : }
5399 :
5400 : void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
5401 : {
5402 : raw_spin_lock_init(&cfs_b->lock);
5403 : cfs_b->runtime = 0;
5404 : cfs_b->quota = RUNTIME_INF;
5405 : cfs_b->period = ns_to_ktime(default_cfs_period());
5406 : cfs_b->burst = 0;
5407 :
5408 : INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq);
5409 : hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);
5410 : cfs_b->period_timer.function = sched_cfs_period_timer;
5411 : hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
5412 : cfs_b->slack_timer.function = sched_cfs_slack_timer;
5413 : cfs_b->slack_started = false;
5414 : }
5415 :
5416 : static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
5417 : {
5418 : cfs_rq->runtime_enabled = 0;
5419 : INIT_LIST_HEAD(&cfs_rq->throttled_list);
5420 : }
5421 :
5422 : void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
5423 : {
5424 : lockdep_assert_held(&cfs_b->lock);
5425 :
5426 : if (cfs_b->period_active)
5427 : return;
5428 :
5429 : cfs_b->period_active = 1;
5430 : hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period);
5431 : hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED);
5432 : }
5433 :
5434 : static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
5435 : {
5436 : /* init_cfs_bandwidth() was not called */
5437 : if (!cfs_b->throttled_cfs_rq.next)
5438 : return;
5439 :
5440 : hrtimer_cancel(&cfs_b->period_timer);
5441 : hrtimer_cancel(&cfs_b->slack_timer);
5442 : }
5443 :
5444 : /*
5445 : * Both these CPU hotplug callbacks race against unregister_fair_sched_group()
5446 : *
5447 : * The race is harmless, since modifying bandwidth settings of unhooked group
5448 : * bits doesn't do much.
5449 : */
5450 :
5451 : /* cpu online callback */
5452 : static void __maybe_unused update_runtime_enabled(struct rq *rq)
5453 : {
5454 : struct task_group *tg;
5455 :
5456 : lockdep_assert_rq_held(rq);
5457 :
5458 : rcu_read_lock();
5459 : list_for_each_entry_rcu(tg, &task_groups, list) {
5460 : struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
5461 : struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
5462 :
5463 : raw_spin_lock(&cfs_b->lock);
5464 : cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF;
5465 : raw_spin_unlock(&cfs_b->lock);
5466 : }
5467 : rcu_read_unlock();
5468 : }
5469 :
5470 : /* cpu offline callback */
5471 : static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq)
5472 : {
5473 : struct task_group *tg;
5474 :
5475 : lockdep_assert_rq_held(rq);
5476 :
5477 : rcu_read_lock();
5478 : list_for_each_entry_rcu(tg, &task_groups, list) {
5479 : struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
5480 :
5481 : if (!cfs_rq->runtime_enabled)
5482 : continue;
5483 :
5484 : /*
5485 : * clock_task is not advancing so we just need to make sure
5486 : * there's some valid quota amount
5487 : */
5488 : cfs_rq->runtime_remaining = 1;
5489 : /*
5490 : * Offline rq is schedulable till CPU is completely disabled
5491 : * in take_cpu_down(), so we prevent new cfs throttling here.
5492 : */
5493 : cfs_rq->runtime_enabled = 0;
5494 :
5495 : if (cfs_rq_throttled(cfs_rq))
5496 : unthrottle_cfs_rq(cfs_rq);
5497 : }
5498 : rcu_read_unlock();
5499 : }
5500 :
5501 : #else /* CONFIG_CFS_BANDWIDTH */
5502 :
5503 : static inline bool cfs_bandwidth_used(void)
5504 : {
5505 : return false;
5506 : }
5507 :
5508 : static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {}
5509 : static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { return false; }
5510 : static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {}
5511 : static inline void sync_throttle(struct task_group *tg, int cpu) {}
5512 : static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
5513 :
5514 : static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
5515 : {
5516 : return 0;
5517 : }
5518 :
5519 : static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
5520 : {
5521 : return 0;
5522 : }
5523 :
5524 : static inline int throttled_lb_pair(struct task_group *tg,
5525 : int src_cpu, int dest_cpu)
5526 : {
5527 : return 0;
5528 : }
5529 :
5530 0 : void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
5531 :
5532 : #ifdef CONFIG_FAIR_GROUP_SCHED
5533 : static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
5534 : #endif
5535 :
5536 : static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
5537 : {
5538 : return NULL;
5539 : }
5540 : static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
5541 : static inline void update_runtime_enabled(struct rq *rq) {}
5542 : static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {}
5543 :
5544 : #endif /* CONFIG_CFS_BANDWIDTH */
5545 :
5546 : /**************************************************
5547 : * CFS operations on tasks:
5548 : */
5549 :
5550 : #ifdef CONFIG_SCHED_HRTICK
5551 : static void hrtick_start_fair(struct rq *rq, struct task_struct *p)
5552 : {
5553 : struct sched_entity *se = &p->se;
5554 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
5555 :
5556 : SCHED_WARN_ON(task_rq(p) != rq);
5557 :
5558 : if (rq->cfs.h_nr_running > 1) {
5559 : u64 slice = sched_slice(cfs_rq, se);
5560 : u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime;
5561 : s64 delta = slice - ran;
5562 :
5563 : if (delta < 0) {
5564 : if (task_current(rq, p))
5565 : resched_curr(rq);
5566 : return;
5567 : }
5568 : hrtick_start(rq, delta);
5569 : }
5570 : }
5571 :
5572 : /*
5573 : * called from enqueue/dequeue and updates the hrtick when the
5574 : * current task is from our class and nr_running is low enough
5575 : * to matter.
5576 : */
5577 : static void hrtick_update(struct rq *rq)
5578 : {
5579 : struct task_struct *curr = rq->curr;
5580 :
5581 : if (!hrtick_enabled_fair(rq) || curr->sched_class != &fair_sched_class)
5582 : return;
5583 :
5584 : if (cfs_rq_of(&curr->se)->nr_running < sched_nr_latency)
5585 : hrtick_start_fair(rq, curr);
5586 : }
5587 : #else /* !CONFIG_SCHED_HRTICK */
5588 : static inline void
5589 : hrtick_start_fair(struct rq *rq, struct task_struct *p)
5590 : {
5591 : }
5592 :
5593 : static inline void hrtick_update(struct rq *rq)
5594 : {
5595 : }
5596 : #endif
5597 :
5598 : #ifdef CONFIG_SMP
5599 : static inline bool cpu_overutilized(int cpu)
5600 : {
5601 : return !fits_capacity(cpu_util_cfs(cpu), capacity_of(cpu));
5602 : }
5603 :
5604 : static inline void update_overutilized_status(struct rq *rq)
5605 : {
5606 : if (!READ_ONCE(rq->rd->overutilized) && cpu_overutilized(rq->cpu)) {
5607 : WRITE_ONCE(rq->rd->overutilized, SG_OVERUTILIZED);
5608 : trace_sched_overutilized_tp(rq->rd, SG_OVERUTILIZED);
5609 : }
5610 : }
5611 : #else
5612 : static inline void update_overutilized_status(struct rq *rq) { }
5613 : #endif
5614 :
5615 : /* Runqueue only has SCHED_IDLE tasks enqueued */
5616 : static int sched_idle_rq(struct rq *rq)
5617 : {
5618 1236 : return unlikely(rq->nr_running == rq->cfs.idle_h_nr_running &&
5619 : rq->nr_running);
5620 : }
5621 :
5622 : /*
5623 : * Returns true if cfs_rq only has SCHED_IDLE entities enqueued. Note the use
5624 : * of idle_nr_running, which does not consider idle descendants of normal
5625 : * entities.
5626 : */
5627 : static bool sched_idle_cfs_rq(struct cfs_rq *cfs_rq)
5628 : {
5629 : return cfs_rq->nr_running &&
5630 : cfs_rq->nr_running == cfs_rq->idle_nr_running;
5631 : }
5632 :
5633 : #ifdef CONFIG_SMP
5634 : static int sched_idle_cpu(int cpu)
5635 : {
5636 : return sched_idle_rq(cpu_rq(cpu));
5637 : }
5638 : #endif
5639 :
5640 : /*
5641 : * The enqueue_task method is called before nr_running is
5642 : * increased. Here we update the fair scheduling stats and
5643 : * then put the task into the rbtree:
5644 : */
5645 : static void
5646 620 : enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
5647 : {
5648 : struct cfs_rq *cfs_rq;
5649 620 : struct sched_entity *se = &p->se;
5650 1240 : int idle_h_nr_running = task_has_idle_policy(p);
5651 620 : int task_new = !(flags & ENQUEUE_WAKEUP);
5652 :
5653 : /*
5654 : * The code below (indirectly) updates schedutil which looks at
5655 : * the cfs_rq utilization to select a frequency.
5656 : * Let's add the task's estimated utilization to the cfs_rq's
5657 : * estimated utilization, before we update schedutil.
5658 : */
5659 620 : util_est_enqueue(&rq->cfs, p);
5660 :
5661 : /*
5662 : * If in_iowait is set, the code below may not trigger any cpufreq
5663 : * utilization updates, so do it here explicitly with the IOWAIT flag
5664 : * passed.
5665 : */
5666 : if (p->in_iowait)
5667 : cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT);
5668 :
5669 1240 : for_each_sched_entity(se) {
5670 620 : if (se->on_rq)
5671 : break;
5672 1240 : cfs_rq = cfs_rq_of(se);
5673 620 : enqueue_entity(cfs_rq, se, flags);
5674 :
5675 620 : cfs_rq->h_nr_running++;
5676 620 : cfs_rq->idle_h_nr_running += idle_h_nr_running;
5677 :
5678 : if (cfs_rq_is_idle(cfs_rq))
5679 : idle_h_nr_running = 1;
5680 :
5681 : /* end evaluation on encountering a throttled cfs_rq */
5682 : if (cfs_rq_throttled(cfs_rq))
5683 : goto enqueue_throttle;
5684 :
5685 : flags = ENQUEUE_WAKEUP;
5686 : }
5687 :
5688 0 : for_each_sched_entity(se) {
5689 0 : cfs_rq = cfs_rq_of(se);
5690 :
5691 0 : update_load_avg(cfs_rq, se, UPDATE_TG);
5692 0 : se_update_runnable(se);
5693 0 : update_cfs_group(se);
5694 :
5695 0 : cfs_rq->h_nr_running++;
5696 0 : cfs_rq->idle_h_nr_running += idle_h_nr_running;
5697 :
5698 : if (cfs_rq_is_idle(cfs_rq))
5699 : idle_h_nr_running = 1;
5700 :
5701 : /* end evaluation on encountering a throttled cfs_rq */
5702 : if (cfs_rq_throttled(cfs_rq))
5703 : goto enqueue_throttle;
5704 :
5705 : /*
5706 : * One parent has been throttled and cfs_rq removed from the
5707 : * list. Add it back to not break the leaf list.
5708 : */
5709 : if (throttled_hierarchy(cfs_rq))
5710 : list_add_leaf_cfs_rq(cfs_rq);
5711 : }
5712 :
5713 : /* At this point se is NULL and we are at root level*/
5714 1240 : add_nr_running(rq, 1);
5715 :
5716 : /*
5717 : * Since new tasks are assigned an initial util_avg equal to
5718 : * half of the spare capacity of their CPU, tiny tasks have the
5719 : * ability to cross the overutilized threshold, which will
5720 : * result in the load balancer ruining all the task placement
5721 : * done by EAS. As a way to mitigate that effect, do not account
5722 : * for the first enqueue operation of new tasks during the
5723 : * overutilized flag detection.
5724 : *
5725 : * A better way of solving this problem would be to wait for
5726 : * the PELT signals of tasks to converge before taking them
5727 : * into account, but that is not straightforward to implement,
5728 : * and the following generally works well enough in practice.
5729 : */
5730 : if (!task_new)
5731 : update_overutilized_status(rq);
5732 :
5733 : enqueue_throttle:
5734 : if (cfs_bandwidth_used()) {
5735 : /*
5736 : * When bandwidth control is enabled; the cfs_rq_throttled()
5737 : * breaks in the above iteration can result in incomplete
5738 : * leaf list maintenance, resulting in triggering the assertion
5739 : * below.
5740 : */
5741 : for_each_sched_entity(se) {
5742 : cfs_rq = cfs_rq_of(se);
5743 :
5744 : if (list_add_leaf_cfs_rq(cfs_rq))
5745 : break;
5746 : }
5747 : }
5748 :
5749 620 : assert_list_leaf_cfs_rq(rq);
5750 :
5751 620 : hrtick_update(rq);
5752 620 : }
5753 :
5754 : static void set_next_buddy(struct sched_entity *se);
5755 :
5756 : /*
5757 : * The dequeue_task method is called before nr_running is
5758 : * decreased. We remove the task from the rbtree and
5759 : * update the fair scheduling stats:
5760 : */
5761 618 : static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
5762 : {
5763 : struct cfs_rq *cfs_rq;
5764 618 : struct sched_entity *se = &p->se;
5765 618 : int task_sleep = flags & DEQUEUE_SLEEP;
5766 1236 : int idle_h_nr_running = task_has_idle_policy(p);
5767 1236 : bool was_sched_idle = sched_idle_rq(rq);
5768 :
5769 618 : util_est_dequeue(&rq->cfs, p);
5770 :
5771 1 : for_each_sched_entity(se) {
5772 1236 : cfs_rq = cfs_rq_of(se);
5773 618 : dequeue_entity(cfs_rq, se, flags);
5774 :
5775 618 : cfs_rq->h_nr_running--;
5776 618 : cfs_rq->idle_h_nr_running -= idle_h_nr_running;
5777 :
5778 : if (cfs_rq_is_idle(cfs_rq))
5779 : idle_h_nr_running = 1;
5780 :
5781 : /* end evaluation on encountering a throttled cfs_rq */
5782 : if (cfs_rq_throttled(cfs_rq))
5783 : goto dequeue_throttle;
5784 :
5785 : /* Don't dequeue parent if it has other entities besides us */
5786 618 : if (cfs_rq->load.weight) {
5787 : /* Avoid re-evaluating load for this entity: */
5788 : se = parent_entity(se);
5789 : /*
5790 : * Bias pick_next to pick a task from this cfs_rq, as
5791 : * p is sleeping when it is within its sched_slice.
5792 : */
5793 : if (task_sleep && se && !throttled_hierarchy(cfs_rq))
5794 : set_next_buddy(se);
5795 : break;
5796 : }
5797 1 : flags |= DEQUEUE_SLEEP;
5798 : }
5799 :
5800 618 : for_each_sched_entity(se) {
5801 0 : cfs_rq = cfs_rq_of(se);
5802 :
5803 0 : update_load_avg(cfs_rq, se, UPDATE_TG);
5804 0 : se_update_runnable(se);
5805 0 : update_cfs_group(se);
5806 :
5807 0 : cfs_rq->h_nr_running--;
5808 0 : cfs_rq->idle_h_nr_running -= idle_h_nr_running;
5809 :
5810 : if (cfs_rq_is_idle(cfs_rq))
5811 : idle_h_nr_running = 1;
5812 :
5813 : /* end evaluation on encountering a throttled cfs_rq */
5814 : if (cfs_rq_throttled(cfs_rq))
5815 : goto dequeue_throttle;
5816 :
5817 : }
5818 :
5819 : /* At this point se is NULL and we are at root level*/
5820 1236 : sub_nr_running(rq, 1);
5821 :
5822 : /* balance early to pull high priority tasks */
5823 1236 : if (unlikely(!was_sched_idle && sched_idle_rq(rq)))
5824 0 : rq->next_balance = jiffies;
5825 :
5826 : dequeue_throttle:
5827 618 : util_est_update(&rq->cfs, p, task_sleep);
5828 618 : hrtick_update(rq);
5829 618 : }
5830 :
5831 : #ifdef CONFIG_SMP
5832 :
5833 : /* Working cpumask for: load_balance, load_balance_newidle. */
5834 : DEFINE_PER_CPU(cpumask_var_t, load_balance_mask);
5835 : DEFINE_PER_CPU(cpumask_var_t, select_idle_mask);
5836 :
5837 : #ifdef CONFIG_NO_HZ_COMMON
5838 :
5839 : static struct {
5840 : cpumask_var_t idle_cpus_mask;
5841 : atomic_t nr_cpus;
5842 : int has_blocked; /* Idle CPUS has blocked load */
5843 : int needs_update; /* Newly idle CPUs need their next_balance collated */
5844 : unsigned long next_balance; /* in jiffy units */
5845 : unsigned long next_blocked; /* Next update of blocked load in jiffies */
5846 : } nohz ____cacheline_aligned;
5847 :
5848 : #endif /* CONFIG_NO_HZ_COMMON */
5849 :
5850 : static unsigned long cpu_load(struct rq *rq)
5851 : {
5852 : return cfs_rq_load_avg(&rq->cfs);
5853 : }
5854 :
5855 : /*
5856 : * cpu_load_without - compute CPU load without any contributions from *p
5857 : * @cpu: the CPU which load is requested
5858 : * @p: the task which load should be discounted
5859 : *
5860 : * The load of a CPU is defined by the load of tasks currently enqueued on that
5861 : * CPU as well as tasks which are currently sleeping after an execution on that
5862 : * CPU.
5863 : *
5864 : * This method returns the load of the specified CPU by discounting the load of
5865 : * the specified task, whenever the task is currently contributing to the CPU
5866 : * load.
5867 : */
5868 : static unsigned long cpu_load_without(struct rq *rq, struct task_struct *p)
5869 : {
5870 : struct cfs_rq *cfs_rq;
5871 : unsigned int load;
5872 :
5873 : /* Task has no contribution or is new */
5874 : if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))
5875 : return cpu_load(rq);
5876 :
5877 : cfs_rq = &rq->cfs;
5878 : load = READ_ONCE(cfs_rq->avg.load_avg);
5879 :
5880 : /* Discount task's util from CPU's util */
5881 : lsub_positive(&load, task_h_load(p));
5882 :
5883 : return load;
5884 : }
5885 :
5886 : static unsigned long cpu_runnable(struct rq *rq)
5887 : {
5888 : return cfs_rq_runnable_avg(&rq->cfs);
5889 : }
5890 :
5891 : static unsigned long cpu_runnable_without(struct rq *rq, struct task_struct *p)
5892 : {
5893 : struct cfs_rq *cfs_rq;
5894 : unsigned int runnable;
5895 :
5896 : /* Task has no contribution or is new */
5897 : if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))
5898 : return cpu_runnable(rq);
5899 :
5900 : cfs_rq = &rq->cfs;
5901 : runnable = READ_ONCE(cfs_rq->avg.runnable_avg);
5902 :
5903 : /* Discount task's runnable from CPU's runnable */
5904 : lsub_positive(&runnable, p->se.avg.runnable_avg);
5905 :
5906 : return runnable;
5907 : }
5908 :
5909 : static unsigned long capacity_of(int cpu)
5910 : {
5911 : return cpu_rq(cpu)->cpu_capacity;
5912 : }
5913 :
5914 : static void record_wakee(struct task_struct *p)
5915 : {
5916 : /*
5917 : * Only decay a single time; tasks that have less then 1 wakeup per
5918 : * jiffy will not have built up many flips.
5919 : */
5920 : if (time_after(jiffies, current->wakee_flip_decay_ts + HZ)) {
5921 : current->wakee_flips >>= 1;
5922 : current->wakee_flip_decay_ts = jiffies;
5923 : }
5924 :
5925 : if (current->last_wakee != p) {
5926 : current->last_wakee = p;
5927 : current->wakee_flips++;
5928 : }
5929 : }
5930 :
5931 : /*
5932 : * Detect M:N waker/wakee relationships via a switching-frequency heuristic.
5933 : *
5934 : * A waker of many should wake a different task than the one last awakened
5935 : * at a frequency roughly N times higher than one of its wakees.
5936 : *
5937 : * In order to determine whether we should let the load spread vs consolidating
5938 : * to shared cache, we look for a minimum 'flip' frequency of llc_size in one
5939 : * partner, and a factor of lls_size higher frequency in the other.
5940 : *
5941 : * With both conditions met, we can be relatively sure that the relationship is
5942 : * non-monogamous, with partner count exceeding socket size.
5943 : *
5944 : * Waker/wakee being client/server, worker/dispatcher, interrupt source or
5945 : * whatever is irrelevant, spread criteria is apparent partner count exceeds
5946 : * socket size.
5947 : */
5948 : static int wake_wide(struct task_struct *p)
5949 : {
5950 : unsigned int master = current->wakee_flips;
5951 : unsigned int slave = p->wakee_flips;
5952 : int factor = __this_cpu_read(sd_llc_size);
5953 :
5954 : if (master < slave)
5955 : swap(master, slave);
5956 : if (slave < factor || master < slave * factor)
5957 : return 0;
5958 : return 1;
5959 : }
5960 :
5961 : /*
5962 : * The purpose of wake_affine() is to quickly determine on which CPU we can run
5963 : * soonest. For the purpose of speed we only consider the waking and previous
5964 : * CPU.
5965 : *
5966 : * wake_affine_idle() - only considers 'now', it check if the waking CPU is
5967 : * cache-affine and is (or will be) idle.
5968 : *
5969 : * wake_affine_weight() - considers the weight to reflect the average
5970 : * scheduling latency of the CPUs. This seems to work
5971 : * for the overloaded case.
5972 : */
5973 : static int
5974 : wake_affine_idle(int this_cpu, int prev_cpu, int sync)
5975 : {
5976 : /*
5977 : * If this_cpu is idle, it implies the wakeup is from interrupt
5978 : * context. Only allow the move if cache is shared. Otherwise an
5979 : * interrupt intensive workload could force all tasks onto one
5980 : * node depending on the IO topology or IRQ affinity settings.
5981 : *
5982 : * If the prev_cpu is idle and cache affine then avoid a migration.
5983 : * There is no guarantee that the cache hot data from an interrupt
5984 : * is more important than cache hot data on the prev_cpu and from
5985 : * a cpufreq perspective, it's better to have higher utilisation
5986 : * on one CPU.
5987 : */
5988 : if (available_idle_cpu(this_cpu) && cpus_share_cache(this_cpu, prev_cpu))
5989 : return available_idle_cpu(prev_cpu) ? prev_cpu : this_cpu;
5990 :
5991 : if (sync && cpu_rq(this_cpu)->nr_running == 1)
5992 : return this_cpu;
5993 :
5994 : if (available_idle_cpu(prev_cpu))
5995 : return prev_cpu;
5996 :
5997 : return nr_cpumask_bits;
5998 : }
5999 :
6000 : static int
6001 : wake_affine_weight(struct sched_domain *sd, struct task_struct *p,
6002 : int this_cpu, int prev_cpu, int sync)
6003 : {
6004 : s64 this_eff_load, prev_eff_load;
6005 : unsigned long task_load;
6006 :
6007 : this_eff_load = cpu_load(cpu_rq(this_cpu));
6008 :
6009 : if (sync) {
6010 : unsigned long current_load = task_h_load(current);
6011 :
6012 : if (current_load > this_eff_load)
6013 : return this_cpu;
6014 :
6015 : this_eff_load -= current_load;
6016 : }
6017 :
6018 : task_load = task_h_load(p);
6019 :
6020 : this_eff_load += task_load;
6021 : if (sched_feat(WA_BIAS))
6022 : this_eff_load *= 100;
6023 : this_eff_load *= capacity_of(prev_cpu);
6024 :
6025 : prev_eff_load = cpu_load(cpu_rq(prev_cpu));
6026 : prev_eff_load -= task_load;
6027 : if (sched_feat(WA_BIAS))
6028 : prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2;
6029 : prev_eff_load *= capacity_of(this_cpu);
6030 :
6031 : /*
6032 : * If sync, adjust the weight of prev_eff_load such that if
6033 : * prev_eff == this_eff that select_idle_sibling() will consider
6034 : * stacking the wakee on top of the waker if no other CPU is
6035 : * idle.
6036 : */
6037 : if (sync)
6038 : prev_eff_load += 1;
6039 :
6040 : return this_eff_load < prev_eff_load ? this_cpu : nr_cpumask_bits;
6041 : }
6042 :
6043 : static int wake_affine(struct sched_domain *sd, struct task_struct *p,
6044 : int this_cpu, int prev_cpu, int sync)
6045 : {
6046 : int target = nr_cpumask_bits;
6047 :
6048 : if (sched_feat(WA_IDLE))
6049 : target = wake_affine_idle(this_cpu, prev_cpu, sync);
6050 :
6051 : if (sched_feat(WA_WEIGHT) && target == nr_cpumask_bits)
6052 : target = wake_affine_weight(sd, p, this_cpu, prev_cpu, sync);
6053 :
6054 : schedstat_inc(p->stats.nr_wakeups_affine_attempts);
6055 : if (target == nr_cpumask_bits)
6056 : return prev_cpu;
6057 :
6058 : schedstat_inc(sd->ttwu_move_affine);
6059 : schedstat_inc(p->stats.nr_wakeups_affine);
6060 : return target;
6061 : }
6062 :
6063 : static struct sched_group *
6064 : find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu);
6065 :
6066 : /*
6067 : * find_idlest_group_cpu - find the idlest CPU among the CPUs in the group.
6068 : */
6069 : static int
6070 : find_idlest_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
6071 : {
6072 : unsigned long load, min_load = ULONG_MAX;
6073 : unsigned int min_exit_latency = UINT_MAX;
6074 : u64 latest_idle_timestamp = 0;
6075 : int least_loaded_cpu = this_cpu;
6076 : int shallowest_idle_cpu = -1;
6077 : int i;
6078 :
6079 : /* Check if we have any choice: */
6080 : if (group->group_weight == 1)
6081 : return cpumask_first(sched_group_span(group));
6082 :
6083 : /* Traverse only the allowed CPUs */
6084 : for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) {
6085 : struct rq *rq = cpu_rq(i);
6086 :
6087 : if (!sched_core_cookie_match(rq, p))
6088 : continue;
6089 :
6090 : if (sched_idle_cpu(i))
6091 : return i;
6092 :
6093 : if (available_idle_cpu(i)) {
6094 : struct cpuidle_state *idle = idle_get_state(rq);
6095 : if (idle && idle->exit_latency < min_exit_latency) {
6096 : /*
6097 : * We give priority to a CPU whose idle state
6098 : * has the smallest exit latency irrespective
6099 : * of any idle timestamp.
6100 : */
6101 : min_exit_latency = idle->exit_latency;
6102 : latest_idle_timestamp = rq->idle_stamp;
6103 : shallowest_idle_cpu = i;
6104 : } else if ((!idle || idle->exit_latency == min_exit_latency) &&
6105 : rq->idle_stamp > latest_idle_timestamp) {
6106 : /*
6107 : * If equal or no active idle state, then
6108 : * the most recently idled CPU might have
6109 : * a warmer cache.
6110 : */
6111 : latest_idle_timestamp = rq->idle_stamp;
6112 : shallowest_idle_cpu = i;
6113 : }
6114 : } else if (shallowest_idle_cpu == -1) {
6115 : load = cpu_load(cpu_rq(i));
6116 : if (load < min_load) {
6117 : min_load = load;
6118 : least_loaded_cpu = i;
6119 : }
6120 : }
6121 : }
6122 :
6123 : return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu;
6124 : }
6125 :
6126 : static inline int find_idlest_cpu(struct sched_domain *sd, struct task_struct *p,
6127 : int cpu, int prev_cpu, int sd_flag)
6128 : {
6129 : int new_cpu = cpu;
6130 :
6131 : if (!cpumask_intersects(sched_domain_span(sd), p->cpus_ptr))
6132 : return prev_cpu;
6133 :
6134 : /*
6135 : * We need task's util for cpu_util_without, sync it up to
6136 : * prev_cpu's last_update_time.
6137 : */
6138 : if (!(sd_flag & SD_BALANCE_FORK))
6139 : sync_entity_load_avg(&p->se);
6140 :
6141 : while (sd) {
6142 : struct sched_group *group;
6143 : struct sched_domain *tmp;
6144 : int weight;
6145 :
6146 : if (!(sd->flags & sd_flag)) {
6147 : sd = sd->child;
6148 : continue;
6149 : }
6150 :
6151 : group = find_idlest_group(sd, p, cpu);
6152 : if (!group) {
6153 : sd = sd->child;
6154 : continue;
6155 : }
6156 :
6157 : new_cpu = find_idlest_group_cpu(group, p, cpu);
6158 : if (new_cpu == cpu) {
6159 : /* Now try balancing at a lower domain level of 'cpu': */
6160 : sd = sd->child;
6161 : continue;
6162 : }
6163 :
6164 : /* Now try balancing at a lower domain level of 'new_cpu': */
6165 : cpu = new_cpu;
6166 : weight = sd->span_weight;
6167 : sd = NULL;
6168 : for_each_domain(cpu, tmp) {
6169 : if (weight <= tmp->span_weight)
6170 : break;
6171 : if (tmp->flags & sd_flag)
6172 : sd = tmp;
6173 : }
6174 : }
6175 :
6176 : return new_cpu;
6177 : }
6178 :
6179 : static inline int __select_idle_cpu(int cpu, struct task_struct *p)
6180 : {
6181 : if ((available_idle_cpu(cpu) || sched_idle_cpu(cpu)) &&
6182 : sched_cpu_cookie_match(cpu_rq(cpu), p))
6183 : return cpu;
6184 :
6185 : return -1;
6186 : }
6187 :
6188 : #ifdef CONFIG_SCHED_SMT
6189 : DEFINE_STATIC_KEY_FALSE(sched_smt_present);
6190 : EXPORT_SYMBOL_GPL(sched_smt_present);
6191 :
6192 : static inline void set_idle_cores(int cpu, int val)
6193 : {
6194 : struct sched_domain_shared *sds;
6195 :
6196 : sds = rcu_dereference(per_cpu(sd_llc_shared, cpu));
6197 : if (sds)
6198 : WRITE_ONCE(sds->has_idle_cores, val);
6199 : }
6200 :
6201 : static inline bool test_idle_cores(int cpu, bool def)
6202 : {
6203 : struct sched_domain_shared *sds;
6204 :
6205 : sds = rcu_dereference(per_cpu(sd_llc_shared, cpu));
6206 : if (sds)
6207 : return READ_ONCE(sds->has_idle_cores);
6208 :
6209 : return def;
6210 : }
6211 :
6212 : /*
6213 : * Scans the local SMT mask to see if the entire core is idle, and records this
6214 : * information in sd_llc_shared->has_idle_cores.
6215 : *
6216 : * Since SMT siblings share all cache levels, inspecting this limited remote
6217 : * state should be fairly cheap.
6218 : */
6219 : void __update_idle_core(struct rq *rq)
6220 : {
6221 : int core = cpu_of(rq);
6222 : int cpu;
6223 :
6224 : rcu_read_lock();
6225 : if (test_idle_cores(core, true))
6226 : goto unlock;
6227 :
6228 : for_each_cpu(cpu, cpu_smt_mask(core)) {
6229 : if (cpu == core)
6230 : continue;
6231 :
6232 : if (!available_idle_cpu(cpu))
6233 : goto unlock;
6234 : }
6235 :
6236 : set_idle_cores(core, 1);
6237 : unlock:
6238 : rcu_read_unlock();
6239 : }
6240 :
6241 : /*
6242 : * Scan the entire LLC domain for idle cores; this dynamically switches off if
6243 : * there are no idle cores left in the system; tracked through
6244 : * sd_llc->shared->has_idle_cores and enabled through update_idle_core() above.
6245 : */
6246 : static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu)
6247 : {
6248 : bool idle = true;
6249 : int cpu;
6250 :
6251 : if (!static_branch_likely(&sched_smt_present))
6252 : return __select_idle_cpu(core, p);
6253 :
6254 : for_each_cpu(cpu, cpu_smt_mask(core)) {
6255 : if (!available_idle_cpu(cpu)) {
6256 : idle = false;
6257 : if (*idle_cpu == -1) {
6258 : if (sched_idle_cpu(cpu) && cpumask_test_cpu(cpu, p->cpus_ptr)) {
6259 : *idle_cpu = cpu;
6260 : break;
6261 : }
6262 : continue;
6263 : }
6264 : break;
6265 : }
6266 : if (*idle_cpu == -1 && cpumask_test_cpu(cpu, p->cpus_ptr))
6267 : *idle_cpu = cpu;
6268 : }
6269 :
6270 : if (idle)
6271 : return core;
6272 :
6273 : cpumask_andnot(cpus, cpus, cpu_smt_mask(core));
6274 : return -1;
6275 : }
6276 :
6277 : /*
6278 : * Scan the local SMT mask for idle CPUs.
6279 : */
6280 : static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target)
6281 : {
6282 : int cpu;
6283 :
6284 : for_each_cpu(cpu, cpu_smt_mask(target)) {
6285 : if (!cpumask_test_cpu(cpu, p->cpus_ptr) ||
6286 : !cpumask_test_cpu(cpu, sched_domain_span(sd)))
6287 : continue;
6288 : if (available_idle_cpu(cpu) || sched_idle_cpu(cpu))
6289 : return cpu;
6290 : }
6291 :
6292 : return -1;
6293 : }
6294 :
6295 : #else /* CONFIG_SCHED_SMT */
6296 :
6297 : static inline void set_idle_cores(int cpu, int val)
6298 : {
6299 : }
6300 :
6301 : static inline bool test_idle_cores(int cpu, bool def)
6302 : {
6303 : return def;
6304 : }
6305 :
6306 : static inline int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu)
6307 : {
6308 : return __select_idle_cpu(core, p);
6309 : }
6310 :
6311 : static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target)
6312 : {
6313 : return -1;
6314 : }
6315 :
6316 : #endif /* CONFIG_SCHED_SMT */
6317 :
6318 : /*
6319 : * Scan the LLC domain for idle CPUs; this is dynamically regulated by
6320 : * comparing the average scan cost (tracked in sd->avg_scan_cost) against the
6321 : * average idle time for this rq (as found in rq->avg_idle).
6322 : */
6323 : static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool has_idle_core, int target)
6324 : {
6325 : struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_idle_mask);
6326 : int i, cpu, idle_cpu = -1, nr = INT_MAX;
6327 : struct rq *this_rq = this_rq();
6328 : int this = smp_processor_id();
6329 : struct sched_domain *this_sd;
6330 : u64 time = 0;
6331 :
6332 : this_sd = rcu_dereference(*this_cpu_ptr(&sd_llc));
6333 : if (!this_sd)
6334 : return -1;
6335 :
6336 : cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr);
6337 :
6338 : if (sched_feat(SIS_PROP) && !has_idle_core) {
6339 : u64 avg_cost, avg_idle, span_avg;
6340 : unsigned long now = jiffies;
6341 :
6342 : /*
6343 : * If we're busy, the assumption that the last idle period
6344 : * predicts the future is flawed; age away the remaining
6345 : * predicted idle time.
6346 : */
6347 : if (unlikely(this_rq->wake_stamp < now)) {
6348 : while (this_rq->wake_stamp < now && this_rq->wake_avg_idle) {
6349 : this_rq->wake_stamp++;
6350 : this_rq->wake_avg_idle >>= 1;
6351 : }
6352 : }
6353 :
6354 : avg_idle = this_rq->wake_avg_idle;
6355 : avg_cost = this_sd->avg_scan_cost + 1;
6356 :
6357 : span_avg = sd->span_weight * avg_idle;
6358 : if (span_avg > 4*avg_cost)
6359 : nr = div_u64(span_avg, avg_cost);
6360 : else
6361 : nr = 4;
6362 :
6363 : time = cpu_clock(this);
6364 : }
6365 :
6366 : for_each_cpu_wrap(cpu, cpus, target + 1) {
6367 : if (has_idle_core) {
6368 : i = select_idle_core(p, cpu, cpus, &idle_cpu);
6369 : if ((unsigned int)i < nr_cpumask_bits)
6370 : return i;
6371 :
6372 : } else {
6373 : if (!--nr)
6374 : return -1;
6375 : idle_cpu = __select_idle_cpu(cpu, p);
6376 : if ((unsigned int)idle_cpu < nr_cpumask_bits)
6377 : break;
6378 : }
6379 : }
6380 :
6381 : if (has_idle_core)
6382 : set_idle_cores(target, false);
6383 :
6384 : if (sched_feat(SIS_PROP) && !has_idle_core) {
6385 : time = cpu_clock(this) - time;
6386 :
6387 : /*
6388 : * Account for the scan cost of wakeups against the average
6389 : * idle time.
6390 : */
6391 : this_rq->wake_avg_idle -= min(this_rq->wake_avg_idle, time);
6392 :
6393 : update_avg(&this_sd->avg_scan_cost, time);
6394 : }
6395 :
6396 : return idle_cpu;
6397 : }
6398 :
6399 : /*
6400 : * Scan the asym_capacity domain for idle CPUs; pick the first idle one on which
6401 : * the task fits. If no CPU is big enough, but there are idle ones, try to
6402 : * maximize capacity.
6403 : */
6404 : static int
6405 : select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target)
6406 : {
6407 : unsigned long task_util, best_cap = 0;
6408 : int cpu, best_cpu = -1;
6409 : struct cpumask *cpus;
6410 :
6411 : cpus = this_cpu_cpumask_var_ptr(select_idle_mask);
6412 : cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr);
6413 :
6414 : task_util = uclamp_task_util(p);
6415 :
6416 : for_each_cpu_wrap(cpu, cpus, target) {
6417 : unsigned long cpu_cap = capacity_of(cpu);
6418 :
6419 : if (!available_idle_cpu(cpu) && !sched_idle_cpu(cpu))
6420 : continue;
6421 : if (fits_capacity(task_util, cpu_cap))
6422 : return cpu;
6423 :
6424 : if (cpu_cap > best_cap) {
6425 : best_cap = cpu_cap;
6426 : best_cpu = cpu;
6427 : }
6428 : }
6429 :
6430 : return best_cpu;
6431 : }
6432 :
6433 : static inline bool asym_fits_capacity(unsigned long task_util, int cpu)
6434 : {
6435 : if (static_branch_unlikely(&sched_asym_cpucapacity))
6436 : return fits_capacity(task_util, capacity_of(cpu));
6437 :
6438 : return true;
6439 : }
6440 :
6441 : /*
6442 : * Try and locate an idle core/thread in the LLC cache domain.
6443 : */
6444 : static int select_idle_sibling(struct task_struct *p, int prev, int target)
6445 : {
6446 : bool has_idle_core = false;
6447 : struct sched_domain *sd;
6448 : unsigned long task_util;
6449 : int i, recent_used_cpu;
6450 :
6451 : /*
6452 : * On asymmetric system, update task utilization because we will check
6453 : * that the task fits with cpu's capacity.
6454 : */
6455 : if (static_branch_unlikely(&sched_asym_cpucapacity)) {
6456 : sync_entity_load_avg(&p->se);
6457 : task_util = uclamp_task_util(p);
6458 : }
6459 :
6460 : /*
6461 : * per-cpu select_idle_mask usage
6462 : */
6463 : lockdep_assert_irqs_disabled();
6464 :
6465 : if ((available_idle_cpu(target) || sched_idle_cpu(target)) &&
6466 : asym_fits_capacity(task_util, target))
6467 : return target;
6468 :
6469 : /*
6470 : * If the previous CPU is cache affine and idle, don't be stupid:
6471 : */
6472 : if (prev != target && cpus_share_cache(prev, target) &&
6473 : (available_idle_cpu(prev) || sched_idle_cpu(prev)) &&
6474 : asym_fits_capacity(task_util, prev))
6475 : return prev;
6476 :
6477 : /*
6478 : * Allow a per-cpu kthread to stack with the wakee if the
6479 : * kworker thread and the tasks previous CPUs are the same.
6480 : * The assumption is that the wakee queued work for the
6481 : * per-cpu kthread that is now complete and the wakeup is
6482 : * essentially a sync wakeup. An obvious example of this
6483 : * pattern is IO completions.
6484 : */
6485 : if (is_per_cpu_kthread(current) &&
6486 : in_task() &&
6487 : prev == smp_processor_id() &&
6488 : this_rq()->nr_running <= 1 &&
6489 : asym_fits_capacity(task_util, prev)) {
6490 : return prev;
6491 : }
6492 :
6493 : /* Check a recently used CPU as a potential idle candidate: */
6494 : recent_used_cpu = p->recent_used_cpu;
6495 : p->recent_used_cpu = prev;
6496 : if (recent_used_cpu != prev &&
6497 : recent_used_cpu != target &&
6498 : cpus_share_cache(recent_used_cpu, target) &&
6499 : (available_idle_cpu(recent_used_cpu) || sched_idle_cpu(recent_used_cpu)) &&
6500 : cpumask_test_cpu(p->recent_used_cpu, p->cpus_ptr) &&
6501 : asym_fits_capacity(task_util, recent_used_cpu)) {
6502 : return recent_used_cpu;
6503 : }
6504 :
6505 : /*
6506 : * For asymmetric CPU capacity systems, our domain of interest is
6507 : * sd_asym_cpucapacity rather than sd_llc.
6508 : */
6509 : if (static_branch_unlikely(&sched_asym_cpucapacity)) {
6510 : sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, target));
6511 : /*
6512 : * On an asymmetric CPU capacity system where an exclusive
6513 : * cpuset defines a symmetric island (i.e. one unique
6514 : * capacity_orig value through the cpuset), the key will be set
6515 : * but the CPUs within that cpuset will not have a domain with
6516 : * SD_ASYM_CPUCAPACITY. These should follow the usual symmetric
6517 : * capacity path.
6518 : */
6519 : if (sd) {
6520 : i = select_idle_capacity(p, sd, target);
6521 : return ((unsigned)i < nr_cpumask_bits) ? i : target;
6522 : }
6523 : }
6524 :
6525 : sd = rcu_dereference(per_cpu(sd_llc, target));
6526 : if (!sd)
6527 : return target;
6528 :
6529 : if (sched_smt_active()) {
6530 : has_idle_core = test_idle_cores(target, false);
6531 :
6532 : if (!has_idle_core && cpus_share_cache(prev, target)) {
6533 : i = select_idle_smt(p, sd, prev);
6534 : if ((unsigned int)i < nr_cpumask_bits)
6535 : return i;
6536 : }
6537 : }
6538 :
6539 : i = select_idle_cpu(p, sd, has_idle_core, target);
6540 : if ((unsigned)i < nr_cpumask_bits)
6541 : return i;
6542 :
6543 : return target;
6544 : }
6545 :
6546 : /*
6547 : * cpu_util_without: compute cpu utilization without any contributions from *p
6548 : * @cpu: the CPU which utilization is requested
6549 : * @p: the task which utilization should be discounted
6550 : *
6551 : * The utilization of a CPU is defined by the utilization of tasks currently
6552 : * enqueued on that CPU as well as tasks which are currently sleeping after an
6553 : * execution on that CPU.
6554 : *
6555 : * This method returns the utilization of the specified CPU by discounting the
6556 : * utilization of the specified task, whenever the task is currently
6557 : * contributing to the CPU utilization.
6558 : */
6559 : static unsigned long cpu_util_without(int cpu, struct task_struct *p)
6560 : {
6561 : struct cfs_rq *cfs_rq;
6562 : unsigned int util;
6563 :
6564 : /* Task has no contribution or is new */
6565 : if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))
6566 : return cpu_util_cfs(cpu);
6567 :
6568 : cfs_rq = &cpu_rq(cpu)->cfs;
6569 : util = READ_ONCE(cfs_rq->avg.util_avg);
6570 :
6571 : /* Discount task's util from CPU's util */
6572 : lsub_positive(&util, task_util(p));
6573 :
6574 : /*
6575 : * Covered cases:
6576 : *
6577 : * a) if *p is the only task sleeping on this CPU, then:
6578 : * cpu_util (== task_util) > util_est (== 0)
6579 : * and thus we return:
6580 : * cpu_util_without = (cpu_util - task_util) = 0
6581 : *
6582 : * b) if other tasks are SLEEPING on this CPU, which is now exiting
6583 : * IDLE, then:
6584 : * cpu_util >= task_util
6585 : * cpu_util > util_est (== 0)
6586 : * and thus we discount *p's blocked utilization to return:
6587 : * cpu_util_without = (cpu_util - task_util) >= 0
6588 : *
6589 : * c) if other tasks are RUNNABLE on that CPU and
6590 : * util_est > cpu_util
6591 : * then we use util_est since it returns a more restrictive
6592 : * estimation of the spare capacity on that CPU, by just
6593 : * considering the expected utilization of tasks already
6594 : * runnable on that CPU.
6595 : *
6596 : * Cases a) and b) are covered by the above code, while case c) is
6597 : * covered by the following code when estimated utilization is
6598 : * enabled.
6599 : */
6600 : if (sched_feat(UTIL_EST)) {
6601 : unsigned int estimated =
6602 : READ_ONCE(cfs_rq->avg.util_est.enqueued);
6603 :
6604 : /*
6605 : * Despite the following checks we still have a small window
6606 : * for a possible race, when an execl's select_task_rq_fair()
6607 : * races with LB's detach_task():
6608 : *
6609 : * detach_task()
6610 : * p->on_rq = TASK_ON_RQ_MIGRATING;
6611 : * ---------------------------------- A
6612 : * deactivate_task() \
6613 : * dequeue_task() + RaceTime
6614 : * util_est_dequeue() /
6615 : * ---------------------------------- B
6616 : *
6617 : * The additional check on "current == p" it's required to
6618 : * properly fix the execl regression and it helps in further
6619 : * reducing the chances for the above race.
6620 : */
6621 : if (unlikely(task_on_rq_queued(p) || current == p))
6622 : lsub_positive(&estimated, _task_util_est(p));
6623 :
6624 : util = max(util, estimated);
6625 : }
6626 :
6627 : /*
6628 : * Utilization (estimated) can exceed the CPU capacity, thus let's
6629 : * clamp to the maximum CPU capacity to ensure consistency with
6630 : * cpu_util.
6631 : */
6632 : return min_t(unsigned long, util, capacity_orig_of(cpu));
6633 : }
6634 :
6635 : /*
6636 : * Predicts what cpu_util(@cpu) would return if @p was migrated (and enqueued)
6637 : * to @dst_cpu.
6638 : */
6639 : static unsigned long cpu_util_next(int cpu, struct task_struct *p, int dst_cpu)
6640 : {
6641 : struct cfs_rq *cfs_rq = &cpu_rq(cpu)->cfs;
6642 : unsigned long util_est, util = READ_ONCE(cfs_rq->avg.util_avg);
6643 :
6644 : /*
6645 : * If @p migrates from @cpu to another, remove its contribution. Or,
6646 : * if @p migrates from another CPU to @cpu, add its contribution. In
6647 : * the other cases, @cpu is not impacted by the migration, so the
6648 : * util_avg should already be correct.
6649 : */
6650 : if (task_cpu(p) == cpu && dst_cpu != cpu)
6651 : lsub_positive(&util, task_util(p));
6652 : else if (task_cpu(p) != cpu && dst_cpu == cpu)
6653 : util += task_util(p);
6654 :
6655 : if (sched_feat(UTIL_EST)) {
6656 : util_est = READ_ONCE(cfs_rq->avg.util_est.enqueued);
6657 :
6658 : /*
6659 : * During wake-up, the task isn't enqueued yet and doesn't
6660 : * appear in the cfs_rq->avg.util_est.enqueued of any rq,
6661 : * so just add it (if needed) to "simulate" what will be
6662 : * cpu_util after the task has been enqueued.
6663 : */
6664 : if (dst_cpu == cpu)
6665 : util_est += _task_util_est(p);
6666 :
6667 : util = max(util, util_est);
6668 : }
6669 :
6670 : return min(util, capacity_orig_of(cpu));
6671 : }
6672 :
6673 : /*
6674 : * compute_energy(): Estimates the energy that @pd would consume if @p was
6675 : * migrated to @dst_cpu. compute_energy() predicts what will be the utilization
6676 : * landscape of @pd's CPUs after the task migration, and uses the Energy Model
6677 : * to compute what would be the energy if we decided to actually migrate that
6678 : * task.
6679 : */
6680 : static long
6681 : compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd)
6682 : {
6683 : struct cpumask *pd_mask = perf_domain_span(pd);
6684 : unsigned long cpu_cap = arch_scale_cpu_capacity(cpumask_first(pd_mask));
6685 : unsigned long max_util = 0, sum_util = 0;
6686 : unsigned long _cpu_cap = cpu_cap;
6687 : int cpu;
6688 :
6689 : _cpu_cap -= arch_scale_thermal_pressure(cpumask_first(pd_mask));
6690 :
6691 : /*
6692 : * The capacity state of CPUs of the current rd can be driven by CPUs
6693 : * of another rd if they belong to the same pd. So, account for the
6694 : * utilization of these CPUs too by masking pd with cpu_online_mask
6695 : * instead of the rd span.
6696 : *
6697 : * If an entire pd is outside of the current rd, it will not appear in
6698 : * its pd list and will not be accounted by compute_energy().
6699 : */
6700 : for_each_cpu_and(cpu, pd_mask, cpu_online_mask) {
6701 : unsigned long util_freq = cpu_util_next(cpu, p, dst_cpu);
6702 : unsigned long cpu_util, util_running = util_freq;
6703 : struct task_struct *tsk = NULL;
6704 :
6705 : /*
6706 : * When @p is placed on @cpu:
6707 : *
6708 : * util_running = max(cpu_util, cpu_util_est) +
6709 : * max(task_util, _task_util_est)
6710 : *
6711 : * while cpu_util_next is: max(cpu_util + task_util,
6712 : * cpu_util_est + _task_util_est)
6713 : */
6714 : if (cpu == dst_cpu) {
6715 : tsk = p;
6716 : util_running =
6717 : cpu_util_next(cpu, p, -1) + task_util_est(p);
6718 : }
6719 :
6720 : /*
6721 : * Busy time computation: utilization clamping is not
6722 : * required since the ratio (sum_util / cpu_capacity)
6723 : * is already enough to scale the EM reported power
6724 : * consumption at the (eventually clamped) cpu_capacity.
6725 : */
6726 : cpu_util = effective_cpu_util(cpu, util_running, cpu_cap,
6727 : ENERGY_UTIL, NULL);
6728 :
6729 : sum_util += min(cpu_util, _cpu_cap);
6730 :
6731 : /*
6732 : * Performance domain frequency: utilization clamping
6733 : * must be considered since it affects the selection
6734 : * of the performance domain frequency.
6735 : * NOTE: in case RT tasks are running, by default the
6736 : * FREQUENCY_UTIL's utilization can be max OPP.
6737 : */
6738 : cpu_util = effective_cpu_util(cpu, util_freq, cpu_cap,
6739 : FREQUENCY_UTIL, tsk);
6740 : max_util = max(max_util, min(cpu_util, _cpu_cap));
6741 : }
6742 :
6743 : return em_cpu_energy(pd->em_pd, max_util, sum_util, _cpu_cap);
6744 : }
6745 :
6746 : /*
6747 : * find_energy_efficient_cpu(): Find most energy-efficient target CPU for the
6748 : * waking task. find_energy_efficient_cpu() looks for the CPU with maximum
6749 : * spare capacity in each performance domain and uses it as a potential
6750 : * candidate to execute the task. Then, it uses the Energy Model to figure
6751 : * out which of the CPU candidates is the most energy-efficient.
6752 : *
6753 : * The rationale for this heuristic is as follows. In a performance domain,
6754 : * all the most energy efficient CPU candidates (according to the Energy
6755 : * Model) are those for which we'll request a low frequency. When there are
6756 : * several CPUs for which the frequency request will be the same, we don't
6757 : * have enough data to break the tie between them, because the Energy Model
6758 : * only includes active power costs. With this model, if we assume that
6759 : * frequency requests follow utilization (e.g. using schedutil), the CPU with
6760 : * the maximum spare capacity in a performance domain is guaranteed to be among
6761 : * the best candidates of the performance domain.
6762 : *
6763 : * In practice, it could be preferable from an energy standpoint to pack
6764 : * small tasks on a CPU in order to let other CPUs go in deeper idle states,
6765 : * but that could also hurt our chances to go cluster idle, and we have no
6766 : * ways to tell with the current Energy Model if this is actually a good
6767 : * idea or not. So, find_energy_efficient_cpu() basically favors
6768 : * cluster-packing, and spreading inside a cluster. That should at least be
6769 : * a good thing for latency, and this is consistent with the idea that most
6770 : * of the energy savings of EAS come from the asymmetry of the system, and
6771 : * not so much from breaking the tie between identical CPUs. That's also the
6772 : * reason why EAS is enabled in the topology code only for systems where
6773 : * SD_ASYM_CPUCAPACITY is set.
6774 : *
6775 : * NOTE: Forkees are not accepted in the energy-aware wake-up path because
6776 : * they don't have any useful utilization data yet and it's not possible to
6777 : * forecast their impact on energy consumption. Consequently, they will be
6778 : * placed by find_idlest_cpu() on the least loaded CPU, which might turn out
6779 : * to be energy-inefficient in some use-cases. The alternative would be to
6780 : * bias new tasks towards specific types of CPUs first, or to try to infer
6781 : * their util_avg from the parent task, but those heuristics could hurt
6782 : * other use-cases too. So, until someone finds a better way to solve this,
6783 : * let's keep things simple by re-using the existing slow path.
6784 : */
6785 : static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu)
6786 : {
6787 : unsigned long prev_delta = ULONG_MAX, best_delta = ULONG_MAX;
6788 : struct root_domain *rd = cpu_rq(smp_processor_id())->rd;
6789 : int cpu, best_energy_cpu = prev_cpu, target = -1;
6790 : unsigned long cpu_cap, util, base_energy = 0;
6791 : struct sched_domain *sd;
6792 : struct perf_domain *pd;
6793 :
6794 : rcu_read_lock();
6795 : pd = rcu_dereference(rd->pd);
6796 : if (!pd || READ_ONCE(rd->overutilized))
6797 : goto unlock;
6798 :
6799 : /*
6800 : * Energy-aware wake-up happens on the lowest sched_domain starting
6801 : * from sd_asym_cpucapacity spanning over this_cpu and prev_cpu.
6802 : */
6803 : sd = rcu_dereference(*this_cpu_ptr(&sd_asym_cpucapacity));
6804 : while (sd && !cpumask_test_cpu(prev_cpu, sched_domain_span(sd)))
6805 : sd = sd->parent;
6806 : if (!sd)
6807 : goto unlock;
6808 :
6809 : target = prev_cpu;
6810 :
6811 : sync_entity_load_avg(&p->se);
6812 : if (!task_util_est(p))
6813 : goto unlock;
6814 :
6815 : for (; pd; pd = pd->next) {
6816 : unsigned long cur_delta, spare_cap, max_spare_cap = 0;
6817 : bool compute_prev_delta = false;
6818 : unsigned long base_energy_pd;
6819 : int max_spare_cap_cpu = -1;
6820 :
6821 : for_each_cpu_and(cpu, perf_domain_span(pd), sched_domain_span(sd)) {
6822 : if (!cpumask_test_cpu(cpu, p->cpus_ptr))
6823 : continue;
6824 :
6825 : util = cpu_util_next(cpu, p, cpu);
6826 : cpu_cap = capacity_of(cpu);
6827 : spare_cap = cpu_cap;
6828 : lsub_positive(&spare_cap, util);
6829 :
6830 : /*
6831 : * Skip CPUs that cannot satisfy the capacity request.
6832 : * IOW, placing the task there would make the CPU
6833 : * overutilized. Take uclamp into account to see how
6834 : * much capacity we can get out of the CPU; this is
6835 : * aligned with sched_cpu_util().
6836 : */
6837 : util = uclamp_rq_util_with(cpu_rq(cpu), util, p);
6838 : if (!fits_capacity(util, cpu_cap))
6839 : continue;
6840 :
6841 : if (cpu == prev_cpu) {
6842 : /* Always use prev_cpu as a candidate. */
6843 : compute_prev_delta = true;
6844 : } else if (spare_cap > max_spare_cap) {
6845 : /*
6846 : * Find the CPU with the maximum spare capacity
6847 : * in the performance domain.
6848 : */
6849 : max_spare_cap = spare_cap;
6850 : max_spare_cap_cpu = cpu;
6851 : }
6852 : }
6853 :
6854 : if (max_spare_cap_cpu < 0 && !compute_prev_delta)
6855 : continue;
6856 :
6857 : /* Compute the 'base' energy of the pd, without @p */
6858 : base_energy_pd = compute_energy(p, -1, pd);
6859 : base_energy += base_energy_pd;
6860 :
6861 : /* Evaluate the energy impact of using prev_cpu. */
6862 : if (compute_prev_delta) {
6863 : prev_delta = compute_energy(p, prev_cpu, pd);
6864 : if (prev_delta < base_energy_pd)
6865 : goto unlock;
6866 : prev_delta -= base_energy_pd;
6867 : best_delta = min(best_delta, prev_delta);
6868 : }
6869 :
6870 : /* Evaluate the energy impact of using max_spare_cap_cpu. */
6871 : if (max_spare_cap_cpu >= 0) {
6872 : cur_delta = compute_energy(p, max_spare_cap_cpu, pd);
6873 : if (cur_delta < base_energy_pd)
6874 : goto unlock;
6875 : cur_delta -= base_energy_pd;
6876 : if (cur_delta < best_delta) {
6877 : best_delta = cur_delta;
6878 : best_energy_cpu = max_spare_cap_cpu;
6879 : }
6880 : }
6881 : }
6882 : rcu_read_unlock();
6883 :
6884 : /*
6885 : * Pick the best CPU if prev_cpu cannot be used, or if it saves at
6886 : * least 6% of the energy used by prev_cpu.
6887 : */
6888 : if ((prev_delta == ULONG_MAX) ||
6889 : (prev_delta - best_delta) > ((prev_delta + base_energy) >> 4))
6890 : target = best_energy_cpu;
6891 :
6892 : return target;
6893 :
6894 : unlock:
6895 : rcu_read_unlock();
6896 :
6897 : return target;
6898 : }
6899 :
6900 : /*
6901 : * select_task_rq_fair: Select target runqueue for the waking task in domains
6902 : * that have the relevant SD flag set. In practice, this is SD_BALANCE_WAKE,
6903 : * SD_BALANCE_FORK, or SD_BALANCE_EXEC.
6904 : *
6905 : * Balances load by selecting the idlest CPU in the idlest group, or under
6906 : * certain conditions an idle sibling CPU if the domain has SD_WAKE_AFFINE set.
6907 : *
6908 : * Returns the target CPU number.
6909 : */
6910 : static int
6911 : select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags)
6912 : {
6913 : int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING);
6914 : struct sched_domain *tmp, *sd = NULL;
6915 : int cpu = smp_processor_id();
6916 : int new_cpu = prev_cpu;
6917 : int want_affine = 0;
6918 : /* SD_flags and WF_flags share the first nibble */
6919 : int sd_flag = wake_flags & 0xF;
6920 :
6921 : /*
6922 : * required for stable ->cpus_allowed
6923 : */
6924 : lockdep_assert_held(&p->pi_lock);
6925 : if (wake_flags & WF_TTWU) {
6926 : record_wakee(p);
6927 :
6928 : if (sched_energy_enabled()) {
6929 : new_cpu = find_energy_efficient_cpu(p, prev_cpu);
6930 : if (new_cpu >= 0)
6931 : return new_cpu;
6932 : new_cpu = prev_cpu;
6933 : }
6934 :
6935 : want_affine = !wake_wide(p) && cpumask_test_cpu(cpu, p->cpus_ptr);
6936 : }
6937 :
6938 : rcu_read_lock();
6939 : for_each_domain(cpu, tmp) {
6940 : /*
6941 : * If both 'cpu' and 'prev_cpu' are part of this domain,
6942 : * cpu is a valid SD_WAKE_AFFINE target.
6943 : */
6944 : if (want_affine && (tmp->flags & SD_WAKE_AFFINE) &&
6945 : cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) {
6946 : if (cpu != prev_cpu)
6947 : new_cpu = wake_affine(tmp, p, cpu, prev_cpu, sync);
6948 :
6949 : sd = NULL; /* Prefer wake_affine over balance flags */
6950 : break;
6951 : }
6952 :
6953 : /*
6954 : * Usually only true for WF_EXEC and WF_FORK, as sched_domains
6955 : * usually do not have SD_BALANCE_WAKE set. That means wakeup
6956 : * will usually go to the fast path.
6957 : */
6958 : if (tmp->flags & sd_flag)
6959 : sd = tmp;
6960 : else if (!want_affine)
6961 : break;
6962 : }
6963 :
6964 : if (unlikely(sd)) {
6965 : /* Slow path */
6966 : new_cpu = find_idlest_cpu(sd, p, cpu, prev_cpu, sd_flag);
6967 : } else if (wake_flags & WF_TTWU) { /* XXX always ? */
6968 : /* Fast path */
6969 : new_cpu = select_idle_sibling(p, prev_cpu, new_cpu);
6970 : }
6971 : rcu_read_unlock();
6972 :
6973 : return new_cpu;
6974 : }
6975 :
6976 : static void detach_entity_cfs_rq(struct sched_entity *se);
6977 :
6978 : /*
6979 : * Called immediately before a task is migrated to a new CPU; task_cpu(p) and
6980 : * cfs_rq_of(p) references at time of call are still valid and identify the
6981 : * previous CPU. The caller guarantees p->pi_lock or task_rq(p)->lock is held.
6982 : */
6983 : static void migrate_task_rq_fair(struct task_struct *p, int new_cpu)
6984 : {
6985 : /*
6986 : * As blocked tasks retain absolute vruntime the migration needs to
6987 : * deal with this by subtracting the old and adding the new
6988 : * min_vruntime -- the latter is done by enqueue_entity() when placing
6989 : * the task on the new runqueue.
6990 : */
6991 : if (READ_ONCE(p->__state) == TASK_WAKING) {
6992 : struct sched_entity *se = &p->se;
6993 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
6994 : u64 min_vruntime;
6995 :
6996 : #ifndef CONFIG_64BIT
6997 : u64 min_vruntime_copy;
6998 :
6999 : do {
7000 : min_vruntime_copy = cfs_rq->min_vruntime_copy;
7001 : smp_rmb();
7002 : min_vruntime = cfs_rq->min_vruntime;
7003 : } while (min_vruntime != min_vruntime_copy);
7004 : #else
7005 : min_vruntime = cfs_rq->min_vruntime;
7006 : #endif
7007 :
7008 : se->vruntime -= min_vruntime;
7009 : }
7010 :
7011 : if (p->on_rq == TASK_ON_RQ_MIGRATING) {
7012 : /*
7013 : * In case of TASK_ON_RQ_MIGRATING we in fact hold the 'old'
7014 : * rq->lock and can modify state directly.
7015 : */
7016 : lockdep_assert_rq_held(task_rq(p));
7017 : detach_entity_cfs_rq(&p->se);
7018 :
7019 : } else {
7020 : /*
7021 : * We are supposed to update the task to "current" time, then
7022 : * its up to date and ready to go to new CPU/cfs_rq. But we
7023 : * have difficulty in getting what current time is, so simply
7024 : * throw away the out-of-date time. This will result in the
7025 : * wakee task is less decayed, but giving the wakee more load
7026 : * sounds not bad.
7027 : */
7028 : remove_entity_load_avg(&p->se);
7029 : }
7030 :
7031 : /* Tell new CPU we are migrated */
7032 : p->se.avg.last_update_time = 0;
7033 :
7034 : /* We have migrated, no longer consider this task hot */
7035 : p->se.exec_start = 0;
7036 :
7037 : update_scan_period(p, new_cpu);
7038 : }
7039 :
7040 : static void task_dead_fair(struct task_struct *p)
7041 : {
7042 : remove_entity_load_avg(&p->se);
7043 : }
7044 :
7045 : static int
7046 : balance_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
7047 : {
7048 : if (rq->nr_running)
7049 : return 1;
7050 :
7051 : return newidle_balance(rq, rf) != 0;
7052 : }
7053 : #endif /* CONFIG_SMP */
7054 :
7055 : static unsigned long wakeup_gran(struct sched_entity *se)
7056 : {
7057 203 : unsigned long gran = sysctl_sched_wakeup_granularity;
7058 :
7059 : /*
7060 : * Since its curr running now, convert the gran from real-time
7061 : * to virtual-time in his units.
7062 : *
7063 : * By using 'se' instead of 'curr' we penalize light tasks, so
7064 : * they get preempted easier. That is, if 'se' < 'curr' then
7065 : * the resulting gran will be larger, therefore penalizing the
7066 : * lighter, if otoh 'se' > 'curr' then the resulting gran will
7067 : * be smaller, again penalizing the lighter task.
7068 : *
7069 : * This is especially important for buddies when the leftmost
7070 : * task is higher priority than the buddy.
7071 : */
7072 203 : return calc_delta_fair(gran, se);
7073 : }
7074 :
7075 : /*
7076 : * Should 'se' preempt 'curr'.
7077 : *
7078 : * |s1
7079 : * |s2
7080 : * |s3
7081 : * g
7082 : * |<--->|c
7083 : *
7084 : * w(c, s1) = -1
7085 : * w(c, s2) = 0
7086 : * w(c, s3) = 1
7087 : *
7088 : */
7089 : static int
7090 716 : wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se)
7091 : {
7092 716 : s64 gran, vdiff = curr->vruntime - se->vruntime;
7093 :
7094 716 : if (vdiff <= 0)
7095 : return -1;
7096 :
7097 203 : gran = wakeup_gran(se);
7098 203 : if (vdiff > gran)
7099 : return 1;
7100 :
7101 : return 0;
7102 : }
7103 :
7104 0 : static void set_last_buddy(struct sched_entity *se)
7105 : {
7106 0 : for_each_sched_entity(se) {
7107 0 : if (SCHED_WARN_ON(!se->on_rq))
7108 : return;
7109 0 : if (se_is_idle(se))
7110 : return;
7111 0 : cfs_rq_of(se)->last = se;
7112 : }
7113 : }
7114 :
7115 200 : static void set_next_buddy(struct sched_entity *se)
7116 : {
7117 400 : for_each_sched_entity(se) {
7118 200 : if (SCHED_WARN_ON(!se->on_rq))
7119 : return;
7120 200 : if (se_is_idle(se))
7121 : return;
7122 400 : cfs_rq_of(se)->next = se;
7123 : }
7124 : }
7125 :
7126 : static void set_skip_buddy(struct sched_entity *se)
7127 : {
7128 0 : for_each_sched_entity(se)
7129 0 : cfs_rq_of(se)->skip = se;
7130 : }
7131 :
7132 : /*
7133 : * Preempt the current task with a newly woken task if needed:
7134 : */
7135 614 : static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_flags)
7136 : {
7137 614 : struct task_struct *curr = rq->curr;
7138 614 : struct sched_entity *se = &curr->se, *pse = &p->se;
7139 1228 : struct cfs_rq *cfs_rq = task_cfs_rq(curr);
7140 614 : int scale = cfs_rq->nr_running >= sched_nr_latency;
7141 614 : int next_buddy_marked = 0;
7142 : int cse_is_idle, pse_is_idle;
7143 :
7144 614 : if (unlikely(se == pse))
7145 : return;
7146 :
7147 : /*
7148 : * This is possible from callers such as attach_tasks(), in which we
7149 : * unconditionally check_preempt_curr() after an enqueue (which may have
7150 : * lead to a throttle). This both saves work and prevents false
7151 : * next-buddy nomination below.
7152 : */
7153 614 : if (unlikely(throttled_hierarchy(cfs_rq_of(pse))))
7154 : return;
7155 :
7156 614 : if (sched_feat(NEXT_BUDDY) && scale && !(wake_flags & WF_FORK)) {
7157 0 : set_next_buddy(pse);
7158 0 : next_buddy_marked = 1;
7159 : }
7160 :
7161 : /*
7162 : * We can come here with TIF_NEED_RESCHED already set from new task
7163 : * wake up path.
7164 : *
7165 : * Note: this also catches the edge-case of curr being in a throttled
7166 : * group (e.g. via set_curr_task), since update_curr() (in the
7167 : * enqueue of curr) will have resulted in resched being set. This
7168 : * prevents us from potentially nominating it as a false LAST_BUDDY
7169 : * below.
7170 : */
7171 614 : if (test_tsk_need_resched(curr))
7172 : return;
7173 :
7174 : /* Idle tasks are by definition preempted by non-idle tasks. */
7175 1032 : if (unlikely(task_has_idle_policy(curr)) &&
7176 0 : likely(!task_has_idle_policy(p)))
7177 : goto preempt;
7178 :
7179 : /*
7180 : * Batch and idle tasks do not preempt non-idle tasks (their preemption
7181 : * is driven by the tick):
7182 : */
7183 516 : if (unlikely(p->policy != SCHED_NORMAL) || !sched_feat(WAKEUP_PREEMPTION))
7184 : return;
7185 :
7186 516 : find_matching_se(&se, &pse);
7187 516 : BUG_ON(!pse);
7188 :
7189 516 : cse_is_idle = se_is_idle(se);
7190 516 : pse_is_idle = se_is_idle(pse);
7191 :
7192 : /*
7193 : * Preempt an idle group in favor of a non-idle group (and don't preempt
7194 : * in the inverse case).
7195 : */
7196 : if (cse_is_idle && !pse_is_idle)
7197 : goto preempt;
7198 : if (cse_is_idle != pse_is_idle)
7199 : return;
7200 :
7201 1032 : update_curr(cfs_rq_of(se));
7202 516 : if (wakeup_preempt_entity(se, pse) == 1) {
7203 : /*
7204 : * Bias pick_next to pick the sched entity that is
7205 : * triggering this preemption.
7206 : */
7207 200 : if (!next_buddy_marked)
7208 200 : set_next_buddy(pse);
7209 : goto preempt;
7210 : }
7211 :
7212 : return;
7213 :
7214 : preempt:
7215 200 : resched_curr(rq);
7216 : /*
7217 : * Only set the backward buddy when the current task is still
7218 : * on the rq. This can happen when a wakeup gets interleaved
7219 : * with schedule on the ->pre_schedule() or idle_balance()
7220 : * point, either of which can * drop the rq lock.
7221 : *
7222 : * Also, during early boot the idle thread is in the fair class,
7223 : * for obvious reasons its a bad idea to schedule back to it.
7224 : */
7225 200 : if (unlikely(!se->on_rq || curr == rq->idle))
7226 : return;
7227 :
7228 200 : if (sched_feat(LAST_BUDDY) && scale && entity_is_task(se))
7229 0 : set_last_buddy(se);
7230 : }
7231 :
7232 : #ifdef CONFIG_SMP
7233 : static struct task_struct *pick_task_fair(struct rq *rq)
7234 : {
7235 : struct sched_entity *se;
7236 : struct cfs_rq *cfs_rq;
7237 :
7238 : again:
7239 : cfs_rq = &rq->cfs;
7240 : if (!cfs_rq->nr_running)
7241 : return NULL;
7242 :
7243 : do {
7244 : struct sched_entity *curr = cfs_rq->curr;
7245 :
7246 : /* When we pick for a remote RQ, we'll not have done put_prev_entity() */
7247 : if (curr) {
7248 : if (curr->on_rq)
7249 : update_curr(cfs_rq);
7250 : else
7251 : curr = NULL;
7252 :
7253 : if (unlikely(check_cfs_rq_runtime(cfs_rq)))
7254 : goto again;
7255 : }
7256 :
7257 : se = pick_next_entity(cfs_rq, curr);
7258 : cfs_rq = group_cfs_rq(se);
7259 : } while (cfs_rq);
7260 :
7261 : return task_of(se);
7262 : }
7263 : #endif
7264 :
7265 : struct task_struct *
7266 618 : pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
7267 : {
7268 618 : struct cfs_rq *cfs_rq = &rq->cfs;
7269 : struct sched_entity *se;
7270 : struct task_struct *p;
7271 : int new_tasks;
7272 :
7273 : again:
7274 618 : if (!sched_fair_runnable(rq))
7275 : goto idle;
7276 :
7277 : #ifdef CONFIG_FAIR_GROUP_SCHED
7278 : if (!prev || prev->sched_class != &fair_sched_class)
7279 : goto simple;
7280 :
7281 : /*
7282 : * Because of the set_next_buddy() in dequeue_task_fair() it is rather
7283 : * likely that a next task is from the same cgroup as the current.
7284 : *
7285 : * Therefore attempt to avoid putting and setting the entire cgroup
7286 : * hierarchy, only change the part that actually changes.
7287 : */
7288 :
7289 : do {
7290 : struct sched_entity *curr = cfs_rq->curr;
7291 :
7292 : /*
7293 : * Since we got here without doing put_prev_entity() we also
7294 : * have to consider cfs_rq->curr. If it is still a runnable
7295 : * entity, update_curr() will update its vruntime, otherwise
7296 : * forget we've ever seen it.
7297 : */
7298 : if (curr) {
7299 : if (curr->on_rq)
7300 : update_curr(cfs_rq);
7301 : else
7302 : curr = NULL;
7303 :
7304 : /*
7305 : * This call to check_cfs_rq_runtime() will do the
7306 : * throttle and dequeue its entity in the parent(s).
7307 : * Therefore the nr_running test will indeed
7308 : * be correct.
7309 : */
7310 : if (unlikely(check_cfs_rq_runtime(cfs_rq))) {
7311 : cfs_rq = &rq->cfs;
7312 :
7313 : if (!cfs_rq->nr_running)
7314 : goto idle;
7315 :
7316 : goto simple;
7317 : }
7318 : }
7319 :
7320 : se = pick_next_entity(cfs_rq, curr);
7321 : cfs_rq = group_cfs_rq(se);
7322 : } while (cfs_rq);
7323 :
7324 : p = task_of(se);
7325 :
7326 : /*
7327 : * Since we haven't yet done put_prev_entity and if the selected task
7328 : * is a different task than we started out with, try and touch the
7329 : * least amount of cfs_rqs.
7330 : */
7331 : if (prev != p) {
7332 : struct sched_entity *pse = &prev->se;
7333 :
7334 : while (!(cfs_rq = is_same_group(se, pse))) {
7335 : int se_depth = se->depth;
7336 : int pse_depth = pse->depth;
7337 :
7338 : if (se_depth <= pse_depth) {
7339 : put_prev_entity(cfs_rq_of(pse), pse);
7340 : pse = parent_entity(pse);
7341 : }
7342 : if (se_depth >= pse_depth) {
7343 : set_next_entity(cfs_rq_of(se), se);
7344 : se = parent_entity(se);
7345 : }
7346 : }
7347 :
7348 : put_prev_entity(cfs_rq, pse);
7349 : set_next_entity(cfs_rq, se);
7350 : }
7351 :
7352 : goto done;
7353 : simple:
7354 : #endif
7355 617 : if (prev)
7356 617 : put_prev_task(rq, prev);
7357 :
7358 : do {
7359 617 : se = pick_next_entity(cfs_rq, NULL);
7360 617 : set_next_entity(cfs_rq, se);
7361 617 : cfs_rq = group_cfs_rq(se);
7362 : } while (cfs_rq);
7363 :
7364 617 : p = task_of(se);
7365 :
7366 : done: __maybe_unused;
7367 : #ifdef CONFIG_SMP
7368 : /*
7369 : * Move the next running task to the front of
7370 : * the list, so our cfs_tasks list becomes MRU
7371 : * one.
7372 : */
7373 : list_move(&p->se.group_node, &rq->cfs_tasks);
7374 : #endif
7375 :
7376 617 : if (hrtick_enabled_fair(rq))
7377 : hrtick_start_fair(rq, p);
7378 :
7379 617 : update_misfit_status(p, rq);
7380 :
7381 617 : return p;
7382 :
7383 : idle:
7384 : if (!rf)
7385 : return NULL;
7386 :
7387 : new_tasks = newidle_balance(rq, rf);
7388 :
7389 : /*
7390 : * Because newidle_balance() releases (and re-acquires) rq->lock, it is
7391 : * possible for any higher priority task to appear. In that case we
7392 : * must re-start the pick_next_entity() loop.
7393 : */
7394 : if (new_tasks < 0)
7395 : return RETRY_TASK;
7396 :
7397 : if (new_tasks > 0)
7398 : goto again;
7399 :
7400 : /*
7401 : * rq is about to be idle, check if we need to update the
7402 : * lost_idle_time of clock_pelt
7403 : */
7404 : update_idle_rq_clock_pelt(rq);
7405 :
7406 : return NULL;
7407 : }
7408 :
7409 0 : static struct task_struct *__pick_next_task_fair(struct rq *rq)
7410 : {
7411 0 : return pick_next_task_fair(rq, NULL, NULL);
7412 : }
7413 :
7414 : /*
7415 : * Account for a descheduled task:
7416 : */
7417 619 : static void put_prev_task_fair(struct rq *rq, struct task_struct *prev)
7418 : {
7419 619 : struct sched_entity *se = &prev->se;
7420 : struct cfs_rq *cfs_rq;
7421 :
7422 1238 : for_each_sched_entity(se) {
7423 1238 : cfs_rq = cfs_rq_of(se);
7424 619 : put_prev_entity(cfs_rq, se);
7425 : }
7426 619 : }
7427 :
7428 : /*
7429 : * sched_yield() is very simple
7430 : *
7431 : * The magic of dealing with the ->skip buddy is in pick_next_entity.
7432 : */
7433 0 : static void yield_task_fair(struct rq *rq)
7434 : {
7435 0 : struct task_struct *curr = rq->curr;
7436 0 : struct cfs_rq *cfs_rq = task_cfs_rq(curr);
7437 0 : struct sched_entity *se = &curr->se;
7438 :
7439 : /*
7440 : * Are we the only task in the tree?
7441 : */
7442 0 : if (unlikely(rq->nr_running == 1))
7443 : return;
7444 :
7445 0 : clear_buddies(cfs_rq, se);
7446 :
7447 0 : if (curr->policy != SCHED_BATCH) {
7448 0 : update_rq_clock(rq);
7449 : /*
7450 : * Update run-time statistics of the 'current'.
7451 : */
7452 0 : update_curr(cfs_rq);
7453 : /*
7454 : * Tell update_rq_clock() that we've just updated,
7455 : * so we don't do microscopic update in schedule()
7456 : * and double the fastpath cost.
7457 : */
7458 0 : rq_clock_skip_update(rq);
7459 : }
7460 :
7461 : set_skip_buddy(se);
7462 : }
7463 :
7464 0 : static bool yield_to_task_fair(struct rq *rq, struct task_struct *p)
7465 : {
7466 0 : struct sched_entity *se = &p->se;
7467 :
7468 : /* throttled hierarchies are not runnable */
7469 0 : if (!se->on_rq || throttled_hierarchy(cfs_rq_of(se)))
7470 : return false;
7471 :
7472 : /* Tell the scheduler that we'd really like pse to run next. */
7473 0 : set_next_buddy(se);
7474 :
7475 0 : yield_task_fair(rq);
7476 :
7477 0 : return true;
7478 : }
7479 :
7480 : #ifdef CONFIG_SMP
7481 : /**************************************************
7482 : * Fair scheduling class load-balancing methods.
7483 : *
7484 : * BASICS
7485 : *
7486 : * The purpose of load-balancing is to achieve the same basic fairness the
7487 : * per-CPU scheduler provides, namely provide a proportional amount of compute
7488 : * time to each task. This is expressed in the following equation:
7489 : *
7490 : * W_i,n/P_i == W_j,n/P_j for all i,j (1)
7491 : *
7492 : * Where W_i,n is the n-th weight average for CPU i. The instantaneous weight
7493 : * W_i,0 is defined as:
7494 : *
7495 : * W_i,0 = \Sum_j w_i,j (2)
7496 : *
7497 : * Where w_i,j is the weight of the j-th runnable task on CPU i. This weight
7498 : * is derived from the nice value as per sched_prio_to_weight[].
7499 : *
7500 : * The weight average is an exponential decay average of the instantaneous
7501 : * weight:
7502 : *
7503 : * W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0 (3)
7504 : *
7505 : * C_i is the compute capacity of CPU i, typically it is the
7506 : * fraction of 'recent' time available for SCHED_OTHER task execution. But it
7507 : * can also include other factors [XXX].
7508 : *
7509 : * To achieve this balance we define a measure of imbalance which follows
7510 : * directly from (1):
7511 : *
7512 : * imb_i,j = max{ avg(W/C), W_i/C_i } - min{ avg(W/C), W_j/C_j } (4)
7513 : *
7514 : * We them move tasks around to minimize the imbalance. In the continuous
7515 : * function space it is obvious this converges, in the discrete case we get
7516 : * a few fun cases generally called infeasible weight scenarios.
7517 : *
7518 : * [XXX expand on:
7519 : * - infeasible weights;
7520 : * - local vs global optima in the discrete case. ]
7521 : *
7522 : *
7523 : * SCHED DOMAINS
7524 : *
7525 : * In order to solve the imbalance equation (4), and avoid the obvious O(n^2)
7526 : * for all i,j solution, we create a tree of CPUs that follows the hardware
7527 : * topology where each level pairs two lower groups (or better). This results
7528 : * in O(log n) layers. Furthermore we reduce the number of CPUs going up the
7529 : * tree to only the first of the previous level and we decrease the frequency
7530 : * of load-balance at each level inv. proportional to the number of CPUs in
7531 : * the groups.
7532 : *
7533 : * This yields:
7534 : *
7535 : * log_2 n 1 n
7536 : * \Sum { --- * --- * 2^i } = O(n) (5)
7537 : * i = 0 2^i 2^i
7538 : * `- size of each group
7539 : * | | `- number of CPUs doing load-balance
7540 : * | `- freq
7541 : * `- sum over all levels
7542 : *
7543 : * Coupled with a limit on how many tasks we can migrate every balance pass,
7544 : * this makes (5) the runtime complexity of the balancer.
7545 : *
7546 : * An important property here is that each CPU is still (indirectly) connected
7547 : * to every other CPU in at most O(log n) steps:
7548 : *
7549 : * The adjacency matrix of the resulting graph is given by:
7550 : *
7551 : * log_2 n
7552 : * A_i,j = \Union (i % 2^k == 0) && i / 2^(k+1) == j / 2^(k+1) (6)
7553 : * k = 0
7554 : *
7555 : * And you'll find that:
7556 : *
7557 : * A^(log_2 n)_i,j != 0 for all i,j (7)
7558 : *
7559 : * Showing there's indeed a path between every CPU in at most O(log n) steps.
7560 : * The task movement gives a factor of O(m), giving a convergence complexity
7561 : * of:
7562 : *
7563 : * O(nm log n), n := nr_cpus, m := nr_tasks (8)
7564 : *
7565 : *
7566 : * WORK CONSERVING
7567 : *
7568 : * In order to avoid CPUs going idle while there's still work to do, new idle
7569 : * balancing is more aggressive and has the newly idle CPU iterate up the domain
7570 : * tree itself instead of relying on other CPUs to bring it work.
7571 : *
7572 : * This adds some complexity to both (5) and (8) but it reduces the total idle
7573 : * time.
7574 : *
7575 : * [XXX more?]
7576 : *
7577 : *
7578 : * CGROUPS
7579 : *
7580 : * Cgroups make a horror show out of (2), instead of a simple sum we get:
7581 : *
7582 : * s_k,i
7583 : * W_i,0 = \Sum_j \Prod_k w_k * ----- (9)
7584 : * S_k
7585 : *
7586 : * Where
7587 : *
7588 : * s_k,i = \Sum_j w_i,j,k and S_k = \Sum_i s_k,i (10)
7589 : *
7590 : * w_i,j,k is the weight of the j-th runnable task in the k-th cgroup on CPU i.
7591 : *
7592 : * The big problem is S_k, its a global sum needed to compute a local (W_i)
7593 : * property.
7594 : *
7595 : * [XXX write more on how we solve this.. _after_ merging pjt's patches that
7596 : * rewrite all of this once again.]
7597 : */
7598 :
7599 : static unsigned long __read_mostly max_load_balance_interval = HZ/10;
7600 :
7601 : enum fbq_type { regular, remote, all };
7602 :
7603 : /*
7604 : * 'group_type' describes the group of CPUs at the moment of load balancing.
7605 : *
7606 : * The enum is ordered by pulling priority, with the group with lowest priority
7607 : * first so the group_type can simply be compared when selecting the busiest
7608 : * group. See update_sd_pick_busiest().
7609 : */
7610 : enum group_type {
7611 : /* The group has spare capacity that can be used to run more tasks. */
7612 : group_has_spare = 0,
7613 : /*
7614 : * The group is fully used and the tasks don't compete for more CPU
7615 : * cycles. Nevertheless, some tasks might wait before running.
7616 : */
7617 : group_fully_busy,
7618 : /*
7619 : * SD_ASYM_CPUCAPACITY only: One task doesn't fit with CPU's capacity
7620 : * and must be migrated to a more powerful CPU.
7621 : */
7622 : group_misfit_task,
7623 : /*
7624 : * SD_ASYM_PACKING only: One local CPU with higher capacity is available,
7625 : * and the task should be migrated to it instead of running on the
7626 : * current CPU.
7627 : */
7628 : group_asym_packing,
7629 : /*
7630 : * The tasks' affinity constraints previously prevented the scheduler
7631 : * from balancing the load across the system.
7632 : */
7633 : group_imbalanced,
7634 : /*
7635 : * The CPU is overloaded and can't provide expected CPU cycles to all
7636 : * tasks.
7637 : */
7638 : group_overloaded
7639 : };
7640 :
7641 : enum migration_type {
7642 : migrate_load = 0,
7643 : migrate_util,
7644 : migrate_task,
7645 : migrate_misfit
7646 : };
7647 :
7648 : #define LBF_ALL_PINNED 0x01
7649 : #define LBF_NEED_BREAK 0x02
7650 : #define LBF_DST_PINNED 0x04
7651 : #define LBF_SOME_PINNED 0x08
7652 : #define LBF_ACTIVE_LB 0x10
7653 :
7654 : struct lb_env {
7655 : struct sched_domain *sd;
7656 :
7657 : struct rq *src_rq;
7658 : int src_cpu;
7659 :
7660 : int dst_cpu;
7661 : struct rq *dst_rq;
7662 :
7663 : struct cpumask *dst_grpmask;
7664 : int new_dst_cpu;
7665 : enum cpu_idle_type idle;
7666 : long imbalance;
7667 : /* The set of CPUs under consideration for load-balancing */
7668 : struct cpumask *cpus;
7669 :
7670 : unsigned int flags;
7671 :
7672 : unsigned int loop;
7673 : unsigned int loop_break;
7674 : unsigned int loop_max;
7675 :
7676 : enum fbq_type fbq_type;
7677 : enum migration_type migration_type;
7678 : struct list_head tasks;
7679 : };
7680 :
7681 : /*
7682 : * Is this task likely cache-hot:
7683 : */
7684 : static int task_hot(struct task_struct *p, struct lb_env *env)
7685 : {
7686 : s64 delta;
7687 :
7688 : lockdep_assert_rq_held(env->src_rq);
7689 :
7690 : if (p->sched_class != &fair_sched_class)
7691 : return 0;
7692 :
7693 : if (unlikely(task_has_idle_policy(p)))
7694 : return 0;
7695 :
7696 : /* SMT siblings share cache */
7697 : if (env->sd->flags & SD_SHARE_CPUCAPACITY)
7698 : return 0;
7699 :
7700 : /*
7701 : * Buddy candidates are cache hot:
7702 : */
7703 : if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running &&
7704 : (&p->se == cfs_rq_of(&p->se)->next ||
7705 : &p->se == cfs_rq_of(&p->se)->last))
7706 : return 1;
7707 :
7708 : if (sysctl_sched_migration_cost == -1)
7709 : return 1;
7710 :
7711 : /*
7712 : * Don't migrate task if the task's cookie does not match
7713 : * with the destination CPU's core cookie.
7714 : */
7715 : if (!sched_core_cookie_match(cpu_rq(env->dst_cpu), p))
7716 : return 1;
7717 :
7718 : if (sysctl_sched_migration_cost == 0)
7719 : return 0;
7720 :
7721 : delta = rq_clock_task(env->src_rq) - p->se.exec_start;
7722 :
7723 : return delta < (s64)sysctl_sched_migration_cost;
7724 : }
7725 :
7726 : #ifdef CONFIG_NUMA_BALANCING
7727 : /*
7728 : * Returns 1, if task migration degrades locality
7729 : * Returns 0, if task migration improves locality i.e migration preferred.
7730 : * Returns -1, if task migration is not affected by locality.
7731 : */
7732 : static int migrate_degrades_locality(struct task_struct *p, struct lb_env *env)
7733 : {
7734 : struct numa_group *numa_group = rcu_dereference(p->numa_group);
7735 : unsigned long src_weight, dst_weight;
7736 : int src_nid, dst_nid, dist;
7737 :
7738 : if (!static_branch_likely(&sched_numa_balancing))
7739 : return -1;
7740 :
7741 : if (!p->numa_faults || !(env->sd->flags & SD_NUMA))
7742 : return -1;
7743 :
7744 : src_nid = cpu_to_node(env->src_cpu);
7745 : dst_nid = cpu_to_node(env->dst_cpu);
7746 :
7747 : if (src_nid == dst_nid)
7748 : return -1;
7749 :
7750 : /* Migrating away from the preferred node is always bad. */
7751 : if (src_nid == p->numa_preferred_nid) {
7752 : if (env->src_rq->nr_running > env->src_rq->nr_preferred_running)
7753 : return 1;
7754 : else
7755 : return -1;
7756 : }
7757 :
7758 : /* Encourage migration to the preferred node. */
7759 : if (dst_nid == p->numa_preferred_nid)
7760 : return 0;
7761 :
7762 : /* Leaving a core idle is often worse than degrading locality. */
7763 : if (env->idle == CPU_IDLE)
7764 : return -1;
7765 :
7766 : dist = node_distance(src_nid, dst_nid);
7767 : if (numa_group) {
7768 : src_weight = group_weight(p, src_nid, dist);
7769 : dst_weight = group_weight(p, dst_nid, dist);
7770 : } else {
7771 : src_weight = task_weight(p, src_nid, dist);
7772 : dst_weight = task_weight(p, dst_nid, dist);
7773 : }
7774 :
7775 : return dst_weight < src_weight;
7776 : }
7777 :
7778 : #else
7779 : static inline int migrate_degrades_locality(struct task_struct *p,
7780 : struct lb_env *env)
7781 : {
7782 : return -1;
7783 : }
7784 : #endif
7785 :
7786 : /*
7787 : * can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
7788 : */
7789 : static
7790 : int can_migrate_task(struct task_struct *p, struct lb_env *env)
7791 : {
7792 : int tsk_cache_hot;
7793 :
7794 : lockdep_assert_rq_held(env->src_rq);
7795 :
7796 : /*
7797 : * We do not migrate tasks that are:
7798 : * 1) throttled_lb_pair, or
7799 : * 2) cannot be migrated to this CPU due to cpus_ptr, or
7800 : * 3) running (obviously), or
7801 : * 4) are cache-hot on their current CPU.
7802 : */
7803 : if (throttled_lb_pair(task_group(p), env->src_cpu, env->dst_cpu))
7804 : return 0;
7805 :
7806 : /* Disregard pcpu kthreads; they are where they need to be. */
7807 : if (kthread_is_per_cpu(p))
7808 : return 0;
7809 :
7810 : if (!cpumask_test_cpu(env->dst_cpu, p->cpus_ptr)) {
7811 : int cpu;
7812 :
7813 : schedstat_inc(p->stats.nr_failed_migrations_affine);
7814 :
7815 : env->flags |= LBF_SOME_PINNED;
7816 :
7817 : /*
7818 : * Remember if this task can be migrated to any other CPU in
7819 : * our sched_group. We may want to revisit it if we couldn't
7820 : * meet load balance goals by pulling other tasks on src_cpu.
7821 : *
7822 : * Avoid computing new_dst_cpu
7823 : * - for NEWLY_IDLE
7824 : * - if we have already computed one in current iteration
7825 : * - if it's an active balance
7826 : */
7827 : if (env->idle == CPU_NEWLY_IDLE ||
7828 : env->flags & (LBF_DST_PINNED | LBF_ACTIVE_LB))
7829 : return 0;
7830 :
7831 : /* Prevent to re-select dst_cpu via env's CPUs: */
7832 : for_each_cpu_and(cpu, env->dst_grpmask, env->cpus) {
7833 : if (cpumask_test_cpu(cpu, p->cpus_ptr)) {
7834 : env->flags |= LBF_DST_PINNED;
7835 : env->new_dst_cpu = cpu;
7836 : break;
7837 : }
7838 : }
7839 :
7840 : return 0;
7841 : }
7842 :
7843 : /* Record that we found at least one task that could run on dst_cpu */
7844 : env->flags &= ~LBF_ALL_PINNED;
7845 :
7846 : if (task_running(env->src_rq, p)) {
7847 : schedstat_inc(p->stats.nr_failed_migrations_running);
7848 : return 0;
7849 : }
7850 :
7851 : /*
7852 : * Aggressive migration if:
7853 : * 1) active balance
7854 : * 2) destination numa is preferred
7855 : * 3) task is cache cold, or
7856 : * 4) too many balance attempts have failed.
7857 : */
7858 : if (env->flags & LBF_ACTIVE_LB)
7859 : return 1;
7860 :
7861 : tsk_cache_hot = migrate_degrades_locality(p, env);
7862 : if (tsk_cache_hot == -1)
7863 : tsk_cache_hot = task_hot(p, env);
7864 :
7865 : if (tsk_cache_hot <= 0 ||
7866 : env->sd->nr_balance_failed > env->sd->cache_nice_tries) {
7867 : if (tsk_cache_hot == 1) {
7868 : schedstat_inc(env->sd->lb_hot_gained[env->idle]);
7869 : schedstat_inc(p->stats.nr_forced_migrations);
7870 : }
7871 : return 1;
7872 : }
7873 :
7874 : schedstat_inc(p->stats.nr_failed_migrations_hot);
7875 : return 0;
7876 : }
7877 :
7878 : /*
7879 : * detach_task() -- detach the task for the migration specified in env
7880 : */
7881 : static void detach_task(struct task_struct *p, struct lb_env *env)
7882 : {
7883 : lockdep_assert_rq_held(env->src_rq);
7884 :
7885 : deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK);
7886 : set_task_cpu(p, env->dst_cpu);
7887 : }
7888 :
7889 : /*
7890 : * detach_one_task() -- tries to dequeue exactly one task from env->src_rq, as
7891 : * part of active balancing operations within "domain".
7892 : *
7893 : * Returns a task if successful and NULL otherwise.
7894 : */
7895 : static struct task_struct *detach_one_task(struct lb_env *env)
7896 : {
7897 : struct task_struct *p;
7898 :
7899 : lockdep_assert_rq_held(env->src_rq);
7900 :
7901 : list_for_each_entry_reverse(p,
7902 : &env->src_rq->cfs_tasks, se.group_node) {
7903 : if (!can_migrate_task(p, env))
7904 : continue;
7905 :
7906 : detach_task(p, env);
7907 :
7908 : /*
7909 : * Right now, this is only the second place where
7910 : * lb_gained[env->idle] is updated (other is detach_tasks)
7911 : * so we can safely collect stats here rather than
7912 : * inside detach_tasks().
7913 : */
7914 : schedstat_inc(env->sd->lb_gained[env->idle]);
7915 : return p;
7916 : }
7917 : return NULL;
7918 : }
7919 :
7920 : static const unsigned int sched_nr_migrate_break = 32;
7921 :
7922 : /*
7923 : * detach_tasks() -- tries to detach up to imbalance load/util/tasks from
7924 : * busiest_rq, as part of a balancing operation within domain "sd".
7925 : *
7926 : * Returns number of detached tasks if successful and 0 otherwise.
7927 : */
7928 : static int detach_tasks(struct lb_env *env)
7929 : {
7930 : struct list_head *tasks = &env->src_rq->cfs_tasks;
7931 : unsigned long util, load;
7932 : struct task_struct *p;
7933 : int detached = 0;
7934 :
7935 : lockdep_assert_rq_held(env->src_rq);
7936 :
7937 : /*
7938 : * Source run queue has been emptied by another CPU, clear
7939 : * LBF_ALL_PINNED flag as we will not test any task.
7940 : */
7941 : if (env->src_rq->nr_running <= 1) {
7942 : env->flags &= ~LBF_ALL_PINNED;
7943 : return 0;
7944 : }
7945 :
7946 : if (env->imbalance <= 0)
7947 : return 0;
7948 :
7949 : while (!list_empty(tasks)) {
7950 : /*
7951 : * We don't want to steal all, otherwise we may be treated likewise,
7952 : * which could at worst lead to a livelock crash.
7953 : */
7954 : if (env->idle != CPU_NOT_IDLE && env->src_rq->nr_running <= 1)
7955 : break;
7956 :
7957 : p = list_last_entry(tasks, struct task_struct, se.group_node);
7958 :
7959 : env->loop++;
7960 : /* We've more or less seen every task there is, call it quits */
7961 : if (env->loop > env->loop_max)
7962 : break;
7963 :
7964 : /* take a breather every nr_migrate tasks */
7965 : if (env->loop > env->loop_break) {
7966 : env->loop_break += sched_nr_migrate_break;
7967 : env->flags |= LBF_NEED_BREAK;
7968 : break;
7969 : }
7970 :
7971 : if (!can_migrate_task(p, env))
7972 : goto next;
7973 :
7974 : switch (env->migration_type) {
7975 : case migrate_load:
7976 : /*
7977 : * Depending of the number of CPUs and tasks and the
7978 : * cgroup hierarchy, task_h_load() can return a null
7979 : * value. Make sure that env->imbalance decreases
7980 : * otherwise detach_tasks() will stop only after
7981 : * detaching up to loop_max tasks.
7982 : */
7983 : load = max_t(unsigned long, task_h_load(p), 1);
7984 :
7985 : if (sched_feat(LB_MIN) &&
7986 : load < 16 && !env->sd->nr_balance_failed)
7987 : goto next;
7988 :
7989 : /*
7990 : * Make sure that we don't migrate too much load.
7991 : * Nevertheless, let relax the constraint if
7992 : * scheduler fails to find a good waiting task to
7993 : * migrate.
7994 : */
7995 : if (shr_bound(load, env->sd->nr_balance_failed) > env->imbalance)
7996 : goto next;
7997 :
7998 : env->imbalance -= load;
7999 : break;
8000 :
8001 : case migrate_util:
8002 : util = task_util_est(p);
8003 :
8004 : if (util > env->imbalance)
8005 : goto next;
8006 :
8007 : env->imbalance -= util;
8008 : break;
8009 :
8010 : case migrate_task:
8011 : env->imbalance--;
8012 : break;
8013 :
8014 : case migrate_misfit:
8015 : /* This is not a misfit task */
8016 : if (task_fits_capacity(p, capacity_of(env->src_cpu)))
8017 : goto next;
8018 :
8019 : env->imbalance = 0;
8020 : break;
8021 : }
8022 :
8023 : detach_task(p, env);
8024 : list_add(&p->se.group_node, &env->tasks);
8025 :
8026 : detached++;
8027 :
8028 : #ifdef CONFIG_PREEMPTION
8029 : /*
8030 : * NEWIDLE balancing is a source of latency, so preemptible
8031 : * kernels will stop after the first task is detached to minimize
8032 : * the critical section.
8033 : */
8034 : if (env->idle == CPU_NEWLY_IDLE)
8035 : break;
8036 : #endif
8037 :
8038 : /*
8039 : * We only want to steal up to the prescribed amount of
8040 : * load/util/tasks.
8041 : */
8042 : if (env->imbalance <= 0)
8043 : break;
8044 :
8045 : continue;
8046 : next:
8047 : list_move(&p->se.group_node, tasks);
8048 : }
8049 :
8050 : /*
8051 : * Right now, this is one of only two places we collect this stat
8052 : * so we can safely collect detach_one_task() stats here rather
8053 : * than inside detach_one_task().
8054 : */
8055 : schedstat_add(env->sd->lb_gained[env->idle], detached);
8056 :
8057 : return detached;
8058 : }
8059 :
8060 : /*
8061 : * attach_task() -- attach the task detached by detach_task() to its new rq.
8062 : */
8063 : static void attach_task(struct rq *rq, struct task_struct *p)
8064 : {
8065 : lockdep_assert_rq_held(rq);
8066 :
8067 : BUG_ON(task_rq(p) != rq);
8068 : activate_task(rq, p, ENQUEUE_NOCLOCK);
8069 : check_preempt_curr(rq, p, 0);
8070 : }
8071 :
8072 : /*
8073 : * attach_one_task() -- attaches the task returned from detach_one_task() to
8074 : * its new rq.
8075 : */
8076 : static void attach_one_task(struct rq *rq, struct task_struct *p)
8077 : {
8078 : struct rq_flags rf;
8079 :
8080 : rq_lock(rq, &rf);
8081 : update_rq_clock(rq);
8082 : attach_task(rq, p);
8083 : rq_unlock(rq, &rf);
8084 : }
8085 :
8086 : /*
8087 : * attach_tasks() -- attaches all tasks detached by detach_tasks() to their
8088 : * new rq.
8089 : */
8090 : static void attach_tasks(struct lb_env *env)
8091 : {
8092 : struct list_head *tasks = &env->tasks;
8093 : struct task_struct *p;
8094 : struct rq_flags rf;
8095 :
8096 : rq_lock(env->dst_rq, &rf);
8097 : update_rq_clock(env->dst_rq);
8098 :
8099 : while (!list_empty(tasks)) {
8100 : p = list_first_entry(tasks, struct task_struct, se.group_node);
8101 : list_del_init(&p->se.group_node);
8102 :
8103 : attach_task(env->dst_rq, p);
8104 : }
8105 :
8106 : rq_unlock(env->dst_rq, &rf);
8107 : }
8108 :
8109 : #ifdef CONFIG_NO_HZ_COMMON
8110 : static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq)
8111 : {
8112 : if (cfs_rq->avg.load_avg)
8113 : return true;
8114 :
8115 : if (cfs_rq->avg.util_avg)
8116 : return true;
8117 :
8118 : return false;
8119 : }
8120 :
8121 : static inline bool others_have_blocked(struct rq *rq)
8122 : {
8123 : if (READ_ONCE(rq->avg_rt.util_avg))
8124 : return true;
8125 :
8126 : if (READ_ONCE(rq->avg_dl.util_avg))
8127 : return true;
8128 :
8129 : if (thermal_load_avg(rq))
8130 : return true;
8131 :
8132 : #ifdef CONFIG_HAVE_SCHED_AVG_IRQ
8133 : if (READ_ONCE(rq->avg_irq.util_avg))
8134 : return true;
8135 : #endif
8136 :
8137 : return false;
8138 : }
8139 :
8140 : static inline void update_blocked_load_tick(struct rq *rq)
8141 : {
8142 : WRITE_ONCE(rq->last_blocked_load_update_tick, jiffies);
8143 : }
8144 :
8145 : static inline void update_blocked_load_status(struct rq *rq, bool has_blocked)
8146 : {
8147 : if (!has_blocked)
8148 : rq->has_blocked_load = 0;
8149 : }
8150 : #else
8151 : static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq) { return false; }
8152 : static inline bool others_have_blocked(struct rq *rq) { return false; }
8153 : static inline void update_blocked_load_tick(struct rq *rq) {}
8154 : static inline void update_blocked_load_status(struct rq *rq, bool has_blocked) {}
8155 : #endif
8156 :
8157 : static bool __update_blocked_others(struct rq *rq, bool *done)
8158 : {
8159 : const struct sched_class *curr_class;
8160 : u64 now = rq_clock_pelt(rq);
8161 : unsigned long thermal_pressure;
8162 : bool decayed;
8163 :
8164 : /*
8165 : * update_load_avg() can call cpufreq_update_util(). Make sure that RT,
8166 : * DL and IRQ signals have been updated before updating CFS.
8167 : */
8168 : curr_class = rq->curr->sched_class;
8169 :
8170 : thermal_pressure = arch_scale_thermal_pressure(cpu_of(rq));
8171 :
8172 : decayed = update_rt_rq_load_avg(now, rq, curr_class == &rt_sched_class) |
8173 : update_dl_rq_load_avg(now, rq, curr_class == &dl_sched_class) |
8174 : update_thermal_load_avg(rq_clock_thermal(rq), rq, thermal_pressure) |
8175 : update_irq_load_avg(rq, 0);
8176 :
8177 : if (others_have_blocked(rq))
8178 : *done = false;
8179 :
8180 : return decayed;
8181 : }
8182 :
8183 : #ifdef CONFIG_FAIR_GROUP_SCHED
8184 :
8185 : static bool __update_blocked_fair(struct rq *rq, bool *done)
8186 : {
8187 : struct cfs_rq *cfs_rq, *pos;
8188 : bool decayed = false;
8189 : int cpu = cpu_of(rq);
8190 :
8191 : /*
8192 : * Iterates the task_group tree in a bottom up fashion, see
8193 : * list_add_leaf_cfs_rq() for details.
8194 : */
8195 : for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) {
8196 : struct sched_entity *se;
8197 :
8198 : if (update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq)) {
8199 : update_tg_load_avg(cfs_rq);
8200 :
8201 : if (cfs_rq == &rq->cfs)
8202 : decayed = true;
8203 : }
8204 :
8205 : /* Propagate pending load changes to the parent, if any: */
8206 : se = cfs_rq->tg->se[cpu];
8207 : if (se && !skip_blocked_update(se))
8208 : update_load_avg(cfs_rq_of(se), se, UPDATE_TG);
8209 :
8210 : /*
8211 : * There can be a lot of idle CPU cgroups. Don't let fully
8212 : * decayed cfs_rqs linger on the list.
8213 : */
8214 : if (cfs_rq_is_decayed(cfs_rq))
8215 : list_del_leaf_cfs_rq(cfs_rq);
8216 :
8217 : /* Don't need periodic decay once load/util_avg are null */
8218 : if (cfs_rq_has_blocked(cfs_rq))
8219 : *done = false;
8220 : }
8221 :
8222 : return decayed;
8223 : }
8224 :
8225 : /*
8226 : * Compute the hierarchical load factor for cfs_rq and all its ascendants.
8227 : * This needs to be done in a top-down fashion because the load of a child
8228 : * group is a fraction of its parents load.
8229 : */
8230 : static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq)
8231 : {
8232 : struct rq *rq = rq_of(cfs_rq);
8233 : struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)];
8234 : unsigned long now = jiffies;
8235 : unsigned long load;
8236 :
8237 : if (cfs_rq->last_h_load_update == now)
8238 : return;
8239 :
8240 : WRITE_ONCE(cfs_rq->h_load_next, NULL);
8241 : for_each_sched_entity(se) {
8242 : cfs_rq = cfs_rq_of(se);
8243 : WRITE_ONCE(cfs_rq->h_load_next, se);
8244 : if (cfs_rq->last_h_load_update == now)
8245 : break;
8246 : }
8247 :
8248 : if (!se) {
8249 : cfs_rq->h_load = cfs_rq_load_avg(cfs_rq);
8250 : cfs_rq->last_h_load_update = now;
8251 : }
8252 :
8253 : while ((se = READ_ONCE(cfs_rq->h_load_next)) != NULL) {
8254 : load = cfs_rq->h_load;
8255 : load = div64_ul(load * se->avg.load_avg,
8256 : cfs_rq_load_avg(cfs_rq) + 1);
8257 : cfs_rq = group_cfs_rq(se);
8258 : cfs_rq->h_load = load;
8259 : cfs_rq->last_h_load_update = now;
8260 : }
8261 : }
8262 :
8263 : static unsigned long task_h_load(struct task_struct *p)
8264 : {
8265 : struct cfs_rq *cfs_rq = task_cfs_rq(p);
8266 :
8267 : update_cfs_rq_h_load(cfs_rq);
8268 : return div64_ul(p->se.avg.load_avg * cfs_rq->h_load,
8269 : cfs_rq_load_avg(cfs_rq) + 1);
8270 : }
8271 : #else
8272 : static bool __update_blocked_fair(struct rq *rq, bool *done)
8273 : {
8274 : struct cfs_rq *cfs_rq = &rq->cfs;
8275 : bool decayed;
8276 :
8277 : decayed = update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq);
8278 : if (cfs_rq_has_blocked(cfs_rq))
8279 : *done = false;
8280 :
8281 : return decayed;
8282 : }
8283 :
8284 : static unsigned long task_h_load(struct task_struct *p)
8285 : {
8286 : return p->se.avg.load_avg;
8287 : }
8288 : #endif
8289 :
8290 : static void update_blocked_averages(int cpu)
8291 : {
8292 : bool decayed = false, done = true;
8293 : struct rq *rq = cpu_rq(cpu);
8294 : struct rq_flags rf;
8295 :
8296 : rq_lock_irqsave(rq, &rf);
8297 : update_blocked_load_tick(rq);
8298 : update_rq_clock(rq);
8299 :
8300 : decayed |= __update_blocked_others(rq, &done);
8301 : decayed |= __update_blocked_fair(rq, &done);
8302 :
8303 : update_blocked_load_status(rq, !done);
8304 : if (decayed)
8305 : cpufreq_update_util(rq, 0);
8306 : rq_unlock_irqrestore(rq, &rf);
8307 : }
8308 :
8309 : /********** Helpers for find_busiest_group ************************/
8310 :
8311 : /*
8312 : * sg_lb_stats - stats of a sched_group required for load_balancing
8313 : */
8314 : struct sg_lb_stats {
8315 : unsigned long avg_load; /*Avg load across the CPUs of the group */
8316 : unsigned long group_load; /* Total load over the CPUs of the group */
8317 : unsigned long group_capacity;
8318 : unsigned long group_util; /* Total utilization over the CPUs of the group */
8319 : unsigned long group_runnable; /* Total runnable time over the CPUs of the group */
8320 : unsigned int sum_nr_running; /* Nr of tasks running in the group */
8321 : unsigned int sum_h_nr_running; /* Nr of CFS tasks running in the group */
8322 : unsigned int idle_cpus;
8323 : unsigned int group_weight;
8324 : enum group_type group_type;
8325 : unsigned int group_asym_packing; /* Tasks should be moved to preferred CPU */
8326 : unsigned long group_misfit_task_load; /* A CPU has a task too big for its capacity */
8327 : #ifdef CONFIG_NUMA_BALANCING
8328 : unsigned int nr_numa_running;
8329 : unsigned int nr_preferred_running;
8330 : #endif
8331 : };
8332 :
8333 : /*
8334 : * sd_lb_stats - Structure to store the statistics of a sched_domain
8335 : * during load balancing.
8336 : */
8337 : struct sd_lb_stats {
8338 : struct sched_group *busiest; /* Busiest group in this sd */
8339 : struct sched_group *local; /* Local group in this sd */
8340 : unsigned long total_load; /* Total load of all groups in sd */
8341 : unsigned long total_capacity; /* Total capacity of all groups in sd */
8342 : unsigned long avg_load; /* Average load across all groups in sd */
8343 : unsigned int prefer_sibling; /* tasks should go to sibling first */
8344 :
8345 : struct sg_lb_stats busiest_stat;/* Statistics of the busiest group */
8346 : struct sg_lb_stats local_stat; /* Statistics of the local group */
8347 : };
8348 :
8349 : static inline void init_sd_lb_stats(struct sd_lb_stats *sds)
8350 : {
8351 : /*
8352 : * Skimp on the clearing to avoid duplicate work. We can avoid clearing
8353 : * local_stat because update_sg_lb_stats() does a full clear/assignment.
8354 : * We must however set busiest_stat::group_type and
8355 : * busiest_stat::idle_cpus to the worst busiest group because
8356 : * update_sd_pick_busiest() reads these before assignment.
8357 : */
8358 : *sds = (struct sd_lb_stats){
8359 : .busiest = NULL,
8360 : .local = NULL,
8361 : .total_load = 0UL,
8362 : .total_capacity = 0UL,
8363 : .busiest_stat = {
8364 : .idle_cpus = UINT_MAX,
8365 : .group_type = group_has_spare,
8366 : },
8367 : };
8368 : }
8369 :
8370 : static unsigned long scale_rt_capacity(int cpu)
8371 : {
8372 : struct rq *rq = cpu_rq(cpu);
8373 : unsigned long max = arch_scale_cpu_capacity(cpu);
8374 : unsigned long used, free;
8375 : unsigned long irq;
8376 :
8377 : irq = cpu_util_irq(rq);
8378 :
8379 : if (unlikely(irq >= max))
8380 : return 1;
8381 :
8382 : /*
8383 : * avg_rt.util_avg and avg_dl.util_avg track binary signals
8384 : * (running and not running) with weights 0 and 1024 respectively.
8385 : * avg_thermal.load_avg tracks thermal pressure and the weighted
8386 : * average uses the actual delta max capacity(load).
8387 : */
8388 : used = READ_ONCE(rq->avg_rt.util_avg);
8389 : used += READ_ONCE(rq->avg_dl.util_avg);
8390 : used += thermal_load_avg(rq);
8391 :
8392 : if (unlikely(used >= max))
8393 : return 1;
8394 :
8395 : free = max - used;
8396 :
8397 : return scale_irq_capacity(free, irq, max);
8398 : }
8399 :
8400 : static void update_cpu_capacity(struct sched_domain *sd, int cpu)
8401 : {
8402 : unsigned long capacity = scale_rt_capacity(cpu);
8403 : struct sched_group *sdg = sd->groups;
8404 :
8405 : cpu_rq(cpu)->cpu_capacity_orig = arch_scale_cpu_capacity(cpu);
8406 :
8407 : if (!capacity)
8408 : capacity = 1;
8409 :
8410 : cpu_rq(cpu)->cpu_capacity = capacity;
8411 : trace_sched_cpu_capacity_tp(cpu_rq(cpu));
8412 :
8413 : sdg->sgc->capacity = capacity;
8414 : sdg->sgc->min_capacity = capacity;
8415 : sdg->sgc->max_capacity = capacity;
8416 : }
8417 :
8418 : void update_group_capacity(struct sched_domain *sd, int cpu)
8419 : {
8420 : struct sched_domain *child = sd->child;
8421 : struct sched_group *group, *sdg = sd->groups;
8422 : unsigned long capacity, min_capacity, max_capacity;
8423 : unsigned long interval;
8424 :
8425 : interval = msecs_to_jiffies(sd->balance_interval);
8426 : interval = clamp(interval, 1UL, max_load_balance_interval);
8427 : sdg->sgc->next_update = jiffies + interval;
8428 :
8429 : if (!child) {
8430 : update_cpu_capacity(sd, cpu);
8431 : return;
8432 : }
8433 :
8434 : capacity = 0;
8435 : min_capacity = ULONG_MAX;
8436 : max_capacity = 0;
8437 :
8438 : if (child->flags & SD_OVERLAP) {
8439 : /*
8440 : * SD_OVERLAP domains cannot assume that child groups
8441 : * span the current group.
8442 : */
8443 :
8444 : for_each_cpu(cpu, sched_group_span(sdg)) {
8445 : unsigned long cpu_cap = capacity_of(cpu);
8446 :
8447 : capacity += cpu_cap;
8448 : min_capacity = min(cpu_cap, min_capacity);
8449 : max_capacity = max(cpu_cap, max_capacity);
8450 : }
8451 : } else {
8452 : /*
8453 : * !SD_OVERLAP domains can assume that child groups
8454 : * span the current group.
8455 : */
8456 :
8457 : group = child->groups;
8458 : do {
8459 : struct sched_group_capacity *sgc = group->sgc;
8460 :
8461 : capacity += sgc->capacity;
8462 : min_capacity = min(sgc->min_capacity, min_capacity);
8463 : max_capacity = max(sgc->max_capacity, max_capacity);
8464 : group = group->next;
8465 : } while (group != child->groups);
8466 : }
8467 :
8468 : sdg->sgc->capacity = capacity;
8469 : sdg->sgc->min_capacity = min_capacity;
8470 : sdg->sgc->max_capacity = max_capacity;
8471 : }
8472 :
8473 : /*
8474 : * Check whether the capacity of the rq has been noticeably reduced by side
8475 : * activity. The imbalance_pct is used for the threshold.
8476 : * Return true is the capacity is reduced
8477 : */
8478 : static inline int
8479 : check_cpu_capacity(struct rq *rq, struct sched_domain *sd)
8480 : {
8481 : return ((rq->cpu_capacity * sd->imbalance_pct) <
8482 : (rq->cpu_capacity_orig * 100));
8483 : }
8484 :
8485 : /*
8486 : * Check whether a rq has a misfit task and if it looks like we can actually
8487 : * help that task: we can migrate the task to a CPU of higher capacity, or
8488 : * the task's current CPU is heavily pressured.
8489 : */
8490 : static inline int check_misfit_status(struct rq *rq, struct sched_domain *sd)
8491 : {
8492 : return rq->misfit_task_load &&
8493 : (rq->cpu_capacity_orig < rq->rd->max_cpu_capacity ||
8494 : check_cpu_capacity(rq, sd));
8495 : }
8496 :
8497 : /*
8498 : * Group imbalance indicates (and tries to solve) the problem where balancing
8499 : * groups is inadequate due to ->cpus_ptr constraints.
8500 : *
8501 : * Imagine a situation of two groups of 4 CPUs each and 4 tasks each with a
8502 : * cpumask covering 1 CPU of the first group and 3 CPUs of the second group.
8503 : * Something like:
8504 : *
8505 : * { 0 1 2 3 } { 4 5 6 7 }
8506 : * * * * *
8507 : *
8508 : * If we were to balance group-wise we'd place two tasks in the first group and
8509 : * two tasks in the second group. Clearly this is undesired as it will overload
8510 : * cpu 3 and leave one of the CPUs in the second group unused.
8511 : *
8512 : * The current solution to this issue is detecting the skew in the first group
8513 : * by noticing the lower domain failed to reach balance and had difficulty
8514 : * moving tasks due to affinity constraints.
8515 : *
8516 : * When this is so detected; this group becomes a candidate for busiest; see
8517 : * update_sd_pick_busiest(). And calculate_imbalance() and
8518 : * find_busiest_group() avoid some of the usual balance conditions to allow it
8519 : * to create an effective group imbalance.
8520 : *
8521 : * This is a somewhat tricky proposition since the next run might not find the
8522 : * group imbalance and decide the groups need to be balanced again. A most
8523 : * subtle and fragile situation.
8524 : */
8525 :
8526 : static inline int sg_imbalanced(struct sched_group *group)
8527 : {
8528 : return group->sgc->imbalance;
8529 : }
8530 :
8531 : /*
8532 : * group_has_capacity returns true if the group has spare capacity that could
8533 : * be used by some tasks.
8534 : * We consider that a group has spare capacity if the * number of task is
8535 : * smaller than the number of CPUs or if the utilization is lower than the
8536 : * available capacity for CFS tasks.
8537 : * For the latter, we use a threshold to stabilize the state, to take into
8538 : * account the variance of the tasks' load and to return true if the available
8539 : * capacity in meaningful for the load balancer.
8540 : * As an example, an available capacity of 1% can appear but it doesn't make
8541 : * any benefit for the load balance.
8542 : */
8543 : static inline bool
8544 : group_has_capacity(unsigned int imbalance_pct, struct sg_lb_stats *sgs)
8545 : {
8546 : if (sgs->sum_nr_running < sgs->group_weight)
8547 : return true;
8548 :
8549 : if ((sgs->group_capacity * imbalance_pct) <
8550 : (sgs->group_runnable * 100))
8551 : return false;
8552 :
8553 : if ((sgs->group_capacity * 100) >
8554 : (sgs->group_util * imbalance_pct))
8555 : return true;
8556 :
8557 : return false;
8558 : }
8559 :
8560 : /*
8561 : * group_is_overloaded returns true if the group has more tasks than it can
8562 : * handle.
8563 : * group_is_overloaded is not equals to !group_has_capacity because a group
8564 : * with the exact right number of tasks, has no more spare capacity but is not
8565 : * overloaded so both group_has_capacity and group_is_overloaded return
8566 : * false.
8567 : */
8568 : static inline bool
8569 : group_is_overloaded(unsigned int imbalance_pct, struct sg_lb_stats *sgs)
8570 : {
8571 : if (sgs->sum_nr_running <= sgs->group_weight)
8572 : return false;
8573 :
8574 : if ((sgs->group_capacity * 100) <
8575 : (sgs->group_util * imbalance_pct))
8576 : return true;
8577 :
8578 : if ((sgs->group_capacity * imbalance_pct) <
8579 : (sgs->group_runnable * 100))
8580 : return true;
8581 :
8582 : return false;
8583 : }
8584 :
8585 : static inline enum
8586 : group_type group_classify(unsigned int imbalance_pct,
8587 : struct sched_group *group,
8588 : struct sg_lb_stats *sgs)
8589 : {
8590 : if (group_is_overloaded(imbalance_pct, sgs))
8591 : return group_overloaded;
8592 :
8593 : if (sg_imbalanced(group))
8594 : return group_imbalanced;
8595 :
8596 : if (sgs->group_asym_packing)
8597 : return group_asym_packing;
8598 :
8599 : if (sgs->group_misfit_task_load)
8600 : return group_misfit_task;
8601 :
8602 : if (!group_has_capacity(imbalance_pct, sgs))
8603 : return group_fully_busy;
8604 :
8605 : return group_has_spare;
8606 : }
8607 :
8608 : /**
8609 : * asym_smt_can_pull_tasks - Check whether the load balancing CPU can pull tasks
8610 : * @dst_cpu: Destination CPU of the load balancing
8611 : * @sds: Load-balancing data with statistics of the local group
8612 : * @sgs: Load-balancing statistics of the candidate busiest group
8613 : * @sg: The candidate busiest group
8614 : *
8615 : * Check the state of the SMT siblings of both @sds::local and @sg and decide
8616 : * if @dst_cpu can pull tasks.
8617 : *
8618 : * If @dst_cpu does not have SMT siblings, it can pull tasks if two or more of
8619 : * the SMT siblings of @sg are busy. If only one CPU in @sg is busy, pull tasks
8620 : * only if @dst_cpu has higher priority.
8621 : *
8622 : * If both @dst_cpu and @sg have SMT siblings, and @sg has exactly one more
8623 : * busy CPU than @sds::local, let @dst_cpu pull tasks if it has higher priority.
8624 : * Bigger imbalances in the number of busy CPUs will be dealt with in
8625 : * update_sd_pick_busiest().
8626 : *
8627 : * If @sg does not have SMT siblings, only pull tasks if all of the SMT siblings
8628 : * of @dst_cpu are idle and @sg has lower priority.
8629 : *
8630 : * Return: true if @dst_cpu can pull tasks, false otherwise.
8631 : */
8632 : static bool asym_smt_can_pull_tasks(int dst_cpu, struct sd_lb_stats *sds,
8633 : struct sg_lb_stats *sgs,
8634 : struct sched_group *sg)
8635 : {
8636 : #ifdef CONFIG_SCHED_SMT
8637 : bool local_is_smt, sg_is_smt;
8638 : int sg_busy_cpus;
8639 :
8640 : local_is_smt = sds->local->flags & SD_SHARE_CPUCAPACITY;
8641 : sg_is_smt = sg->flags & SD_SHARE_CPUCAPACITY;
8642 :
8643 : sg_busy_cpus = sgs->group_weight - sgs->idle_cpus;
8644 :
8645 : if (!local_is_smt) {
8646 : /*
8647 : * If we are here, @dst_cpu is idle and does not have SMT
8648 : * siblings. Pull tasks if candidate group has two or more
8649 : * busy CPUs.
8650 : */
8651 : if (sg_busy_cpus >= 2) /* implies sg_is_smt */
8652 : return true;
8653 :
8654 : /*
8655 : * @dst_cpu does not have SMT siblings. @sg may have SMT
8656 : * siblings and only one is busy. In such case, @dst_cpu
8657 : * can help if it has higher priority and is idle (i.e.,
8658 : * it has no running tasks).
8659 : */
8660 : return sched_asym_prefer(dst_cpu, sg->asym_prefer_cpu);
8661 : }
8662 :
8663 : /* @dst_cpu has SMT siblings. */
8664 :
8665 : if (sg_is_smt) {
8666 : int local_busy_cpus = sds->local->group_weight -
8667 : sds->local_stat.idle_cpus;
8668 : int busy_cpus_delta = sg_busy_cpus - local_busy_cpus;
8669 :
8670 : if (busy_cpus_delta == 1)
8671 : return sched_asym_prefer(dst_cpu, sg->asym_prefer_cpu);
8672 :
8673 : return false;
8674 : }
8675 :
8676 : /*
8677 : * @sg does not have SMT siblings. Ensure that @sds::local does not end
8678 : * up with more than one busy SMT sibling and only pull tasks if there
8679 : * are not busy CPUs (i.e., no CPU has running tasks).
8680 : */
8681 : if (!sds->local_stat.sum_nr_running)
8682 : return sched_asym_prefer(dst_cpu, sg->asym_prefer_cpu);
8683 :
8684 : return false;
8685 : #else
8686 : /* Always return false so that callers deal with non-SMT cases. */
8687 : return false;
8688 : #endif
8689 : }
8690 :
8691 : static inline bool
8692 : sched_asym(struct lb_env *env, struct sd_lb_stats *sds, struct sg_lb_stats *sgs,
8693 : struct sched_group *group)
8694 : {
8695 : /* Only do SMT checks if either local or candidate have SMT siblings */
8696 : if ((sds->local->flags & SD_SHARE_CPUCAPACITY) ||
8697 : (group->flags & SD_SHARE_CPUCAPACITY))
8698 : return asym_smt_can_pull_tasks(env->dst_cpu, sds, sgs, group);
8699 :
8700 : return sched_asym_prefer(env->dst_cpu, group->asym_prefer_cpu);
8701 : }
8702 :
8703 : /**
8704 : * update_sg_lb_stats - Update sched_group's statistics for load balancing.
8705 : * @env: The load balancing environment.
8706 : * @sds: Load-balancing data with statistics of the local group.
8707 : * @group: sched_group whose statistics are to be updated.
8708 : * @sgs: variable to hold the statistics for this group.
8709 : * @sg_status: Holds flag indicating the status of the sched_group
8710 : */
8711 : static inline void update_sg_lb_stats(struct lb_env *env,
8712 : struct sd_lb_stats *sds,
8713 : struct sched_group *group,
8714 : struct sg_lb_stats *sgs,
8715 : int *sg_status)
8716 : {
8717 : int i, nr_running, local_group;
8718 :
8719 : memset(sgs, 0, sizeof(*sgs));
8720 :
8721 : local_group = group == sds->local;
8722 :
8723 : for_each_cpu_and(i, sched_group_span(group), env->cpus) {
8724 : struct rq *rq = cpu_rq(i);
8725 :
8726 : sgs->group_load += cpu_load(rq);
8727 : sgs->group_util += cpu_util_cfs(i);
8728 : sgs->group_runnable += cpu_runnable(rq);
8729 : sgs->sum_h_nr_running += rq->cfs.h_nr_running;
8730 :
8731 : nr_running = rq->nr_running;
8732 : sgs->sum_nr_running += nr_running;
8733 :
8734 : if (nr_running > 1)
8735 : *sg_status |= SG_OVERLOAD;
8736 :
8737 : if (cpu_overutilized(i))
8738 : *sg_status |= SG_OVERUTILIZED;
8739 :
8740 : #ifdef CONFIG_NUMA_BALANCING
8741 : sgs->nr_numa_running += rq->nr_numa_running;
8742 : sgs->nr_preferred_running += rq->nr_preferred_running;
8743 : #endif
8744 : /*
8745 : * No need to call idle_cpu() if nr_running is not 0
8746 : */
8747 : if (!nr_running && idle_cpu(i)) {
8748 : sgs->idle_cpus++;
8749 : /* Idle cpu can't have misfit task */
8750 : continue;
8751 : }
8752 :
8753 : if (local_group)
8754 : continue;
8755 :
8756 : /* Check for a misfit task on the cpu */
8757 : if (env->sd->flags & SD_ASYM_CPUCAPACITY &&
8758 : sgs->group_misfit_task_load < rq->misfit_task_load) {
8759 : sgs->group_misfit_task_load = rq->misfit_task_load;
8760 : *sg_status |= SG_OVERLOAD;
8761 : }
8762 : }
8763 :
8764 : sgs->group_capacity = group->sgc->capacity;
8765 :
8766 : sgs->group_weight = group->group_weight;
8767 :
8768 : /* Check if dst CPU is idle and preferred to this group */
8769 : if (!local_group && env->sd->flags & SD_ASYM_PACKING &&
8770 : env->idle != CPU_NOT_IDLE && sgs->sum_h_nr_running &&
8771 : sched_asym(env, sds, sgs, group)) {
8772 : sgs->group_asym_packing = 1;
8773 : }
8774 :
8775 : sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs);
8776 :
8777 : /* Computing avg_load makes sense only when group is overloaded */
8778 : if (sgs->group_type == group_overloaded)
8779 : sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) /
8780 : sgs->group_capacity;
8781 : }
8782 :
8783 : /**
8784 : * update_sd_pick_busiest - return 1 on busiest group
8785 : * @env: The load balancing environment.
8786 : * @sds: sched_domain statistics
8787 : * @sg: sched_group candidate to be checked for being the busiest
8788 : * @sgs: sched_group statistics
8789 : *
8790 : * Determine if @sg is a busier group than the previously selected
8791 : * busiest group.
8792 : *
8793 : * Return: %true if @sg is a busier group than the previously selected
8794 : * busiest group. %false otherwise.
8795 : */
8796 : static bool update_sd_pick_busiest(struct lb_env *env,
8797 : struct sd_lb_stats *sds,
8798 : struct sched_group *sg,
8799 : struct sg_lb_stats *sgs)
8800 : {
8801 : struct sg_lb_stats *busiest = &sds->busiest_stat;
8802 :
8803 : /* Make sure that there is at least one task to pull */
8804 : if (!sgs->sum_h_nr_running)
8805 : return false;
8806 :
8807 : /*
8808 : * Don't try to pull misfit tasks we can't help.
8809 : * We can use max_capacity here as reduction in capacity on some
8810 : * CPUs in the group should either be possible to resolve
8811 : * internally or be covered by avg_load imbalance (eventually).
8812 : */
8813 : if (sgs->group_type == group_misfit_task &&
8814 : (!capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) ||
8815 : sds->local_stat.group_type != group_has_spare))
8816 : return false;
8817 :
8818 : if (sgs->group_type > busiest->group_type)
8819 : return true;
8820 :
8821 : if (sgs->group_type < busiest->group_type)
8822 : return false;
8823 :
8824 : /*
8825 : * The candidate and the current busiest group are the same type of
8826 : * group. Let check which one is the busiest according to the type.
8827 : */
8828 :
8829 : switch (sgs->group_type) {
8830 : case group_overloaded:
8831 : /* Select the overloaded group with highest avg_load. */
8832 : if (sgs->avg_load <= busiest->avg_load)
8833 : return false;
8834 : break;
8835 :
8836 : case group_imbalanced:
8837 : /*
8838 : * Select the 1st imbalanced group as we don't have any way to
8839 : * choose one more than another.
8840 : */
8841 : return false;
8842 :
8843 : case group_asym_packing:
8844 : /* Prefer to move from lowest priority CPU's work */
8845 : if (sched_asym_prefer(sg->asym_prefer_cpu, sds->busiest->asym_prefer_cpu))
8846 : return false;
8847 : break;
8848 :
8849 : case group_misfit_task:
8850 : /*
8851 : * If we have more than one misfit sg go with the biggest
8852 : * misfit.
8853 : */
8854 : if (sgs->group_misfit_task_load < busiest->group_misfit_task_load)
8855 : return false;
8856 : break;
8857 :
8858 : case group_fully_busy:
8859 : /*
8860 : * Select the fully busy group with highest avg_load. In
8861 : * theory, there is no need to pull task from such kind of
8862 : * group because tasks have all compute capacity that they need
8863 : * but we can still improve the overall throughput by reducing
8864 : * contention when accessing shared HW resources.
8865 : *
8866 : * XXX for now avg_load is not computed and always 0 so we
8867 : * select the 1st one.
8868 : */
8869 : if (sgs->avg_load <= busiest->avg_load)
8870 : return false;
8871 : break;
8872 :
8873 : case group_has_spare:
8874 : /*
8875 : * Select not overloaded group with lowest number of idle cpus
8876 : * and highest number of running tasks. We could also compare
8877 : * the spare capacity which is more stable but it can end up
8878 : * that the group has less spare capacity but finally more idle
8879 : * CPUs which means less opportunity to pull tasks.
8880 : */
8881 : if (sgs->idle_cpus > busiest->idle_cpus)
8882 : return false;
8883 : else if ((sgs->idle_cpus == busiest->idle_cpus) &&
8884 : (sgs->sum_nr_running <= busiest->sum_nr_running))
8885 : return false;
8886 :
8887 : break;
8888 : }
8889 :
8890 : /*
8891 : * Candidate sg has no more than one task per CPU and has higher
8892 : * per-CPU capacity. Migrating tasks to less capable CPUs may harm
8893 : * throughput. Maximize throughput, power/energy consequences are not
8894 : * considered.
8895 : */
8896 : if ((env->sd->flags & SD_ASYM_CPUCAPACITY) &&
8897 : (sgs->group_type <= group_fully_busy) &&
8898 : (capacity_greater(sg->sgc->min_capacity, capacity_of(env->dst_cpu))))
8899 : return false;
8900 :
8901 : return true;
8902 : }
8903 :
8904 : #ifdef CONFIG_NUMA_BALANCING
8905 : static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs)
8906 : {
8907 : if (sgs->sum_h_nr_running > sgs->nr_numa_running)
8908 : return regular;
8909 : if (sgs->sum_h_nr_running > sgs->nr_preferred_running)
8910 : return remote;
8911 : return all;
8912 : }
8913 :
8914 : static inline enum fbq_type fbq_classify_rq(struct rq *rq)
8915 : {
8916 : if (rq->nr_running > rq->nr_numa_running)
8917 : return regular;
8918 : if (rq->nr_running > rq->nr_preferred_running)
8919 : return remote;
8920 : return all;
8921 : }
8922 : #else
8923 : static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs)
8924 : {
8925 : return all;
8926 : }
8927 :
8928 : static inline enum fbq_type fbq_classify_rq(struct rq *rq)
8929 : {
8930 : return regular;
8931 : }
8932 : #endif /* CONFIG_NUMA_BALANCING */
8933 :
8934 :
8935 : struct sg_lb_stats;
8936 :
8937 : /*
8938 : * task_running_on_cpu - return 1 if @p is running on @cpu.
8939 : */
8940 :
8941 : static unsigned int task_running_on_cpu(int cpu, struct task_struct *p)
8942 : {
8943 : /* Task has no contribution or is new */
8944 : if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))
8945 : return 0;
8946 :
8947 : if (task_on_rq_queued(p))
8948 : return 1;
8949 :
8950 : return 0;
8951 : }
8952 :
8953 : /**
8954 : * idle_cpu_without - would a given CPU be idle without p ?
8955 : * @cpu: the processor on which idleness is tested.
8956 : * @p: task which should be ignored.
8957 : *
8958 : * Return: 1 if the CPU would be idle. 0 otherwise.
8959 : */
8960 : static int idle_cpu_without(int cpu, struct task_struct *p)
8961 : {
8962 : struct rq *rq = cpu_rq(cpu);
8963 :
8964 : if (rq->curr != rq->idle && rq->curr != p)
8965 : return 0;
8966 :
8967 : /*
8968 : * rq->nr_running can't be used but an updated version without the
8969 : * impact of p on cpu must be used instead. The updated nr_running
8970 : * be computed and tested before calling idle_cpu_without().
8971 : */
8972 :
8973 : #ifdef CONFIG_SMP
8974 : if (rq->ttwu_pending)
8975 : return 0;
8976 : #endif
8977 :
8978 : return 1;
8979 : }
8980 :
8981 : /*
8982 : * update_sg_wakeup_stats - Update sched_group's statistics for wakeup.
8983 : * @sd: The sched_domain level to look for idlest group.
8984 : * @group: sched_group whose statistics are to be updated.
8985 : * @sgs: variable to hold the statistics for this group.
8986 : * @p: The task for which we look for the idlest group/CPU.
8987 : */
8988 : static inline void update_sg_wakeup_stats(struct sched_domain *sd,
8989 : struct sched_group *group,
8990 : struct sg_lb_stats *sgs,
8991 : struct task_struct *p)
8992 : {
8993 : int i, nr_running;
8994 :
8995 : memset(sgs, 0, sizeof(*sgs));
8996 :
8997 : for_each_cpu(i, sched_group_span(group)) {
8998 : struct rq *rq = cpu_rq(i);
8999 : unsigned int local;
9000 :
9001 : sgs->group_load += cpu_load_without(rq, p);
9002 : sgs->group_util += cpu_util_without(i, p);
9003 : sgs->group_runnable += cpu_runnable_without(rq, p);
9004 : local = task_running_on_cpu(i, p);
9005 : sgs->sum_h_nr_running += rq->cfs.h_nr_running - local;
9006 :
9007 : nr_running = rq->nr_running - local;
9008 : sgs->sum_nr_running += nr_running;
9009 :
9010 : /*
9011 : * No need to call idle_cpu_without() if nr_running is not 0
9012 : */
9013 : if (!nr_running && idle_cpu_without(i, p))
9014 : sgs->idle_cpus++;
9015 :
9016 : }
9017 :
9018 : /* Check if task fits in the group */
9019 : if (sd->flags & SD_ASYM_CPUCAPACITY &&
9020 : !task_fits_capacity(p, group->sgc->max_capacity)) {
9021 : sgs->group_misfit_task_load = 1;
9022 : }
9023 :
9024 : sgs->group_capacity = group->sgc->capacity;
9025 :
9026 : sgs->group_weight = group->group_weight;
9027 :
9028 : sgs->group_type = group_classify(sd->imbalance_pct, group, sgs);
9029 :
9030 : /*
9031 : * Computing avg_load makes sense only when group is fully busy or
9032 : * overloaded
9033 : */
9034 : if (sgs->group_type == group_fully_busy ||
9035 : sgs->group_type == group_overloaded)
9036 : sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) /
9037 : sgs->group_capacity;
9038 : }
9039 :
9040 : static bool update_pick_idlest(struct sched_group *idlest,
9041 : struct sg_lb_stats *idlest_sgs,
9042 : struct sched_group *group,
9043 : struct sg_lb_stats *sgs)
9044 : {
9045 : if (sgs->group_type < idlest_sgs->group_type)
9046 : return true;
9047 :
9048 : if (sgs->group_type > idlest_sgs->group_type)
9049 : return false;
9050 :
9051 : /*
9052 : * The candidate and the current idlest group are the same type of
9053 : * group. Let check which one is the idlest according to the type.
9054 : */
9055 :
9056 : switch (sgs->group_type) {
9057 : case group_overloaded:
9058 : case group_fully_busy:
9059 : /* Select the group with lowest avg_load. */
9060 : if (idlest_sgs->avg_load <= sgs->avg_load)
9061 : return false;
9062 : break;
9063 :
9064 : case group_imbalanced:
9065 : case group_asym_packing:
9066 : /* Those types are not used in the slow wakeup path */
9067 : return false;
9068 :
9069 : case group_misfit_task:
9070 : /* Select group with the highest max capacity */
9071 : if (idlest->sgc->max_capacity >= group->sgc->max_capacity)
9072 : return false;
9073 : break;
9074 :
9075 : case group_has_spare:
9076 : /* Select group with most idle CPUs */
9077 : if (idlest_sgs->idle_cpus > sgs->idle_cpus)
9078 : return false;
9079 :
9080 : /* Select group with lowest group_util */
9081 : if (idlest_sgs->idle_cpus == sgs->idle_cpus &&
9082 : idlest_sgs->group_util <= sgs->group_util)
9083 : return false;
9084 :
9085 : break;
9086 : }
9087 :
9088 : return true;
9089 : }
9090 :
9091 : /*
9092 : * Allow a NUMA imbalance if busy CPUs is less than 25% of the domain.
9093 : * This is an approximation as the number of running tasks may not be
9094 : * related to the number of busy CPUs due to sched_setaffinity.
9095 : */
9096 : static inline bool allow_numa_imbalance(int running, int imb_numa_nr)
9097 : {
9098 : return running <= imb_numa_nr;
9099 : }
9100 :
9101 : /*
9102 : * find_idlest_group() finds and returns the least busy CPU group within the
9103 : * domain.
9104 : *
9105 : * Assumes p is allowed on at least one CPU in sd.
9106 : */
9107 : static struct sched_group *
9108 : find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
9109 : {
9110 : struct sched_group *idlest = NULL, *local = NULL, *group = sd->groups;
9111 : struct sg_lb_stats local_sgs, tmp_sgs;
9112 : struct sg_lb_stats *sgs;
9113 : unsigned long imbalance;
9114 : struct sg_lb_stats idlest_sgs = {
9115 : .avg_load = UINT_MAX,
9116 : .group_type = group_overloaded,
9117 : };
9118 :
9119 : do {
9120 : int local_group;
9121 :
9122 : /* Skip over this group if it has no CPUs allowed */
9123 : if (!cpumask_intersects(sched_group_span(group),
9124 : p->cpus_ptr))
9125 : continue;
9126 :
9127 : /* Skip over this group if no cookie matched */
9128 : if (!sched_group_cookie_match(cpu_rq(this_cpu), p, group))
9129 : continue;
9130 :
9131 : local_group = cpumask_test_cpu(this_cpu,
9132 : sched_group_span(group));
9133 :
9134 : if (local_group) {
9135 : sgs = &local_sgs;
9136 : local = group;
9137 : } else {
9138 : sgs = &tmp_sgs;
9139 : }
9140 :
9141 : update_sg_wakeup_stats(sd, group, sgs, p);
9142 :
9143 : if (!local_group && update_pick_idlest(idlest, &idlest_sgs, group, sgs)) {
9144 : idlest = group;
9145 : idlest_sgs = *sgs;
9146 : }
9147 :
9148 : } while (group = group->next, group != sd->groups);
9149 :
9150 :
9151 : /* There is no idlest group to push tasks to */
9152 : if (!idlest)
9153 : return NULL;
9154 :
9155 : /* The local group has been skipped because of CPU affinity */
9156 : if (!local)
9157 : return idlest;
9158 :
9159 : /*
9160 : * If the local group is idler than the selected idlest group
9161 : * don't try and push the task.
9162 : */
9163 : if (local_sgs.group_type < idlest_sgs.group_type)
9164 : return NULL;
9165 :
9166 : /*
9167 : * If the local group is busier than the selected idlest group
9168 : * try and push the task.
9169 : */
9170 : if (local_sgs.group_type > idlest_sgs.group_type)
9171 : return idlest;
9172 :
9173 : switch (local_sgs.group_type) {
9174 : case group_overloaded:
9175 : case group_fully_busy:
9176 :
9177 : /* Calculate allowed imbalance based on load */
9178 : imbalance = scale_load_down(NICE_0_LOAD) *
9179 : (sd->imbalance_pct-100) / 100;
9180 :
9181 : /*
9182 : * When comparing groups across NUMA domains, it's possible for
9183 : * the local domain to be very lightly loaded relative to the
9184 : * remote domains but "imbalance" skews the comparison making
9185 : * remote CPUs look much more favourable. When considering
9186 : * cross-domain, add imbalance to the load on the remote node
9187 : * and consider staying local.
9188 : */
9189 :
9190 : if ((sd->flags & SD_NUMA) &&
9191 : ((idlest_sgs.avg_load + imbalance) >= local_sgs.avg_load))
9192 : return NULL;
9193 :
9194 : /*
9195 : * If the local group is less loaded than the selected
9196 : * idlest group don't try and push any tasks.
9197 : */
9198 : if (idlest_sgs.avg_load >= (local_sgs.avg_load + imbalance))
9199 : return NULL;
9200 :
9201 : if (100 * local_sgs.avg_load <= sd->imbalance_pct * idlest_sgs.avg_load)
9202 : return NULL;
9203 : break;
9204 :
9205 : case group_imbalanced:
9206 : case group_asym_packing:
9207 : /* Those type are not used in the slow wakeup path */
9208 : return NULL;
9209 :
9210 : case group_misfit_task:
9211 : /* Select group with the highest max capacity */
9212 : if (local->sgc->max_capacity >= idlest->sgc->max_capacity)
9213 : return NULL;
9214 : break;
9215 :
9216 : case group_has_spare:
9217 : if (sd->flags & SD_NUMA) {
9218 : #ifdef CONFIG_NUMA_BALANCING
9219 : int idlest_cpu;
9220 : /*
9221 : * If there is spare capacity at NUMA, try to select
9222 : * the preferred node
9223 : */
9224 : if (cpu_to_node(this_cpu) == p->numa_preferred_nid)
9225 : return NULL;
9226 :
9227 : idlest_cpu = cpumask_first(sched_group_span(idlest));
9228 : if (cpu_to_node(idlest_cpu) == p->numa_preferred_nid)
9229 : return idlest;
9230 : #endif
9231 : /*
9232 : * Otherwise, keep the task close to the wakeup source
9233 : * and improve locality if the number of running tasks
9234 : * would remain below threshold where an imbalance is
9235 : * allowed. If there is a real need of migration,
9236 : * periodic load balance will take care of it.
9237 : */
9238 : if (allow_numa_imbalance(local_sgs.sum_nr_running + 1, sd->imb_numa_nr))
9239 : return NULL;
9240 : }
9241 :
9242 : /*
9243 : * Select group with highest number of idle CPUs. We could also
9244 : * compare the utilization which is more stable but it can end
9245 : * up that the group has less spare capacity but finally more
9246 : * idle CPUs which means more opportunity to run task.
9247 : */
9248 : if (local_sgs.idle_cpus >= idlest_sgs.idle_cpus)
9249 : return NULL;
9250 : break;
9251 : }
9252 :
9253 : return idlest;
9254 : }
9255 :
9256 : /**
9257 : * update_sd_lb_stats - Update sched_domain's statistics for load balancing.
9258 : * @env: The load balancing environment.
9259 : * @sds: variable to hold the statistics for this sched_domain.
9260 : */
9261 :
9262 : static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds)
9263 : {
9264 : struct sched_domain *child = env->sd->child;
9265 : struct sched_group *sg = env->sd->groups;
9266 : struct sg_lb_stats *local = &sds->local_stat;
9267 : struct sg_lb_stats tmp_sgs;
9268 : int sg_status = 0;
9269 :
9270 : do {
9271 : struct sg_lb_stats *sgs = &tmp_sgs;
9272 : int local_group;
9273 :
9274 : local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(sg));
9275 : if (local_group) {
9276 : sds->local = sg;
9277 : sgs = local;
9278 :
9279 : if (env->idle != CPU_NEWLY_IDLE ||
9280 : time_after_eq(jiffies, sg->sgc->next_update))
9281 : update_group_capacity(env->sd, env->dst_cpu);
9282 : }
9283 :
9284 : update_sg_lb_stats(env, sds, sg, sgs, &sg_status);
9285 :
9286 : if (local_group)
9287 : goto next_group;
9288 :
9289 :
9290 : if (update_sd_pick_busiest(env, sds, sg, sgs)) {
9291 : sds->busiest = sg;
9292 : sds->busiest_stat = *sgs;
9293 : }
9294 :
9295 : next_group:
9296 : /* Now, start updating sd_lb_stats */
9297 : sds->total_load += sgs->group_load;
9298 : sds->total_capacity += sgs->group_capacity;
9299 :
9300 : sg = sg->next;
9301 : } while (sg != env->sd->groups);
9302 :
9303 : /* Tag domain that child domain prefers tasks go to siblings first */
9304 : sds->prefer_sibling = child && child->flags & SD_PREFER_SIBLING;
9305 :
9306 :
9307 : if (env->sd->flags & SD_NUMA)
9308 : env->fbq_type = fbq_classify_group(&sds->busiest_stat);
9309 :
9310 : if (!env->sd->parent) {
9311 : struct root_domain *rd = env->dst_rq->rd;
9312 :
9313 : /* update overload indicator if we are at root domain */
9314 : WRITE_ONCE(rd->overload, sg_status & SG_OVERLOAD);
9315 :
9316 : /* Update over-utilization (tipping point, U >= 0) indicator */
9317 : WRITE_ONCE(rd->overutilized, sg_status & SG_OVERUTILIZED);
9318 : trace_sched_overutilized_tp(rd, sg_status & SG_OVERUTILIZED);
9319 : } else if (sg_status & SG_OVERUTILIZED) {
9320 : struct root_domain *rd = env->dst_rq->rd;
9321 :
9322 : WRITE_ONCE(rd->overutilized, SG_OVERUTILIZED);
9323 : trace_sched_overutilized_tp(rd, SG_OVERUTILIZED);
9324 : }
9325 : }
9326 :
9327 : #define NUMA_IMBALANCE_MIN 2
9328 :
9329 : static inline long adjust_numa_imbalance(int imbalance,
9330 : int dst_running, int imb_numa_nr)
9331 : {
9332 : if (!allow_numa_imbalance(dst_running, imb_numa_nr))
9333 : return imbalance;
9334 :
9335 : /*
9336 : * Allow a small imbalance based on a simple pair of communicating
9337 : * tasks that remain local when the destination is lightly loaded.
9338 : */
9339 : if (imbalance <= NUMA_IMBALANCE_MIN)
9340 : return 0;
9341 :
9342 : return imbalance;
9343 : }
9344 :
9345 : /**
9346 : * calculate_imbalance - Calculate the amount of imbalance present within the
9347 : * groups of a given sched_domain during load balance.
9348 : * @env: load balance environment
9349 : * @sds: statistics of the sched_domain whose imbalance is to be calculated.
9350 : */
9351 : static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
9352 : {
9353 : struct sg_lb_stats *local, *busiest;
9354 :
9355 : local = &sds->local_stat;
9356 : busiest = &sds->busiest_stat;
9357 :
9358 : if (busiest->group_type == group_misfit_task) {
9359 : /* Set imbalance to allow misfit tasks to be balanced. */
9360 : env->migration_type = migrate_misfit;
9361 : env->imbalance = 1;
9362 : return;
9363 : }
9364 :
9365 : if (busiest->group_type == group_asym_packing) {
9366 : /*
9367 : * In case of asym capacity, we will try to migrate all load to
9368 : * the preferred CPU.
9369 : */
9370 : env->migration_type = migrate_task;
9371 : env->imbalance = busiest->sum_h_nr_running;
9372 : return;
9373 : }
9374 :
9375 : if (busiest->group_type == group_imbalanced) {
9376 : /*
9377 : * In the group_imb case we cannot rely on group-wide averages
9378 : * to ensure CPU-load equilibrium, try to move any task to fix
9379 : * the imbalance. The next load balance will take care of
9380 : * balancing back the system.
9381 : */
9382 : env->migration_type = migrate_task;
9383 : env->imbalance = 1;
9384 : return;
9385 : }
9386 :
9387 : /*
9388 : * Try to use spare capacity of local group without overloading it or
9389 : * emptying busiest.
9390 : */
9391 : if (local->group_type == group_has_spare) {
9392 : if ((busiest->group_type > group_fully_busy) &&
9393 : !(env->sd->flags & SD_SHARE_PKG_RESOURCES)) {
9394 : /*
9395 : * If busiest is overloaded, try to fill spare
9396 : * capacity. This might end up creating spare capacity
9397 : * in busiest or busiest still being overloaded but
9398 : * there is no simple way to directly compute the
9399 : * amount of load to migrate in order to balance the
9400 : * system.
9401 : */
9402 : env->migration_type = migrate_util;
9403 : env->imbalance = max(local->group_capacity, local->group_util) -
9404 : local->group_util;
9405 :
9406 : /*
9407 : * In some cases, the group's utilization is max or even
9408 : * higher than capacity because of migrations but the
9409 : * local CPU is (newly) idle. There is at least one
9410 : * waiting task in this overloaded busiest group. Let's
9411 : * try to pull it.
9412 : */
9413 : if (env->idle != CPU_NOT_IDLE && env->imbalance == 0) {
9414 : env->migration_type = migrate_task;
9415 : env->imbalance = 1;
9416 : }
9417 :
9418 : return;
9419 : }
9420 :
9421 : if (busiest->group_weight == 1 || sds->prefer_sibling) {
9422 : unsigned int nr_diff = busiest->sum_nr_running;
9423 : /*
9424 : * When prefer sibling, evenly spread running tasks on
9425 : * groups.
9426 : */
9427 : env->migration_type = migrate_task;
9428 : lsub_positive(&nr_diff, local->sum_nr_running);
9429 : env->imbalance = nr_diff >> 1;
9430 : } else {
9431 :
9432 : /*
9433 : * If there is no overload, we just want to even the number of
9434 : * idle cpus.
9435 : */
9436 : env->migration_type = migrate_task;
9437 : env->imbalance = max_t(long, 0, (local->idle_cpus -
9438 : busiest->idle_cpus) >> 1);
9439 : }
9440 :
9441 : /* Consider allowing a small imbalance between NUMA groups */
9442 : if (env->sd->flags & SD_NUMA) {
9443 : env->imbalance = adjust_numa_imbalance(env->imbalance,
9444 : local->sum_nr_running + 1, env->sd->imb_numa_nr);
9445 : }
9446 :
9447 : return;
9448 : }
9449 :
9450 : /*
9451 : * Local is fully busy but has to take more load to relieve the
9452 : * busiest group
9453 : */
9454 : if (local->group_type < group_overloaded) {
9455 : /*
9456 : * Local will become overloaded so the avg_load metrics are
9457 : * finally needed.
9458 : */
9459 :
9460 : local->avg_load = (local->group_load * SCHED_CAPACITY_SCALE) /
9461 : local->group_capacity;
9462 :
9463 : sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) /
9464 : sds->total_capacity;
9465 : /*
9466 : * If the local group is more loaded than the selected
9467 : * busiest group don't try to pull any tasks.
9468 : */
9469 : if (local->avg_load >= busiest->avg_load) {
9470 : env->imbalance = 0;
9471 : return;
9472 : }
9473 : }
9474 :
9475 : /*
9476 : * Both group are or will become overloaded and we're trying to get all
9477 : * the CPUs to the average_load, so we don't want to push ourselves
9478 : * above the average load, nor do we wish to reduce the max loaded CPU
9479 : * below the average load. At the same time, we also don't want to
9480 : * reduce the group load below the group capacity. Thus we look for
9481 : * the minimum possible imbalance.
9482 : */
9483 : env->migration_type = migrate_load;
9484 : env->imbalance = min(
9485 : (busiest->avg_load - sds->avg_load) * busiest->group_capacity,
9486 : (sds->avg_load - local->avg_load) * local->group_capacity
9487 : ) / SCHED_CAPACITY_SCALE;
9488 : }
9489 :
9490 : /******* find_busiest_group() helpers end here *********************/
9491 :
9492 : /*
9493 : * Decision matrix according to the local and busiest group type:
9494 : *
9495 : * busiest \ local has_spare fully_busy misfit asym imbalanced overloaded
9496 : * has_spare nr_idle balanced N/A N/A balanced balanced
9497 : * fully_busy nr_idle nr_idle N/A N/A balanced balanced
9498 : * misfit_task force N/A N/A N/A force force
9499 : * asym_packing force force N/A N/A force force
9500 : * imbalanced force force N/A N/A force force
9501 : * overloaded force force N/A N/A force avg_load
9502 : *
9503 : * N/A : Not Applicable because already filtered while updating
9504 : * statistics.
9505 : * balanced : The system is balanced for these 2 groups.
9506 : * force : Calculate the imbalance as load migration is probably needed.
9507 : * avg_load : Only if imbalance is significant enough.
9508 : * nr_idle : dst_cpu is not busy and the number of idle CPUs is quite
9509 : * different in groups.
9510 : */
9511 :
9512 : /**
9513 : * find_busiest_group - Returns the busiest group within the sched_domain
9514 : * if there is an imbalance.
9515 : * @env: The load balancing environment.
9516 : *
9517 : * Also calculates the amount of runnable load which should be moved
9518 : * to restore balance.
9519 : *
9520 : * Return: - The busiest group if imbalance exists.
9521 : */
9522 : static struct sched_group *find_busiest_group(struct lb_env *env)
9523 : {
9524 : struct sg_lb_stats *local, *busiest;
9525 : struct sd_lb_stats sds;
9526 :
9527 : init_sd_lb_stats(&sds);
9528 :
9529 : /*
9530 : * Compute the various statistics relevant for load balancing at
9531 : * this level.
9532 : */
9533 : update_sd_lb_stats(env, &sds);
9534 :
9535 : if (sched_energy_enabled()) {
9536 : struct root_domain *rd = env->dst_rq->rd;
9537 :
9538 : if (rcu_dereference(rd->pd) && !READ_ONCE(rd->overutilized))
9539 : goto out_balanced;
9540 : }
9541 :
9542 : local = &sds.local_stat;
9543 : busiest = &sds.busiest_stat;
9544 :
9545 : /* There is no busy sibling group to pull tasks from */
9546 : if (!sds.busiest)
9547 : goto out_balanced;
9548 :
9549 : /* Misfit tasks should be dealt with regardless of the avg load */
9550 : if (busiest->group_type == group_misfit_task)
9551 : goto force_balance;
9552 :
9553 : /* ASYM feature bypasses nice load balance check */
9554 : if (busiest->group_type == group_asym_packing)
9555 : goto force_balance;
9556 :
9557 : /*
9558 : * If the busiest group is imbalanced the below checks don't
9559 : * work because they assume all things are equal, which typically
9560 : * isn't true due to cpus_ptr constraints and the like.
9561 : */
9562 : if (busiest->group_type == group_imbalanced)
9563 : goto force_balance;
9564 :
9565 : /*
9566 : * If the local group is busier than the selected busiest group
9567 : * don't try and pull any tasks.
9568 : */
9569 : if (local->group_type > busiest->group_type)
9570 : goto out_balanced;
9571 :
9572 : /*
9573 : * When groups are overloaded, use the avg_load to ensure fairness
9574 : * between tasks.
9575 : */
9576 : if (local->group_type == group_overloaded) {
9577 : /*
9578 : * If the local group is more loaded than the selected
9579 : * busiest group don't try to pull any tasks.
9580 : */
9581 : if (local->avg_load >= busiest->avg_load)
9582 : goto out_balanced;
9583 :
9584 : /* XXX broken for overlapping NUMA groups */
9585 : sds.avg_load = (sds.total_load * SCHED_CAPACITY_SCALE) /
9586 : sds.total_capacity;
9587 :
9588 : /*
9589 : * Don't pull any tasks if this group is already above the
9590 : * domain average load.
9591 : */
9592 : if (local->avg_load >= sds.avg_load)
9593 : goto out_balanced;
9594 :
9595 : /*
9596 : * If the busiest group is more loaded, use imbalance_pct to be
9597 : * conservative.
9598 : */
9599 : if (100 * busiest->avg_load <=
9600 : env->sd->imbalance_pct * local->avg_load)
9601 : goto out_balanced;
9602 : }
9603 :
9604 : /* Try to move all excess tasks to child's sibling domain */
9605 : if (sds.prefer_sibling && local->group_type == group_has_spare &&
9606 : busiest->sum_nr_running > local->sum_nr_running + 1)
9607 : goto force_balance;
9608 :
9609 : if (busiest->group_type != group_overloaded) {
9610 : if (env->idle == CPU_NOT_IDLE)
9611 : /*
9612 : * If the busiest group is not overloaded (and as a
9613 : * result the local one too) but this CPU is already
9614 : * busy, let another idle CPU try to pull task.
9615 : */
9616 : goto out_balanced;
9617 :
9618 : if (busiest->group_weight > 1 &&
9619 : local->idle_cpus <= (busiest->idle_cpus + 1))
9620 : /*
9621 : * If the busiest group is not overloaded
9622 : * and there is no imbalance between this and busiest
9623 : * group wrt idle CPUs, it is balanced. The imbalance
9624 : * becomes significant if the diff is greater than 1
9625 : * otherwise we might end up to just move the imbalance
9626 : * on another group. Of course this applies only if
9627 : * there is more than 1 CPU per group.
9628 : */
9629 : goto out_balanced;
9630 :
9631 : if (busiest->sum_h_nr_running == 1)
9632 : /*
9633 : * busiest doesn't have any tasks waiting to run
9634 : */
9635 : goto out_balanced;
9636 : }
9637 :
9638 : force_balance:
9639 : /* Looks like there is an imbalance. Compute it */
9640 : calculate_imbalance(env, &sds);
9641 : return env->imbalance ? sds.busiest : NULL;
9642 :
9643 : out_balanced:
9644 : env->imbalance = 0;
9645 : return NULL;
9646 : }
9647 :
9648 : /*
9649 : * find_busiest_queue - find the busiest runqueue among the CPUs in the group.
9650 : */
9651 : static struct rq *find_busiest_queue(struct lb_env *env,
9652 : struct sched_group *group)
9653 : {
9654 : struct rq *busiest = NULL, *rq;
9655 : unsigned long busiest_util = 0, busiest_load = 0, busiest_capacity = 1;
9656 : unsigned int busiest_nr = 0;
9657 : int i;
9658 :
9659 : for_each_cpu_and(i, sched_group_span(group), env->cpus) {
9660 : unsigned long capacity, load, util;
9661 : unsigned int nr_running;
9662 : enum fbq_type rt;
9663 :
9664 : rq = cpu_rq(i);
9665 : rt = fbq_classify_rq(rq);
9666 :
9667 : /*
9668 : * We classify groups/runqueues into three groups:
9669 : * - regular: there are !numa tasks
9670 : * - remote: there are numa tasks that run on the 'wrong' node
9671 : * - all: there is no distinction
9672 : *
9673 : * In order to avoid migrating ideally placed numa tasks,
9674 : * ignore those when there's better options.
9675 : *
9676 : * If we ignore the actual busiest queue to migrate another
9677 : * task, the next balance pass can still reduce the busiest
9678 : * queue by moving tasks around inside the node.
9679 : *
9680 : * If we cannot move enough load due to this classification
9681 : * the next pass will adjust the group classification and
9682 : * allow migration of more tasks.
9683 : *
9684 : * Both cases only affect the total convergence complexity.
9685 : */
9686 : if (rt > env->fbq_type)
9687 : continue;
9688 :
9689 : nr_running = rq->cfs.h_nr_running;
9690 : if (!nr_running)
9691 : continue;
9692 :
9693 : capacity = capacity_of(i);
9694 :
9695 : /*
9696 : * For ASYM_CPUCAPACITY domains, don't pick a CPU that could
9697 : * eventually lead to active_balancing high->low capacity.
9698 : * Higher per-CPU capacity is considered better than balancing
9699 : * average load.
9700 : */
9701 : if (env->sd->flags & SD_ASYM_CPUCAPACITY &&
9702 : !capacity_greater(capacity_of(env->dst_cpu), capacity) &&
9703 : nr_running == 1)
9704 : continue;
9705 :
9706 : /* Make sure we only pull tasks from a CPU of lower priority */
9707 : if ((env->sd->flags & SD_ASYM_PACKING) &&
9708 : sched_asym_prefer(i, env->dst_cpu) &&
9709 : nr_running == 1)
9710 : continue;
9711 :
9712 : switch (env->migration_type) {
9713 : case migrate_load:
9714 : /*
9715 : * When comparing with load imbalance, use cpu_load()
9716 : * which is not scaled with the CPU capacity.
9717 : */
9718 : load = cpu_load(rq);
9719 :
9720 : if (nr_running == 1 && load > env->imbalance &&
9721 : !check_cpu_capacity(rq, env->sd))
9722 : break;
9723 :
9724 : /*
9725 : * For the load comparisons with the other CPUs,
9726 : * consider the cpu_load() scaled with the CPU
9727 : * capacity, so that the load can be moved away
9728 : * from the CPU that is potentially running at a
9729 : * lower capacity.
9730 : *
9731 : * Thus we're looking for max(load_i / capacity_i),
9732 : * crosswise multiplication to rid ourselves of the
9733 : * division works out to:
9734 : * load_i * capacity_j > load_j * capacity_i;
9735 : * where j is our previous maximum.
9736 : */
9737 : if (load * busiest_capacity > busiest_load * capacity) {
9738 : busiest_load = load;
9739 : busiest_capacity = capacity;
9740 : busiest = rq;
9741 : }
9742 : break;
9743 :
9744 : case migrate_util:
9745 : util = cpu_util_cfs(i);
9746 :
9747 : /*
9748 : * Don't try to pull utilization from a CPU with one
9749 : * running task. Whatever its utilization, we will fail
9750 : * detach the task.
9751 : */
9752 : if (nr_running <= 1)
9753 : continue;
9754 :
9755 : if (busiest_util < util) {
9756 : busiest_util = util;
9757 : busiest = rq;
9758 : }
9759 : break;
9760 :
9761 : case migrate_task:
9762 : if (busiest_nr < nr_running) {
9763 : busiest_nr = nr_running;
9764 : busiest = rq;
9765 : }
9766 : break;
9767 :
9768 : case migrate_misfit:
9769 : /*
9770 : * For ASYM_CPUCAPACITY domains with misfit tasks we
9771 : * simply seek the "biggest" misfit task.
9772 : */
9773 : if (rq->misfit_task_load > busiest_load) {
9774 : busiest_load = rq->misfit_task_load;
9775 : busiest = rq;
9776 : }
9777 :
9778 : break;
9779 :
9780 : }
9781 : }
9782 :
9783 : return busiest;
9784 : }
9785 :
9786 : /*
9787 : * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but
9788 : * so long as it is large enough.
9789 : */
9790 : #define MAX_PINNED_INTERVAL 512
9791 :
9792 : static inline bool
9793 : asym_active_balance(struct lb_env *env)
9794 : {
9795 : /*
9796 : * ASYM_PACKING needs to force migrate tasks from busy but
9797 : * lower priority CPUs in order to pack all tasks in the
9798 : * highest priority CPUs.
9799 : */
9800 : return env->idle != CPU_NOT_IDLE && (env->sd->flags & SD_ASYM_PACKING) &&
9801 : sched_asym_prefer(env->dst_cpu, env->src_cpu);
9802 : }
9803 :
9804 : static inline bool
9805 : imbalanced_active_balance(struct lb_env *env)
9806 : {
9807 : struct sched_domain *sd = env->sd;
9808 :
9809 : /*
9810 : * The imbalanced case includes the case of pinned tasks preventing a fair
9811 : * distribution of the load on the system but also the even distribution of the
9812 : * threads on a system with spare capacity
9813 : */
9814 : if ((env->migration_type == migrate_task) &&
9815 : (sd->nr_balance_failed > sd->cache_nice_tries+2))
9816 : return 1;
9817 :
9818 : return 0;
9819 : }
9820 :
9821 : static int need_active_balance(struct lb_env *env)
9822 : {
9823 : struct sched_domain *sd = env->sd;
9824 :
9825 : if (asym_active_balance(env))
9826 : return 1;
9827 :
9828 : if (imbalanced_active_balance(env))
9829 : return 1;
9830 :
9831 : /*
9832 : * The dst_cpu is idle and the src_cpu CPU has only 1 CFS task.
9833 : * It's worth migrating the task if the src_cpu's capacity is reduced
9834 : * because of other sched_class or IRQs if more capacity stays
9835 : * available on dst_cpu.
9836 : */
9837 : if ((env->idle != CPU_NOT_IDLE) &&
9838 : (env->src_rq->cfs.h_nr_running == 1)) {
9839 : if ((check_cpu_capacity(env->src_rq, sd)) &&
9840 : (capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100))
9841 : return 1;
9842 : }
9843 :
9844 : if (env->migration_type == migrate_misfit)
9845 : return 1;
9846 :
9847 : return 0;
9848 : }
9849 :
9850 : static int active_load_balance_cpu_stop(void *data);
9851 :
9852 : static int should_we_balance(struct lb_env *env)
9853 : {
9854 : struct sched_group *sg = env->sd->groups;
9855 : int cpu;
9856 :
9857 : /*
9858 : * Ensure the balancing environment is consistent; can happen
9859 : * when the softirq triggers 'during' hotplug.
9860 : */
9861 : if (!cpumask_test_cpu(env->dst_cpu, env->cpus))
9862 : return 0;
9863 :
9864 : /*
9865 : * In the newly idle case, we will allow all the CPUs
9866 : * to do the newly idle load balance.
9867 : */
9868 : if (env->idle == CPU_NEWLY_IDLE)
9869 : return 1;
9870 :
9871 : /* Try to find first idle CPU */
9872 : for_each_cpu_and(cpu, group_balance_mask(sg), env->cpus) {
9873 : if (!idle_cpu(cpu))
9874 : continue;
9875 :
9876 : /* Are we the first idle CPU? */
9877 : return cpu == env->dst_cpu;
9878 : }
9879 :
9880 : /* Are we the first CPU of this group ? */
9881 : return group_balance_cpu(sg) == env->dst_cpu;
9882 : }
9883 :
9884 : /*
9885 : * Check this_cpu to ensure it is balanced within domain. Attempt to move
9886 : * tasks if there is an imbalance.
9887 : */
9888 : static int load_balance(int this_cpu, struct rq *this_rq,
9889 : struct sched_domain *sd, enum cpu_idle_type idle,
9890 : int *continue_balancing)
9891 : {
9892 : int ld_moved, cur_ld_moved, active_balance = 0;
9893 : struct sched_domain *sd_parent = sd->parent;
9894 : struct sched_group *group;
9895 : struct rq *busiest;
9896 : struct rq_flags rf;
9897 : struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask);
9898 :
9899 : struct lb_env env = {
9900 : .sd = sd,
9901 : .dst_cpu = this_cpu,
9902 : .dst_rq = this_rq,
9903 : .dst_grpmask = sched_group_span(sd->groups),
9904 : .idle = idle,
9905 : .loop_break = sched_nr_migrate_break,
9906 : .cpus = cpus,
9907 : .fbq_type = all,
9908 : .tasks = LIST_HEAD_INIT(env.tasks),
9909 : };
9910 :
9911 : cpumask_and(cpus, sched_domain_span(sd), cpu_active_mask);
9912 :
9913 : schedstat_inc(sd->lb_count[idle]);
9914 :
9915 : redo:
9916 : if (!should_we_balance(&env)) {
9917 : *continue_balancing = 0;
9918 : goto out_balanced;
9919 : }
9920 :
9921 : group = find_busiest_group(&env);
9922 : if (!group) {
9923 : schedstat_inc(sd->lb_nobusyg[idle]);
9924 : goto out_balanced;
9925 : }
9926 :
9927 : busiest = find_busiest_queue(&env, group);
9928 : if (!busiest) {
9929 : schedstat_inc(sd->lb_nobusyq[idle]);
9930 : goto out_balanced;
9931 : }
9932 :
9933 : BUG_ON(busiest == env.dst_rq);
9934 :
9935 : schedstat_add(sd->lb_imbalance[idle], env.imbalance);
9936 :
9937 : env.src_cpu = busiest->cpu;
9938 : env.src_rq = busiest;
9939 :
9940 : ld_moved = 0;
9941 : /* Clear this flag as soon as we find a pullable task */
9942 : env.flags |= LBF_ALL_PINNED;
9943 : if (busiest->nr_running > 1) {
9944 : /*
9945 : * Attempt to move tasks. If find_busiest_group has found
9946 : * an imbalance but busiest->nr_running <= 1, the group is
9947 : * still unbalanced. ld_moved simply stays zero, so it is
9948 : * correctly treated as an imbalance.
9949 : */
9950 : env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running);
9951 :
9952 : more_balance:
9953 : rq_lock_irqsave(busiest, &rf);
9954 : update_rq_clock(busiest);
9955 :
9956 : /*
9957 : * cur_ld_moved - load moved in current iteration
9958 : * ld_moved - cumulative load moved across iterations
9959 : */
9960 : cur_ld_moved = detach_tasks(&env);
9961 :
9962 : /*
9963 : * We've detached some tasks from busiest_rq. Every
9964 : * task is masked "TASK_ON_RQ_MIGRATING", so we can safely
9965 : * unlock busiest->lock, and we are able to be sure
9966 : * that nobody can manipulate the tasks in parallel.
9967 : * See task_rq_lock() family for the details.
9968 : */
9969 :
9970 : rq_unlock(busiest, &rf);
9971 :
9972 : if (cur_ld_moved) {
9973 : attach_tasks(&env);
9974 : ld_moved += cur_ld_moved;
9975 : }
9976 :
9977 : local_irq_restore(rf.flags);
9978 :
9979 : if (env.flags & LBF_NEED_BREAK) {
9980 : env.flags &= ~LBF_NEED_BREAK;
9981 : goto more_balance;
9982 : }
9983 :
9984 : /*
9985 : * Revisit (affine) tasks on src_cpu that couldn't be moved to
9986 : * us and move them to an alternate dst_cpu in our sched_group
9987 : * where they can run. The upper limit on how many times we
9988 : * iterate on same src_cpu is dependent on number of CPUs in our
9989 : * sched_group.
9990 : *
9991 : * This changes load balance semantics a bit on who can move
9992 : * load to a given_cpu. In addition to the given_cpu itself
9993 : * (or a ilb_cpu acting on its behalf where given_cpu is
9994 : * nohz-idle), we now have balance_cpu in a position to move
9995 : * load to given_cpu. In rare situations, this may cause
9996 : * conflicts (balance_cpu and given_cpu/ilb_cpu deciding
9997 : * _independently_ and at _same_ time to move some load to
9998 : * given_cpu) causing excess load to be moved to given_cpu.
9999 : * This however should not happen so much in practice and
10000 : * moreover subsequent load balance cycles should correct the
10001 : * excess load moved.
10002 : */
10003 : if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) {
10004 :
10005 : /* Prevent to re-select dst_cpu via env's CPUs */
10006 : __cpumask_clear_cpu(env.dst_cpu, env.cpus);
10007 :
10008 : env.dst_rq = cpu_rq(env.new_dst_cpu);
10009 : env.dst_cpu = env.new_dst_cpu;
10010 : env.flags &= ~LBF_DST_PINNED;
10011 : env.loop = 0;
10012 : env.loop_break = sched_nr_migrate_break;
10013 :
10014 : /*
10015 : * Go back to "more_balance" rather than "redo" since we
10016 : * need to continue with same src_cpu.
10017 : */
10018 : goto more_balance;
10019 : }
10020 :
10021 : /*
10022 : * We failed to reach balance because of affinity.
10023 : */
10024 : if (sd_parent) {
10025 : int *group_imbalance = &sd_parent->groups->sgc->imbalance;
10026 :
10027 : if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0)
10028 : *group_imbalance = 1;
10029 : }
10030 :
10031 : /* All tasks on this runqueue were pinned by CPU affinity */
10032 : if (unlikely(env.flags & LBF_ALL_PINNED)) {
10033 : __cpumask_clear_cpu(cpu_of(busiest), cpus);
10034 : /*
10035 : * Attempting to continue load balancing at the current
10036 : * sched_domain level only makes sense if there are
10037 : * active CPUs remaining as possible busiest CPUs to
10038 : * pull load from which are not contained within the
10039 : * destination group that is receiving any migrated
10040 : * load.
10041 : */
10042 : if (!cpumask_subset(cpus, env.dst_grpmask)) {
10043 : env.loop = 0;
10044 : env.loop_break = sched_nr_migrate_break;
10045 : goto redo;
10046 : }
10047 : goto out_all_pinned;
10048 : }
10049 : }
10050 :
10051 : if (!ld_moved) {
10052 : schedstat_inc(sd->lb_failed[idle]);
10053 : /*
10054 : * Increment the failure counter only on periodic balance.
10055 : * We do not want newidle balance, which can be very
10056 : * frequent, pollute the failure counter causing
10057 : * excessive cache_hot migrations and active balances.
10058 : */
10059 : if (idle != CPU_NEWLY_IDLE)
10060 : sd->nr_balance_failed++;
10061 :
10062 : if (need_active_balance(&env)) {
10063 : unsigned long flags;
10064 :
10065 : raw_spin_rq_lock_irqsave(busiest, flags);
10066 :
10067 : /*
10068 : * Don't kick the active_load_balance_cpu_stop,
10069 : * if the curr task on busiest CPU can't be
10070 : * moved to this_cpu:
10071 : */
10072 : if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) {
10073 : raw_spin_rq_unlock_irqrestore(busiest, flags);
10074 : goto out_one_pinned;
10075 : }
10076 :
10077 : /* Record that we found at least one task that could run on this_cpu */
10078 : env.flags &= ~LBF_ALL_PINNED;
10079 :
10080 : /*
10081 : * ->active_balance synchronizes accesses to
10082 : * ->active_balance_work. Once set, it's cleared
10083 : * only after active load balance is finished.
10084 : */
10085 : if (!busiest->active_balance) {
10086 : busiest->active_balance = 1;
10087 : busiest->push_cpu = this_cpu;
10088 : active_balance = 1;
10089 : }
10090 : raw_spin_rq_unlock_irqrestore(busiest, flags);
10091 :
10092 : if (active_balance) {
10093 : stop_one_cpu_nowait(cpu_of(busiest),
10094 : active_load_balance_cpu_stop, busiest,
10095 : &busiest->active_balance_work);
10096 : }
10097 : }
10098 : } else {
10099 : sd->nr_balance_failed = 0;
10100 : }
10101 :
10102 : if (likely(!active_balance) || need_active_balance(&env)) {
10103 : /* We were unbalanced, so reset the balancing interval */
10104 : sd->balance_interval = sd->min_interval;
10105 : }
10106 :
10107 : goto out;
10108 :
10109 : out_balanced:
10110 : /*
10111 : * We reach balance although we may have faced some affinity
10112 : * constraints. Clear the imbalance flag only if other tasks got
10113 : * a chance to move and fix the imbalance.
10114 : */
10115 : if (sd_parent && !(env.flags & LBF_ALL_PINNED)) {
10116 : int *group_imbalance = &sd_parent->groups->sgc->imbalance;
10117 :
10118 : if (*group_imbalance)
10119 : *group_imbalance = 0;
10120 : }
10121 :
10122 : out_all_pinned:
10123 : /*
10124 : * We reach balance because all tasks are pinned at this level so
10125 : * we can't migrate them. Let the imbalance flag set so parent level
10126 : * can try to migrate them.
10127 : */
10128 : schedstat_inc(sd->lb_balanced[idle]);
10129 :
10130 : sd->nr_balance_failed = 0;
10131 :
10132 : out_one_pinned:
10133 : ld_moved = 0;
10134 :
10135 : /*
10136 : * newidle_balance() disregards balance intervals, so we could
10137 : * repeatedly reach this code, which would lead to balance_interval
10138 : * skyrocketing in a short amount of time. Skip the balance_interval
10139 : * increase logic to avoid that.
10140 : */
10141 : if (env.idle == CPU_NEWLY_IDLE)
10142 : goto out;
10143 :
10144 : /* tune up the balancing interval */
10145 : if ((env.flags & LBF_ALL_PINNED &&
10146 : sd->balance_interval < MAX_PINNED_INTERVAL) ||
10147 : sd->balance_interval < sd->max_interval)
10148 : sd->balance_interval *= 2;
10149 : out:
10150 : return ld_moved;
10151 : }
10152 :
10153 : static inline unsigned long
10154 : get_sd_balance_interval(struct sched_domain *sd, int cpu_busy)
10155 : {
10156 : unsigned long interval = sd->balance_interval;
10157 :
10158 : if (cpu_busy)
10159 : interval *= sd->busy_factor;
10160 :
10161 : /* scale ms to jiffies */
10162 : interval = msecs_to_jiffies(interval);
10163 :
10164 : /*
10165 : * Reduce likelihood of busy balancing at higher domains racing with
10166 : * balancing at lower domains by preventing their balancing periods
10167 : * from being multiples of each other.
10168 : */
10169 : if (cpu_busy)
10170 : interval -= 1;
10171 :
10172 : interval = clamp(interval, 1UL, max_load_balance_interval);
10173 :
10174 : return interval;
10175 : }
10176 :
10177 : static inline void
10178 : update_next_balance(struct sched_domain *sd, unsigned long *next_balance)
10179 : {
10180 : unsigned long interval, next;
10181 :
10182 : /* used by idle balance, so cpu_busy = 0 */
10183 : interval = get_sd_balance_interval(sd, 0);
10184 : next = sd->last_balance + interval;
10185 :
10186 : if (time_after(*next_balance, next))
10187 : *next_balance = next;
10188 : }
10189 :
10190 : /*
10191 : * active_load_balance_cpu_stop is run by the CPU stopper. It pushes
10192 : * running tasks off the busiest CPU onto idle CPUs. It requires at
10193 : * least 1 task to be running on each physical CPU where possible, and
10194 : * avoids physical / logical imbalances.
10195 : */
10196 : static int active_load_balance_cpu_stop(void *data)
10197 : {
10198 : struct rq *busiest_rq = data;
10199 : int busiest_cpu = cpu_of(busiest_rq);
10200 : int target_cpu = busiest_rq->push_cpu;
10201 : struct rq *target_rq = cpu_rq(target_cpu);
10202 : struct sched_domain *sd;
10203 : struct task_struct *p = NULL;
10204 : struct rq_flags rf;
10205 :
10206 : rq_lock_irq(busiest_rq, &rf);
10207 : /*
10208 : * Between queueing the stop-work and running it is a hole in which
10209 : * CPUs can become inactive. We should not move tasks from or to
10210 : * inactive CPUs.
10211 : */
10212 : if (!cpu_active(busiest_cpu) || !cpu_active(target_cpu))
10213 : goto out_unlock;
10214 :
10215 : /* Make sure the requested CPU hasn't gone down in the meantime: */
10216 : if (unlikely(busiest_cpu != smp_processor_id() ||
10217 : !busiest_rq->active_balance))
10218 : goto out_unlock;
10219 :
10220 : /* Is there any task to move? */
10221 : if (busiest_rq->nr_running <= 1)
10222 : goto out_unlock;
10223 :
10224 : /*
10225 : * This condition is "impossible", if it occurs
10226 : * we need to fix it. Originally reported by
10227 : * Bjorn Helgaas on a 128-CPU setup.
10228 : */
10229 : BUG_ON(busiest_rq == target_rq);
10230 :
10231 : /* Search for an sd spanning us and the target CPU. */
10232 : rcu_read_lock();
10233 : for_each_domain(target_cpu, sd) {
10234 : if (cpumask_test_cpu(busiest_cpu, sched_domain_span(sd)))
10235 : break;
10236 : }
10237 :
10238 : if (likely(sd)) {
10239 : struct lb_env env = {
10240 : .sd = sd,
10241 : .dst_cpu = target_cpu,
10242 : .dst_rq = target_rq,
10243 : .src_cpu = busiest_rq->cpu,
10244 : .src_rq = busiest_rq,
10245 : .idle = CPU_IDLE,
10246 : .flags = LBF_ACTIVE_LB,
10247 : };
10248 :
10249 : schedstat_inc(sd->alb_count);
10250 : update_rq_clock(busiest_rq);
10251 :
10252 : p = detach_one_task(&env);
10253 : if (p) {
10254 : schedstat_inc(sd->alb_pushed);
10255 : /* Active balancing done, reset the failure counter. */
10256 : sd->nr_balance_failed = 0;
10257 : } else {
10258 : schedstat_inc(sd->alb_failed);
10259 : }
10260 : }
10261 : rcu_read_unlock();
10262 : out_unlock:
10263 : busiest_rq->active_balance = 0;
10264 : rq_unlock(busiest_rq, &rf);
10265 :
10266 : if (p)
10267 : attach_one_task(target_rq, p);
10268 :
10269 : local_irq_enable();
10270 :
10271 : return 0;
10272 : }
10273 :
10274 : static DEFINE_SPINLOCK(balancing);
10275 :
10276 : /*
10277 : * Scale the max load_balance interval with the number of CPUs in the system.
10278 : * This trades load-balance latency on larger machines for less cross talk.
10279 : */
10280 : void update_max_interval(void)
10281 : {
10282 : max_load_balance_interval = HZ*num_online_cpus()/10;
10283 : }
10284 :
10285 : static inline bool update_newidle_cost(struct sched_domain *sd, u64 cost)
10286 : {
10287 : if (cost > sd->max_newidle_lb_cost) {
10288 : /*
10289 : * Track max cost of a domain to make sure to not delay the
10290 : * next wakeup on the CPU.
10291 : */
10292 : sd->max_newidle_lb_cost = cost;
10293 : sd->last_decay_max_lb_cost = jiffies;
10294 : } else if (time_after(jiffies, sd->last_decay_max_lb_cost + HZ)) {
10295 : /*
10296 : * Decay the newidle max times by ~1% per second to ensure that
10297 : * it is not outdated and the current max cost is actually
10298 : * shorter.
10299 : */
10300 : sd->max_newidle_lb_cost = (sd->max_newidle_lb_cost * 253) / 256;
10301 : sd->last_decay_max_lb_cost = jiffies;
10302 :
10303 : return true;
10304 : }
10305 :
10306 : return false;
10307 : }
10308 :
10309 : /*
10310 : * It checks each scheduling domain to see if it is due to be balanced,
10311 : * and initiates a balancing operation if so.
10312 : *
10313 : * Balancing parameters are set up in init_sched_domains.
10314 : */
10315 : static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle)
10316 : {
10317 : int continue_balancing = 1;
10318 : int cpu = rq->cpu;
10319 : int busy = idle != CPU_IDLE && !sched_idle_cpu(cpu);
10320 : unsigned long interval;
10321 : struct sched_domain *sd;
10322 : /* Earliest time when we have to do rebalance again */
10323 : unsigned long next_balance = jiffies + 60*HZ;
10324 : int update_next_balance = 0;
10325 : int need_serialize, need_decay = 0;
10326 : u64 max_cost = 0;
10327 :
10328 : rcu_read_lock();
10329 : for_each_domain(cpu, sd) {
10330 : /*
10331 : * Decay the newidle max times here because this is a regular
10332 : * visit to all the domains.
10333 : */
10334 : need_decay = update_newidle_cost(sd, 0);
10335 : max_cost += sd->max_newidle_lb_cost;
10336 :
10337 : /*
10338 : * Stop the load balance at this level. There is another
10339 : * CPU in our sched group which is doing load balancing more
10340 : * actively.
10341 : */
10342 : if (!continue_balancing) {
10343 : if (need_decay)
10344 : continue;
10345 : break;
10346 : }
10347 :
10348 : interval = get_sd_balance_interval(sd, busy);
10349 :
10350 : need_serialize = sd->flags & SD_SERIALIZE;
10351 : if (need_serialize) {
10352 : if (!spin_trylock(&balancing))
10353 : goto out;
10354 : }
10355 :
10356 : if (time_after_eq(jiffies, sd->last_balance + interval)) {
10357 : if (load_balance(cpu, rq, sd, idle, &continue_balancing)) {
10358 : /*
10359 : * The LBF_DST_PINNED logic could have changed
10360 : * env->dst_cpu, so we can't know our idle
10361 : * state even if we migrated tasks. Update it.
10362 : */
10363 : idle = idle_cpu(cpu) ? CPU_IDLE : CPU_NOT_IDLE;
10364 : busy = idle != CPU_IDLE && !sched_idle_cpu(cpu);
10365 : }
10366 : sd->last_balance = jiffies;
10367 : interval = get_sd_balance_interval(sd, busy);
10368 : }
10369 : if (need_serialize)
10370 : spin_unlock(&balancing);
10371 : out:
10372 : if (time_after(next_balance, sd->last_balance + interval)) {
10373 : next_balance = sd->last_balance + interval;
10374 : update_next_balance = 1;
10375 : }
10376 : }
10377 : if (need_decay) {
10378 : /*
10379 : * Ensure the rq-wide value also decays but keep it at a
10380 : * reasonable floor to avoid funnies with rq->avg_idle.
10381 : */
10382 : rq->max_idle_balance_cost =
10383 : max((u64)sysctl_sched_migration_cost, max_cost);
10384 : }
10385 : rcu_read_unlock();
10386 :
10387 : /*
10388 : * next_balance will be updated only when there is a need.
10389 : * When the cpu is attached to null domain for ex, it will not be
10390 : * updated.
10391 : */
10392 : if (likely(update_next_balance))
10393 : rq->next_balance = next_balance;
10394 :
10395 : }
10396 :
10397 : static inline int on_null_domain(struct rq *rq)
10398 : {
10399 : return unlikely(!rcu_dereference_sched(rq->sd));
10400 : }
10401 :
10402 : #ifdef CONFIG_NO_HZ_COMMON
10403 : /*
10404 : * idle load balancing details
10405 : * - When one of the busy CPUs notice that there may be an idle rebalancing
10406 : * needed, they will kick the idle load balancer, which then does idle
10407 : * load balancing for all the idle CPUs.
10408 : * - HK_TYPE_MISC CPUs are used for this task, because HK_TYPE_SCHED not set
10409 : * anywhere yet.
10410 : */
10411 :
10412 : static inline int find_new_ilb(void)
10413 : {
10414 : int ilb;
10415 : const struct cpumask *hk_mask;
10416 :
10417 : hk_mask = housekeeping_cpumask(HK_TYPE_MISC);
10418 :
10419 : for_each_cpu_and(ilb, nohz.idle_cpus_mask, hk_mask) {
10420 :
10421 : if (ilb == smp_processor_id())
10422 : continue;
10423 :
10424 : if (idle_cpu(ilb))
10425 : return ilb;
10426 : }
10427 :
10428 : return nr_cpu_ids;
10429 : }
10430 :
10431 : /*
10432 : * Kick a CPU to do the nohz balancing, if it is time for it. We pick any
10433 : * idle CPU in the HK_TYPE_MISC housekeeping set (if there is one).
10434 : */
10435 : static void kick_ilb(unsigned int flags)
10436 : {
10437 : int ilb_cpu;
10438 :
10439 : /*
10440 : * Increase nohz.next_balance only when if full ilb is triggered but
10441 : * not if we only update stats.
10442 : */
10443 : if (flags & NOHZ_BALANCE_KICK)
10444 : nohz.next_balance = jiffies+1;
10445 :
10446 : ilb_cpu = find_new_ilb();
10447 :
10448 : if (ilb_cpu >= nr_cpu_ids)
10449 : return;
10450 :
10451 : /*
10452 : * Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets
10453 : * the first flag owns it; cleared by nohz_csd_func().
10454 : */
10455 : flags = atomic_fetch_or(flags, nohz_flags(ilb_cpu));
10456 : if (flags & NOHZ_KICK_MASK)
10457 : return;
10458 :
10459 : /*
10460 : * This way we generate an IPI on the target CPU which
10461 : * is idle. And the softirq performing nohz idle load balance
10462 : * will be run before returning from the IPI.
10463 : */
10464 : smp_call_function_single_async(ilb_cpu, &cpu_rq(ilb_cpu)->nohz_csd);
10465 : }
10466 :
10467 : /*
10468 : * Current decision point for kicking the idle load balancer in the presence
10469 : * of idle CPUs in the system.
10470 : */
10471 : static void nohz_balancer_kick(struct rq *rq)
10472 : {
10473 : unsigned long now = jiffies;
10474 : struct sched_domain_shared *sds;
10475 : struct sched_domain *sd;
10476 : int nr_busy, i, cpu = rq->cpu;
10477 : unsigned int flags = 0;
10478 :
10479 : if (unlikely(rq->idle_balance))
10480 : return;
10481 :
10482 : /*
10483 : * We may be recently in ticked or tickless idle mode. At the first
10484 : * busy tick after returning from idle, we will update the busy stats.
10485 : */
10486 : nohz_balance_exit_idle(rq);
10487 :
10488 : /*
10489 : * None are in tickless mode and hence no need for NOHZ idle load
10490 : * balancing.
10491 : */
10492 : if (likely(!atomic_read(&nohz.nr_cpus)))
10493 : return;
10494 :
10495 : if (READ_ONCE(nohz.has_blocked) &&
10496 : time_after(now, READ_ONCE(nohz.next_blocked)))
10497 : flags = NOHZ_STATS_KICK;
10498 :
10499 : if (time_before(now, nohz.next_balance))
10500 : goto out;
10501 :
10502 : if (rq->nr_running >= 2) {
10503 : flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
10504 : goto out;
10505 : }
10506 :
10507 : rcu_read_lock();
10508 :
10509 : sd = rcu_dereference(rq->sd);
10510 : if (sd) {
10511 : /*
10512 : * If there's a CFS task and the current CPU has reduced
10513 : * capacity; kick the ILB to see if there's a better CPU to run
10514 : * on.
10515 : */
10516 : if (rq->cfs.h_nr_running >= 1 && check_cpu_capacity(rq, sd)) {
10517 : flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
10518 : goto unlock;
10519 : }
10520 : }
10521 :
10522 : sd = rcu_dereference(per_cpu(sd_asym_packing, cpu));
10523 : if (sd) {
10524 : /*
10525 : * When ASYM_PACKING; see if there's a more preferred CPU
10526 : * currently idle; in which case, kick the ILB to move tasks
10527 : * around.
10528 : */
10529 : for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) {
10530 : if (sched_asym_prefer(i, cpu)) {
10531 : flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
10532 : goto unlock;
10533 : }
10534 : }
10535 : }
10536 :
10537 : sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, cpu));
10538 : if (sd) {
10539 : /*
10540 : * When ASYM_CPUCAPACITY; see if there's a higher capacity CPU
10541 : * to run the misfit task on.
10542 : */
10543 : if (check_misfit_status(rq, sd)) {
10544 : flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
10545 : goto unlock;
10546 : }
10547 :
10548 : /*
10549 : * For asymmetric systems, we do not want to nicely balance
10550 : * cache use, instead we want to embrace asymmetry and only
10551 : * ensure tasks have enough CPU capacity.
10552 : *
10553 : * Skip the LLC logic because it's not relevant in that case.
10554 : */
10555 : goto unlock;
10556 : }
10557 :
10558 : sds = rcu_dereference(per_cpu(sd_llc_shared, cpu));
10559 : if (sds) {
10560 : /*
10561 : * If there is an imbalance between LLC domains (IOW we could
10562 : * increase the overall cache use), we need some less-loaded LLC
10563 : * domain to pull some load. Likewise, we may need to spread
10564 : * load within the current LLC domain (e.g. packed SMT cores but
10565 : * other CPUs are idle). We can't really know from here how busy
10566 : * the others are - so just get a nohz balance going if it looks
10567 : * like this LLC domain has tasks we could move.
10568 : */
10569 : nr_busy = atomic_read(&sds->nr_busy_cpus);
10570 : if (nr_busy > 1) {
10571 : flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
10572 : goto unlock;
10573 : }
10574 : }
10575 : unlock:
10576 : rcu_read_unlock();
10577 : out:
10578 : if (READ_ONCE(nohz.needs_update))
10579 : flags |= NOHZ_NEXT_KICK;
10580 :
10581 : if (flags)
10582 : kick_ilb(flags);
10583 : }
10584 :
10585 : static void set_cpu_sd_state_busy(int cpu)
10586 : {
10587 : struct sched_domain *sd;
10588 :
10589 : rcu_read_lock();
10590 : sd = rcu_dereference(per_cpu(sd_llc, cpu));
10591 :
10592 : if (!sd || !sd->nohz_idle)
10593 : goto unlock;
10594 : sd->nohz_idle = 0;
10595 :
10596 : atomic_inc(&sd->shared->nr_busy_cpus);
10597 : unlock:
10598 : rcu_read_unlock();
10599 : }
10600 :
10601 : void nohz_balance_exit_idle(struct rq *rq)
10602 : {
10603 : SCHED_WARN_ON(rq != this_rq());
10604 :
10605 : if (likely(!rq->nohz_tick_stopped))
10606 : return;
10607 :
10608 : rq->nohz_tick_stopped = 0;
10609 : cpumask_clear_cpu(rq->cpu, nohz.idle_cpus_mask);
10610 : atomic_dec(&nohz.nr_cpus);
10611 :
10612 : set_cpu_sd_state_busy(rq->cpu);
10613 : }
10614 :
10615 : static void set_cpu_sd_state_idle(int cpu)
10616 : {
10617 : struct sched_domain *sd;
10618 :
10619 : rcu_read_lock();
10620 : sd = rcu_dereference(per_cpu(sd_llc, cpu));
10621 :
10622 : if (!sd || sd->nohz_idle)
10623 : goto unlock;
10624 : sd->nohz_idle = 1;
10625 :
10626 : atomic_dec(&sd->shared->nr_busy_cpus);
10627 : unlock:
10628 : rcu_read_unlock();
10629 : }
10630 :
10631 : /*
10632 : * This routine will record that the CPU is going idle with tick stopped.
10633 : * This info will be used in performing idle load balancing in the future.
10634 : */
10635 : void nohz_balance_enter_idle(int cpu)
10636 : {
10637 : struct rq *rq = cpu_rq(cpu);
10638 :
10639 : SCHED_WARN_ON(cpu != smp_processor_id());
10640 :
10641 : /* If this CPU is going down, then nothing needs to be done: */
10642 : if (!cpu_active(cpu))
10643 : return;
10644 :
10645 : /* Spare idle load balancing on CPUs that don't want to be disturbed: */
10646 : if (!housekeeping_cpu(cpu, HK_TYPE_SCHED))
10647 : return;
10648 :
10649 : /*
10650 : * Can be set safely without rq->lock held
10651 : * If a clear happens, it will have evaluated last additions because
10652 : * rq->lock is held during the check and the clear
10653 : */
10654 : rq->has_blocked_load = 1;
10655 :
10656 : /*
10657 : * The tick is still stopped but load could have been added in the
10658 : * meantime. We set the nohz.has_blocked flag to trig a check of the
10659 : * *_avg. The CPU is already part of nohz.idle_cpus_mask so the clear
10660 : * of nohz.has_blocked can only happen after checking the new load
10661 : */
10662 : if (rq->nohz_tick_stopped)
10663 : goto out;
10664 :
10665 : /* If we're a completely isolated CPU, we don't play: */
10666 : if (on_null_domain(rq))
10667 : return;
10668 :
10669 : rq->nohz_tick_stopped = 1;
10670 :
10671 : cpumask_set_cpu(cpu, nohz.idle_cpus_mask);
10672 : atomic_inc(&nohz.nr_cpus);
10673 :
10674 : /*
10675 : * Ensures that if nohz_idle_balance() fails to observe our
10676 : * @idle_cpus_mask store, it must observe the @has_blocked
10677 : * and @needs_update stores.
10678 : */
10679 : smp_mb__after_atomic();
10680 :
10681 : set_cpu_sd_state_idle(cpu);
10682 :
10683 : WRITE_ONCE(nohz.needs_update, 1);
10684 : out:
10685 : /*
10686 : * Each time a cpu enter idle, we assume that it has blocked load and
10687 : * enable the periodic update of the load of idle cpus
10688 : */
10689 : WRITE_ONCE(nohz.has_blocked, 1);
10690 : }
10691 :
10692 : static bool update_nohz_stats(struct rq *rq)
10693 : {
10694 : unsigned int cpu = rq->cpu;
10695 :
10696 : if (!rq->has_blocked_load)
10697 : return false;
10698 :
10699 : if (!cpumask_test_cpu(cpu, nohz.idle_cpus_mask))
10700 : return false;
10701 :
10702 : if (!time_after(jiffies, READ_ONCE(rq->last_blocked_load_update_tick)))
10703 : return true;
10704 :
10705 : update_blocked_averages(cpu);
10706 :
10707 : return rq->has_blocked_load;
10708 : }
10709 :
10710 : /*
10711 : * Internal function that runs load balance for all idle cpus. The load balance
10712 : * can be a simple update of blocked load or a complete load balance with
10713 : * tasks movement depending of flags.
10714 : */
10715 : static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags,
10716 : enum cpu_idle_type idle)
10717 : {
10718 : /* Earliest time when we have to do rebalance again */
10719 : unsigned long now = jiffies;
10720 : unsigned long next_balance = now + 60*HZ;
10721 : bool has_blocked_load = false;
10722 : int update_next_balance = 0;
10723 : int this_cpu = this_rq->cpu;
10724 : int balance_cpu;
10725 : struct rq *rq;
10726 :
10727 : SCHED_WARN_ON((flags & NOHZ_KICK_MASK) == NOHZ_BALANCE_KICK);
10728 :
10729 : /*
10730 : * We assume there will be no idle load after this update and clear
10731 : * the has_blocked flag. If a cpu enters idle in the mean time, it will
10732 : * set the has_blocked flag and trigger another update of idle load.
10733 : * Because a cpu that becomes idle, is added to idle_cpus_mask before
10734 : * setting the flag, we are sure to not clear the state and not
10735 : * check the load of an idle cpu.
10736 : *
10737 : * Same applies to idle_cpus_mask vs needs_update.
10738 : */
10739 : if (flags & NOHZ_STATS_KICK)
10740 : WRITE_ONCE(nohz.has_blocked, 0);
10741 : if (flags & NOHZ_NEXT_KICK)
10742 : WRITE_ONCE(nohz.needs_update, 0);
10743 :
10744 : /*
10745 : * Ensures that if we miss the CPU, we must see the has_blocked
10746 : * store from nohz_balance_enter_idle().
10747 : */
10748 : smp_mb();
10749 :
10750 : /*
10751 : * Start with the next CPU after this_cpu so we will end with this_cpu and let a
10752 : * chance for other idle cpu to pull load.
10753 : */
10754 : for_each_cpu_wrap(balance_cpu, nohz.idle_cpus_mask, this_cpu+1) {
10755 : if (!idle_cpu(balance_cpu))
10756 : continue;
10757 :
10758 : /*
10759 : * If this CPU gets work to do, stop the load balancing
10760 : * work being done for other CPUs. Next load
10761 : * balancing owner will pick it up.
10762 : */
10763 : if (need_resched()) {
10764 : if (flags & NOHZ_STATS_KICK)
10765 : has_blocked_load = true;
10766 : if (flags & NOHZ_NEXT_KICK)
10767 : WRITE_ONCE(nohz.needs_update, 1);
10768 : goto abort;
10769 : }
10770 :
10771 : rq = cpu_rq(balance_cpu);
10772 :
10773 : if (flags & NOHZ_STATS_KICK)
10774 : has_blocked_load |= update_nohz_stats(rq);
10775 :
10776 : /*
10777 : * If time for next balance is due,
10778 : * do the balance.
10779 : */
10780 : if (time_after_eq(jiffies, rq->next_balance)) {
10781 : struct rq_flags rf;
10782 :
10783 : rq_lock_irqsave(rq, &rf);
10784 : update_rq_clock(rq);
10785 : rq_unlock_irqrestore(rq, &rf);
10786 :
10787 : if (flags & NOHZ_BALANCE_KICK)
10788 : rebalance_domains(rq, CPU_IDLE);
10789 : }
10790 :
10791 : if (time_after(next_balance, rq->next_balance)) {
10792 : next_balance = rq->next_balance;
10793 : update_next_balance = 1;
10794 : }
10795 : }
10796 :
10797 : /*
10798 : * next_balance will be updated only when there is a need.
10799 : * When the CPU is attached to null domain for ex, it will not be
10800 : * updated.
10801 : */
10802 : if (likely(update_next_balance))
10803 : nohz.next_balance = next_balance;
10804 :
10805 : if (flags & NOHZ_STATS_KICK)
10806 : WRITE_ONCE(nohz.next_blocked,
10807 : now + msecs_to_jiffies(LOAD_AVG_PERIOD));
10808 :
10809 : abort:
10810 : /* There is still blocked load, enable periodic update */
10811 : if (has_blocked_load)
10812 : WRITE_ONCE(nohz.has_blocked, 1);
10813 : }
10814 :
10815 : /*
10816 : * In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the
10817 : * rebalancing for all the cpus for whom scheduler ticks are stopped.
10818 : */
10819 : static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
10820 : {
10821 : unsigned int flags = this_rq->nohz_idle_balance;
10822 :
10823 : if (!flags)
10824 : return false;
10825 :
10826 : this_rq->nohz_idle_balance = 0;
10827 :
10828 : if (idle != CPU_IDLE)
10829 : return false;
10830 :
10831 : _nohz_idle_balance(this_rq, flags, idle);
10832 :
10833 : return true;
10834 : }
10835 :
10836 : /*
10837 : * Check if we need to run the ILB for updating blocked load before entering
10838 : * idle state.
10839 : */
10840 : void nohz_run_idle_balance(int cpu)
10841 : {
10842 : unsigned int flags;
10843 :
10844 : flags = atomic_fetch_andnot(NOHZ_NEWILB_KICK, nohz_flags(cpu));
10845 :
10846 : /*
10847 : * Update the blocked load only if no SCHED_SOFTIRQ is about to happen
10848 : * (ie NOHZ_STATS_KICK set) and will do the same.
10849 : */
10850 : if ((flags == NOHZ_NEWILB_KICK) && !need_resched())
10851 : _nohz_idle_balance(cpu_rq(cpu), NOHZ_STATS_KICK, CPU_IDLE);
10852 : }
10853 :
10854 : static void nohz_newidle_balance(struct rq *this_rq)
10855 : {
10856 : int this_cpu = this_rq->cpu;
10857 :
10858 : /*
10859 : * This CPU doesn't want to be disturbed by scheduler
10860 : * housekeeping
10861 : */
10862 : if (!housekeeping_cpu(this_cpu, HK_TYPE_SCHED))
10863 : return;
10864 :
10865 : /* Will wake up very soon. No time for doing anything else*/
10866 : if (this_rq->avg_idle < sysctl_sched_migration_cost)
10867 : return;
10868 :
10869 : /* Don't need to update blocked load of idle CPUs*/
10870 : if (!READ_ONCE(nohz.has_blocked) ||
10871 : time_before(jiffies, READ_ONCE(nohz.next_blocked)))
10872 : return;
10873 :
10874 : /*
10875 : * Set the need to trigger ILB in order to update blocked load
10876 : * before entering idle state.
10877 : */
10878 : atomic_or(NOHZ_NEWILB_KICK, nohz_flags(this_cpu));
10879 : }
10880 :
10881 : #else /* !CONFIG_NO_HZ_COMMON */
10882 : static inline void nohz_balancer_kick(struct rq *rq) { }
10883 :
10884 : static inline bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
10885 : {
10886 : return false;
10887 : }
10888 :
10889 : static inline void nohz_newidle_balance(struct rq *this_rq) { }
10890 : #endif /* CONFIG_NO_HZ_COMMON */
10891 :
10892 : /*
10893 : * newidle_balance is called by schedule() if this_cpu is about to become
10894 : * idle. Attempts to pull tasks from other CPUs.
10895 : *
10896 : * Returns:
10897 : * < 0 - we released the lock and there are !fair tasks present
10898 : * 0 - failed, no new tasks
10899 : * > 0 - success, new (fair) tasks present
10900 : */
10901 : static int newidle_balance(struct rq *this_rq, struct rq_flags *rf)
10902 : {
10903 : unsigned long next_balance = jiffies + HZ;
10904 : int this_cpu = this_rq->cpu;
10905 : u64 t0, t1, curr_cost = 0;
10906 : struct sched_domain *sd;
10907 : int pulled_task = 0;
10908 :
10909 : update_misfit_status(NULL, this_rq);
10910 :
10911 : /*
10912 : * There is a task waiting to run. No need to search for one.
10913 : * Return 0; the task will be enqueued when switching to idle.
10914 : */
10915 : if (this_rq->ttwu_pending)
10916 : return 0;
10917 :
10918 : /*
10919 : * We must set idle_stamp _before_ calling idle_balance(), such that we
10920 : * measure the duration of idle_balance() as idle time.
10921 : */
10922 : this_rq->idle_stamp = rq_clock(this_rq);
10923 :
10924 : /*
10925 : * Do not pull tasks towards !active CPUs...
10926 : */
10927 : if (!cpu_active(this_cpu))
10928 : return 0;
10929 :
10930 : /*
10931 : * This is OK, because current is on_cpu, which avoids it being picked
10932 : * for load-balance and preemption/IRQs are still disabled avoiding
10933 : * further scheduler activity on it and we're being very careful to
10934 : * re-start the picking loop.
10935 : */
10936 : rq_unpin_lock(this_rq, rf);
10937 :
10938 : rcu_read_lock();
10939 : sd = rcu_dereference_check_sched_domain(this_rq->sd);
10940 :
10941 : if (!READ_ONCE(this_rq->rd->overload) ||
10942 : (sd && this_rq->avg_idle < sd->max_newidle_lb_cost)) {
10943 :
10944 : if (sd)
10945 : update_next_balance(sd, &next_balance);
10946 : rcu_read_unlock();
10947 :
10948 : goto out;
10949 : }
10950 : rcu_read_unlock();
10951 :
10952 : raw_spin_rq_unlock(this_rq);
10953 :
10954 : t0 = sched_clock_cpu(this_cpu);
10955 : update_blocked_averages(this_cpu);
10956 :
10957 : rcu_read_lock();
10958 : for_each_domain(this_cpu, sd) {
10959 : int continue_balancing = 1;
10960 : u64 domain_cost;
10961 :
10962 : update_next_balance(sd, &next_balance);
10963 :
10964 : if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost)
10965 : break;
10966 :
10967 : if (sd->flags & SD_BALANCE_NEWIDLE) {
10968 :
10969 : pulled_task = load_balance(this_cpu, this_rq,
10970 : sd, CPU_NEWLY_IDLE,
10971 : &continue_balancing);
10972 :
10973 : t1 = sched_clock_cpu(this_cpu);
10974 : domain_cost = t1 - t0;
10975 : update_newidle_cost(sd, domain_cost);
10976 :
10977 : curr_cost += domain_cost;
10978 : t0 = t1;
10979 : }
10980 :
10981 : /*
10982 : * Stop searching for tasks to pull if there are
10983 : * now runnable tasks on this rq.
10984 : */
10985 : if (pulled_task || this_rq->nr_running > 0 ||
10986 : this_rq->ttwu_pending)
10987 : break;
10988 : }
10989 : rcu_read_unlock();
10990 :
10991 : raw_spin_rq_lock(this_rq);
10992 :
10993 : if (curr_cost > this_rq->max_idle_balance_cost)
10994 : this_rq->max_idle_balance_cost = curr_cost;
10995 :
10996 : /*
10997 : * While browsing the domains, we released the rq lock, a task could
10998 : * have been enqueued in the meantime. Since we're not going idle,
10999 : * pretend we pulled a task.
11000 : */
11001 : if (this_rq->cfs.h_nr_running && !pulled_task)
11002 : pulled_task = 1;
11003 :
11004 : /* Is there a task of a high priority class? */
11005 : if (this_rq->nr_running != this_rq->cfs.h_nr_running)
11006 : pulled_task = -1;
11007 :
11008 : out:
11009 : /* Move the next balance forward */
11010 : if (time_after(this_rq->next_balance, next_balance))
11011 : this_rq->next_balance = next_balance;
11012 :
11013 : if (pulled_task)
11014 : this_rq->idle_stamp = 0;
11015 : else
11016 : nohz_newidle_balance(this_rq);
11017 :
11018 : rq_repin_lock(this_rq, rf);
11019 :
11020 : return pulled_task;
11021 : }
11022 :
11023 : /*
11024 : * run_rebalance_domains is triggered when needed from the scheduler tick.
11025 : * Also triggered for nohz idle balancing (with nohz_balancing_kick set).
11026 : */
11027 : static __latent_entropy void run_rebalance_domains(struct softirq_action *h)
11028 : {
11029 : struct rq *this_rq = this_rq();
11030 : enum cpu_idle_type idle = this_rq->idle_balance ?
11031 : CPU_IDLE : CPU_NOT_IDLE;
11032 :
11033 : /*
11034 : * If this CPU has a pending nohz_balance_kick, then do the
11035 : * balancing on behalf of the other idle CPUs whose ticks are
11036 : * stopped. Do nohz_idle_balance *before* rebalance_domains to
11037 : * give the idle CPUs a chance to load balance. Else we may
11038 : * load balance only within the local sched_domain hierarchy
11039 : * and abort nohz_idle_balance altogether if we pull some load.
11040 : */
11041 : if (nohz_idle_balance(this_rq, idle))
11042 : return;
11043 :
11044 : /* normal load balance */
11045 : update_blocked_averages(this_rq->cpu);
11046 : rebalance_domains(this_rq, idle);
11047 : }
11048 :
11049 : /*
11050 : * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing.
11051 : */
11052 : void trigger_load_balance(struct rq *rq)
11053 : {
11054 : /*
11055 : * Don't need to rebalance while attached to NULL domain or
11056 : * runqueue CPU is not active
11057 : */
11058 : if (unlikely(on_null_domain(rq) || !cpu_active(cpu_of(rq))))
11059 : return;
11060 :
11061 : if (time_after_eq(jiffies, rq->next_balance))
11062 : raise_softirq(SCHED_SOFTIRQ);
11063 :
11064 : nohz_balancer_kick(rq);
11065 : }
11066 :
11067 : static void rq_online_fair(struct rq *rq)
11068 : {
11069 : update_sysctl();
11070 :
11071 : update_runtime_enabled(rq);
11072 : }
11073 :
11074 : static void rq_offline_fair(struct rq *rq)
11075 : {
11076 : update_sysctl();
11077 :
11078 : /* Ensure any throttled groups are reachable by pick_next_task */
11079 : unthrottle_offline_cfs_rqs(rq);
11080 : }
11081 :
11082 : #endif /* CONFIG_SMP */
11083 :
11084 : #ifdef CONFIG_SCHED_CORE
11085 : static inline bool
11086 : __entity_slice_used(struct sched_entity *se, int min_nr_tasks)
11087 : {
11088 : u64 slice = sched_slice(cfs_rq_of(se), se);
11089 : u64 rtime = se->sum_exec_runtime - se->prev_sum_exec_runtime;
11090 :
11091 : return (rtime * min_nr_tasks > slice);
11092 : }
11093 :
11094 : #define MIN_NR_TASKS_DURING_FORCEIDLE 2
11095 : static inline void task_tick_core(struct rq *rq, struct task_struct *curr)
11096 : {
11097 : if (!sched_core_enabled(rq))
11098 : return;
11099 :
11100 : /*
11101 : * If runqueue has only one task which used up its slice and
11102 : * if the sibling is forced idle, then trigger schedule to
11103 : * give forced idle task a chance.
11104 : *
11105 : * sched_slice() considers only this active rq and it gets the
11106 : * whole slice. But during force idle, we have siblings acting
11107 : * like a single runqueue and hence we need to consider runnable
11108 : * tasks on this CPU and the forced idle CPU. Ideally, we should
11109 : * go through the forced idle rq, but that would be a perf hit.
11110 : * We can assume that the forced idle CPU has at least
11111 : * MIN_NR_TASKS_DURING_FORCEIDLE - 1 tasks and use that to check
11112 : * if we need to give up the CPU.
11113 : */
11114 : if (rq->core->core_forceidle_count && rq->cfs.nr_running == 1 &&
11115 : __entity_slice_used(&curr->se, MIN_NR_TASKS_DURING_FORCEIDLE))
11116 : resched_curr(rq);
11117 : }
11118 :
11119 : /*
11120 : * se_fi_update - Update the cfs_rq->min_vruntime_fi in a CFS hierarchy if needed.
11121 : */
11122 : static void se_fi_update(struct sched_entity *se, unsigned int fi_seq, bool forceidle)
11123 : {
11124 : for_each_sched_entity(se) {
11125 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
11126 :
11127 : if (forceidle) {
11128 : if (cfs_rq->forceidle_seq == fi_seq)
11129 : break;
11130 : cfs_rq->forceidle_seq = fi_seq;
11131 : }
11132 :
11133 : cfs_rq->min_vruntime_fi = cfs_rq->min_vruntime;
11134 : }
11135 : }
11136 :
11137 : void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi)
11138 : {
11139 : struct sched_entity *se = &p->se;
11140 :
11141 : if (p->sched_class != &fair_sched_class)
11142 : return;
11143 :
11144 : se_fi_update(se, rq->core->core_forceidle_seq, in_fi);
11145 : }
11146 :
11147 : bool cfs_prio_less(struct task_struct *a, struct task_struct *b, bool in_fi)
11148 : {
11149 : struct rq *rq = task_rq(a);
11150 : struct sched_entity *sea = &a->se;
11151 : struct sched_entity *seb = &b->se;
11152 : struct cfs_rq *cfs_rqa;
11153 : struct cfs_rq *cfs_rqb;
11154 : s64 delta;
11155 :
11156 : SCHED_WARN_ON(task_rq(b)->core != rq->core);
11157 :
11158 : #ifdef CONFIG_FAIR_GROUP_SCHED
11159 : /*
11160 : * Find an se in the hierarchy for tasks a and b, such that the se's
11161 : * are immediate siblings.
11162 : */
11163 : while (sea->cfs_rq->tg != seb->cfs_rq->tg) {
11164 : int sea_depth = sea->depth;
11165 : int seb_depth = seb->depth;
11166 :
11167 : if (sea_depth >= seb_depth)
11168 : sea = parent_entity(sea);
11169 : if (sea_depth <= seb_depth)
11170 : seb = parent_entity(seb);
11171 : }
11172 :
11173 : se_fi_update(sea, rq->core->core_forceidle_seq, in_fi);
11174 : se_fi_update(seb, rq->core->core_forceidle_seq, in_fi);
11175 :
11176 : cfs_rqa = sea->cfs_rq;
11177 : cfs_rqb = seb->cfs_rq;
11178 : #else
11179 : cfs_rqa = &task_rq(a)->cfs;
11180 : cfs_rqb = &task_rq(b)->cfs;
11181 : #endif
11182 :
11183 : /*
11184 : * Find delta after normalizing se's vruntime with its cfs_rq's
11185 : * min_vruntime_fi, which would have been updated in prior calls
11186 : * to se_fi_update().
11187 : */
11188 : delta = (s64)(sea->vruntime - seb->vruntime) +
11189 : (s64)(cfs_rqb->min_vruntime_fi - cfs_rqa->min_vruntime_fi);
11190 :
11191 : return delta > 0;
11192 : }
11193 : #else
11194 : static inline void task_tick_core(struct rq *rq, struct task_struct *curr) {}
11195 : #endif
11196 :
11197 : /*
11198 : * scheduler tick hitting a task of our scheduling class.
11199 : *
11200 : * NOTE: This function can be called remotely by the tick offload that
11201 : * goes along full dynticks. Therefore no local assumption can be made
11202 : * and everything must be accessed through the @rq and @curr passed in
11203 : * parameters.
11204 : */
11205 3 : static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued)
11206 : {
11207 : struct cfs_rq *cfs_rq;
11208 3 : struct sched_entity *se = &curr->se;
11209 :
11210 6 : for_each_sched_entity(se) {
11211 6 : cfs_rq = cfs_rq_of(se);
11212 3 : entity_tick(cfs_rq, se, queued);
11213 : }
11214 :
11215 3 : if (static_branch_unlikely(&sched_numa_balancing))
11216 : task_tick_numa(rq, curr);
11217 :
11218 3 : update_misfit_status(curr, rq);
11219 3 : update_overutilized_status(task_rq(curr));
11220 :
11221 3 : task_tick_core(rq, curr);
11222 3 : }
11223 :
11224 : /*
11225 : * called on fork with the child task as argument from the parent's context
11226 : * - child not yet on the tasklist
11227 : * - preemption disabled
11228 : */
11229 107 : static void task_fork_fair(struct task_struct *p)
11230 : {
11231 : struct cfs_rq *cfs_rq;
11232 107 : struct sched_entity *se = &p->se, *curr;
11233 107 : struct rq *rq = this_rq();
11234 : struct rq_flags rf;
11235 :
11236 107 : rq_lock(rq, &rf);
11237 107 : update_rq_clock(rq);
11238 :
11239 214 : cfs_rq = task_cfs_rq(current);
11240 107 : curr = cfs_rq->curr;
11241 107 : if (curr) {
11242 105 : update_curr(cfs_rq);
11243 105 : se->vruntime = curr->vruntime;
11244 : }
11245 107 : place_entity(cfs_rq, se, 1);
11246 :
11247 107 : if (sysctl_sched_child_runs_first && curr && entity_before(curr, se)) {
11248 : /*
11249 : * Upon rescheduling, sched_class::put_prev_task() will place
11250 : * 'current' within the tree based on its new key value.
11251 : */
11252 0 : swap(curr->vruntime, se->vruntime);
11253 0 : resched_curr(rq);
11254 : }
11255 :
11256 107 : se->vruntime -= cfs_rq->min_vruntime;
11257 107 : rq_unlock(rq, &rf);
11258 107 : }
11259 :
11260 : /*
11261 : * Priority of the task has changed. Check to see if we preempt
11262 : * the current task.
11263 : */
11264 : static void
11265 4 : prio_changed_fair(struct rq *rq, struct task_struct *p, int oldprio)
11266 : {
11267 4 : if (!task_on_rq_queued(p))
11268 : return;
11269 :
11270 3 : if (rq->cfs.nr_running == 1)
11271 : return;
11272 :
11273 : /*
11274 : * Reschedule if we are currently running on this runqueue and
11275 : * our priority decreased, or if we are not currently running on
11276 : * this runqueue and our priority is higher than the current's
11277 : */
11278 3 : if (task_current(rq, p)) {
11279 3 : if (p->prio > oldprio)
11280 0 : resched_curr(rq);
11281 : } else
11282 0 : check_preempt_curr(rq, p, 0);
11283 : }
11284 :
11285 : static inline bool vruntime_normalized(struct task_struct *p)
11286 : {
11287 0 : struct sched_entity *se = &p->se;
11288 :
11289 : /*
11290 : * In both the TASK_ON_RQ_QUEUED and TASK_ON_RQ_MIGRATING cases,
11291 : * the dequeue_entity(.flags=0) will already have normalized the
11292 : * vruntime.
11293 : */
11294 0 : if (p->on_rq)
11295 : return true;
11296 :
11297 : /*
11298 : * When !on_rq, vruntime of the task has usually NOT been normalized.
11299 : * But there are some cases where it has already been normalized:
11300 : *
11301 : * - A forked child which is waiting for being woken up by
11302 : * wake_up_new_task().
11303 : * - A task which has been woken up by try_to_wake_up() and
11304 : * waiting for actually being woken up by sched_ttwu_pending().
11305 : */
11306 0 : if (!se->sum_exec_runtime ||
11307 0 : (READ_ONCE(p->__state) == TASK_WAKING && p->sched_remote_wakeup))
11308 : return true;
11309 :
11310 : return false;
11311 : }
11312 :
11313 : #ifdef CONFIG_FAIR_GROUP_SCHED
11314 : /*
11315 : * Propagate the changes of the sched_entity across the tg tree to make it
11316 : * visible to the root
11317 : */
11318 : static void propagate_entity_cfs_rq(struct sched_entity *se)
11319 : {
11320 : struct cfs_rq *cfs_rq;
11321 :
11322 : list_add_leaf_cfs_rq(cfs_rq_of(se));
11323 :
11324 : /* Start to propagate at parent */
11325 : se = se->parent;
11326 :
11327 : for_each_sched_entity(se) {
11328 : cfs_rq = cfs_rq_of(se);
11329 :
11330 : if (!cfs_rq_throttled(cfs_rq)){
11331 : update_load_avg(cfs_rq, se, UPDATE_TG);
11332 : list_add_leaf_cfs_rq(cfs_rq);
11333 : continue;
11334 : }
11335 :
11336 : if (list_add_leaf_cfs_rq(cfs_rq))
11337 : break;
11338 : }
11339 : }
11340 : #else
11341 : static void propagate_entity_cfs_rq(struct sched_entity *se) { }
11342 : #endif
11343 :
11344 : static void detach_entity_cfs_rq(struct sched_entity *se)
11345 : {
11346 0 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
11347 :
11348 : /* Catch up with the cfs_rq and remove our load when we leave */
11349 0 : update_load_avg(cfs_rq, se, 0);
11350 0 : detach_entity_load_avg(cfs_rq, se);
11351 : update_tg_load_avg(cfs_rq);
11352 0 : propagate_entity_cfs_rq(se);
11353 : }
11354 :
11355 : static void attach_entity_cfs_rq(struct sched_entity *se)
11356 : {
11357 0 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
11358 :
11359 : #ifdef CONFIG_FAIR_GROUP_SCHED
11360 : /*
11361 : * Since the real-depth could have been changed (only FAIR
11362 : * class maintain depth value), reset depth properly.
11363 : */
11364 : se->depth = se->parent ? se->parent->depth + 1 : 0;
11365 : #endif
11366 :
11367 : /* Synchronize entity with its cfs_rq */
11368 0 : update_load_avg(cfs_rq, se, sched_feat(ATTACH_AGE_LOAD) ? 0 : SKIP_AGE_LOAD);
11369 0 : attach_entity_load_avg(cfs_rq, se);
11370 : update_tg_load_avg(cfs_rq);
11371 0 : propagate_entity_cfs_rq(se);
11372 : }
11373 :
11374 0 : static void detach_task_cfs_rq(struct task_struct *p)
11375 : {
11376 0 : struct sched_entity *se = &p->se;
11377 0 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
11378 :
11379 0 : if (!vruntime_normalized(p)) {
11380 : /*
11381 : * Fix up our vruntime so that the current sleep doesn't
11382 : * cause 'unlimited' sleep bonus.
11383 : */
11384 0 : place_entity(cfs_rq, se, 0);
11385 0 : se->vruntime -= cfs_rq->min_vruntime;
11386 : }
11387 :
11388 0 : detach_entity_cfs_rq(se);
11389 0 : }
11390 :
11391 : static void attach_task_cfs_rq(struct task_struct *p)
11392 : {
11393 0 : struct sched_entity *se = &p->se;
11394 0 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
11395 :
11396 0 : attach_entity_cfs_rq(se);
11397 :
11398 0 : if (!vruntime_normalized(p))
11399 0 : se->vruntime += cfs_rq->min_vruntime;
11400 : }
11401 :
11402 0 : static void switched_from_fair(struct rq *rq, struct task_struct *p)
11403 : {
11404 0 : detach_task_cfs_rq(p);
11405 0 : }
11406 :
11407 0 : static void switched_to_fair(struct rq *rq, struct task_struct *p)
11408 : {
11409 0 : attach_task_cfs_rq(p);
11410 :
11411 0 : if (task_on_rq_queued(p)) {
11412 : /*
11413 : * We were most likely switched from sched_rt, so
11414 : * kick off the schedule if running, otherwise just see
11415 : * if we can still preempt the current task.
11416 : */
11417 0 : if (task_current(rq, p))
11418 0 : resched_curr(rq);
11419 : else
11420 0 : check_preempt_curr(rq, p, 0);
11421 : }
11422 0 : }
11423 :
11424 : /* Account for a task changing its policy or group.
11425 : *
11426 : * This routine is mostly called to set cfs_rq->curr field when a task
11427 : * migrates between groups/classes.
11428 : */
11429 3 : static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first)
11430 : {
11431 3 : struct sched_entity *se = &p->se;
11432 :
11433 : #ifdef CONFIG_SMP
11434 : if (task_on_rq_queued(p)) {
11435 : /*
11436 : * Move the next running task to the front of the list, so our
11437 : * cfs_tasks list becomes MRU one.
11438 : */
11439 : list_move(&se->group_node, &rq->cfs_tasks);
11440 : }
11441 : #endif
11442 :
11443 6 : for_each_sched_entity(se) {
11444 6 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
11445 :
11446 3 : set_next_entity(cfs_rq, se);
11447 : /* ensure bandwidth has been allocated on our new cfs_rq */
11448 3 : account_cfs_rq_runtime(cfs_rq, 0);
11449 : }
11450 3 : }
11451 :
11452 1 : void init_cfs_rq(struct cfs_rq *cfs_rq)
11453 : {
11454 1 : cfs_rq->tasks_timeline = RB_ROOT_CACHED;
11455 1 : cfs_rq->min_vruntime = (u64)(-(1LL << 20));
11456 : #ifndef CONFIG_64BIT
11457 : cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime;
11458 : #endif
11459 : #ifdef CONFIG_SMP
11460 : raw_spin_lock_init(&cfs_rq->removed.lock);
11461 : #endif
11462 1 : }
11463 :
11464 : #ifdef CONFIG_FAIR_GROUP_SCHED
11465 : static void task_set_group_fair(struct task_struct *p)
11466 : {
11467 : struct sched_entity *se = &p->se;
11468 :
11469 : set_task_rq(p, task_cpu(p));
11470 : se->depth = se->parent ? se->parent->depth + 1 : 0;
11471 : }
11472 :
11473 : static void task_move_group_fair(struct task_struct *p)
11474 : {
11475 : detach_task_cfs_rq(p);
11476 : set_task_rq(p, task_cpu(p));
11477 :
11478 : #ifdef CONFIG_SMP
11479 : /* Tell se's cfs_rq has been changed -- migrated */
11480 : p->se.avg.last_update_time = 0;
11481 : #endif
11482 : attach_task_cfs_rq(p);
11483 : }
11484 :
11485 : static void task_change_group_fair(struct task_struct *p, int type)
11486 : {
11487 : switch (type) {
11488 : case TASK_SET_GROUP:
11489 : task_set_group_fair(p);
11490 : break;
11491 :
11492 : case TASK_MOVE_GROUP:
11493 : task_move_group_fair(p);
11494 : break;
11495 : }
11496 : }
11497 :
11498 : void free_fair_sched_group(struct task_group *tg)
11499 : {
11500 : int i;
11501 :
11502 : for_each_possible_cpu(i) {
11503 : if (tg->cfs_rq)
11504 : kfree(tg->cfs_rq[i]);
11505 : if (tg->se)
11506 : kfree(tg->se[i]);
11507 : }
11508 :
11509 : kfree(tg->cfs_rq);
11510 : kfree(tg->se);
11511 : }
11512 :
11513 : int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
11514 : {
11515 : struct sched_entity *se;
11516 : struct cfs_rq *cfs_rq;
11517 : int i;
11518 :
11519 : tg->cfs_rq = kcalloc(nr_cpu_ids, sizeof(cfs_rq), GFP_KERNEL);
11520 : if (!tg->cfs_rq)
11521 : goto err;
11522 : tg->se = kcalloc(nr_cpu_ids, sizeof(se), GFP_KERNEL);
11523 : if (!tg->se)
11524 : goto err;
11525 :
11526 : tg->shares = NICE_0_LOAD;
11527 :
11528 : init_cfs_bandwidth(tg_cfs_bandwidth(tg));
11529 :
11530 : for_each_possible_cpu(i) {
11531 : cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
11532 : GFP_KERNEL, cpu_to_node(i));
11533 : if (!cfs_rq)
11534 : goto err;
11535 :
11536 : se = kzalloc_node(sizeof(struct sched_entity_stats),
11537 : GFP_KERNEL, cpu_to_node(i));
11538 : if (!se)
11539 : goto err_free_rq;
11540 :
11541 : init_cfs_rq(cfs_rq);
11542 : init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]);
11543 : init_entity_runnable_average(se);
11544 : }
11545 :
11546 : return 1;
11547 :
11548 : err_free_rq:
11549 : kfree(cfs_rq);
11550 : err:
11551 : return 0;
11552 : }
11553 :
11554 : void online_fair_sched_group(struct task_group *tg)
11555 : {
11556 : struct sched_entity *se;
11557 : struct rq_flags rf;
11558 : struct rq *rq;
11559 : int i;
11560 :
11561 : for_each_possible_cpu(i) {
11562 : rq = cpu_rq(i);
11563 : se = tg->se[i];
11564 : rq_lock_irq(rq, &rf);
11565 : update_rq_clock(rq);
11566 : attach_entity_cfs_rq(se);
11567 : sync_throttle(tg, i);
11568 : rq_unlock_irq(rq, &rf);
11569 : }
11570 : }
11571 :
11572 : void unregister_fair_sched_group(struct task_group *tg)
11573 : {
11574 : unsigned long flags;
11575 : struct rq *rq;
11576 : int cpu;
11577 :
11578 : destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));
11579 :
11580 : for_each_possible_cpu(cpu) {
11581 : if (tg->se[cpu])
11582 : remove_entity_load_avg(tg->se[cpu]);
11583 :
11584 : /*
11585 : * Only empty task groups can be destroyed; so we can speculatively
11586 : * check on_list without danger of it being re-added.
11587 : */
11588 : if (!tg->cfs_rq[cpu]->on_list)
11589 : continue;
11590 :
11591 : rq = cpu_rq(cpu);
11592 :
11593 : raw_spin_rq_lock_irqsave(rq, flags);
11594 : list_del_leaf_cfs_rq(tg->cfs_rq[cpu]);
11595 : raw_spin_rq_unlock_irqrestore(rq, flags);
11596 : }
11597 : }
11598 :
11599 : void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
11600 : struct sched_entity *se, int cpu,
11601 : struct sched_entity *parent)
11602 : {
11603 : struct rq *rq = cpu_rq(cpu);
11604 :
11605 : cfs_rq->tg = tg;
11606 : cfs_rq->rq = rq;
11607 : init_cfs_rq_runtime(cfs_rq);
11608 :
11609 : tg->cfs_rq[cpu] = cfs_rq;
11610 : tg->se[cpu] = se;
11611 :
11612 : /* se could be NULL for root_task_group */
11613 : if (!se)
11614 : return;
11615 :
11616 : if (!parent) {
11617 : se->cfs_rq = &rq->cfs;
11618 : se->depth = 0;
11619 : } else {
11620 : se->cfs_rq = parent->my_q;
11621 : se->depth = parent->depth + 1;
11622 : }
11623 :
11624 : se->my_q = cfs_rq;
11625 : /* guarantee group entities always have weight */
11626 : update_load_set(&se->load, NICE_0_LOAD);
11627 : se->parent = parent;
11628 : }
11629 :
11630 : static DEFINE_MUTEX(shares_mutex);
11631 :
11632 : static int __sched_group_set_shares(struct task_group *tg, unsigned long shares)
11633 : {
11634 : int i;
11635 :
11636 : lockdep_assert_held(&shares_mutex);
11637 :
11638 : /*
11639 : * We can't change the weight of the root cgroup.
11640 : */
11641 : if (!tg->se[0])
11642 : return -EINVAL;
11643 :
11644 : shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES));
11645 :
11646 : if (tg->shares == shares)
11647 : return 0;
11648 :
11649 : tg->shares = shares;
11650 : for_each_possible_cpu(i) {
11651 : struct rq *rq = cpu_rq(i);
11652 : struct sched_entity *se = tg->se[i];
11653 : struct rq_flags rf;
11654 :
11655 : /* Propagate contribution to hierarchy */
11656 : rq_lock_irqsave(rq, &rf);
11657 : update_rq_clock(rq);
11658 : for_each_sched_entity(se) {
11659 : update_load_avg(cfs_rq_of(se), se, UPDATE_TG);
11660 : update_cfs_group(se);
11661 : }
11662 : rq_unlock_irqrestore(rq, &rf);
11663 : }
11664 :
11665 : return 0;
11666 : }
11667 :
11668 : int sched_group_set_shares(struct task_group *tg, unsigned long shares)
11669 : {
11670 : int ret;
11671 :
11672 : mutex_lock(&shares_mutex);
11673 : if (tg_is_idle(tg))
11674 : ret = -EINVAL;
11675 : else
11676 : ret = __sched_group_set_shares(tg, shares);
11677 : mutex_unlock(&shares_mutex);
11678 :
11679 : return ret;
11680 : }
11681 :
11682 : int sched_group_set_idle(struct task_group *tg, long idle)
11683 : {
11684 : int i;
11685 :
11686 : if (tg == &root_task_group)
11687 : return -EINVAL;
11688 :
11689 : if (idle < 0 || idle > 1)
11690 : return -EINVAL;
11691 :
11692 : mutex_lock(&shares_mutex);
11693 :
11694 : if (tg->idle == idle) {
11695 : mutex_unlock(&shares_mutex);
11696 : return 0;
11697 : }
11698 :
11699 : tg->idle = idle;
11700 :
11701 : for_each_possible_cpu(i) {
11702 : struct rq *rq = cpu_rq(i);
11703 : struct sched_entity *se = tg->se[i];
11704 : struct cfs_rq *parent_cfs_rq, *grp_cfs_rq = tg->cfs_rq[i];
11705 : bool was_idle = cfs_rq_is_idle(grp_cfs_rq);
11706 : long idle_task_delta;
11707 : struct rq_flags rf;
11708 :
11709 : rq_lock_irqsave(rq, &rf);
11710 :
11711 : grp_cfs_rq->idle = idle;
11712 : if (WARN_ON_ONCE(was_idle == cfs_rq_is_idle(grp_cfs_rq)))
11713 : goto next_cpu;
11714 :
11715 : if (se->on_rq) {
11716 : parent_cfs_rq = cfs_rq_of(se);
11717 : if (cfs_rq_is_idle(grp_cfs_rq))
11718 : parent_cfs_rq->idle_nr_running++;
11719 : else
11720 : parent_cfs_rq->idle_nr_running--;
11721 : }
11722 :
11723 : idle_task_delta = grp_cfs_rq->h_nr_running -
11724 : grp_cfs_rq->idle_h_nr_running;
11725 : if (!cfs_rq_is_idle(grp_cfs_rq))
11726 : idle_task_delta *= -1;
11727 :
11728 : for_each_sched_entity(se) {
11729 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
11730 :
11731 : if (!se->on_rq)
11732 : break;
11733 :
11734 : cfs_rq->idle_h_nr_running += idle_task_delta;
11735 :
11736 : /* Already accounted at parent level and above. */
11737 : if (cfs_rq_is_idle(cfs_rq))
11738 : break;
11739 : }
11740 :
11741 : next_cpu:
11742 : rq_unlock_irqrestore(rq, &rf);
11743 : }
11744 :
11745 : /* Idle groups have minimum weight. */
11746 : if (tg_is_idle(tg))
11747 : __sched_group_set_shares(tg, scale_load(WEIGHT_IDLEPRIO));
11748 : else
11749 : __sched_group_set_shares(tg, NICE_0_LOAD);
11750 :
11751 : mutex_unlock(&shares_mutex);
11752 : return 0;
11753 : }
11754 :
11755 : #else /* CONFIG_FAIR_GROUP_SCHED */
11756 :
11757 0 : void free_fair_sched_group(struct task_group *tg) { }
11758 :
11759 0 : int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
11760 : {
11761 0 : return 1;
11762 : }
11763 :
11764 0 : void online_fair_sched_group(struct task_group *tg) { }
11765 :
11766 0 : void unregister_fair_sched_group(struct task_group *tg) { }
11767 :
11768 : #endif /* CONFIG_FAIR_GROUP_SCHED */
11769 :
11770 :
11771 0 : static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task)
11772 : {
11773 0 : struct sched_entity *se = &task->se;
11774 0 : unsigned int rr_interval = 0;
11775 :
11776 : /*
11777 : * Time slice is 0 for SCHED_OTHER tasks that are on an otherwise
11778 : * idle runqueue:
11779 : */
11780 0 : if (rq->cfs.load.weight)
11781 0 : rr_interval = NS_TO_JIFFIES(sched_slice(cfs_rq_of(se), se));
11782 :
11783 0 : return rr_interval;
11784 : }
11785 :
11786 : /*
11787 : * All the scheduling class methods:
11788 : */
11789 : DEFINE_SCHED_CLASS(fair) = {
11790 :
11791 : .enqueue_task = enqueue_task_fair,
11792 : .dequeue_task = dequeue_task_fair,
11793 : .yield_task = yield_task_fair,
11794 : .yield_to_task = yield_to_task_fair,
11795 :
11796 : .check_preempt_curr = check_preempt_wakeup,
11797 :
11798 : .pick_next_task = __pick_next_task_fair,
11799 : .put_prev_task = put_prev_task_fair,
11800 : .set_next_task = set_next_task_fair,
11801 :
11802 : #ifdef CONFIG_SMP
11803 : .balance = balance_fair,
11804 : .pick_task = pick_task_fair,
11805 : .select_task_rq = select_task_rq_fair,
11806 : .migrate_task_rq = migrate_task_rq_fair,
11807 :
11808 : .rq_online = rq_online_fair,
11809 : .rq_offline = rq_offline_fair,
11810 :
11811 : .task_dead = task_dead_fair,
11812 : .set_cpus_allowed = set_cpus_allowed_common,
11813 : #endif
11814 :
11815 : .task_tick = task_tick_fair,
11816 : .task_fork = task_fork_fair,
11817 :
11818 : .prio_changed = prio_changed_fair,
11819 : .switched_from = switched_from_fair,
11820 : .switched_to = switched_to_fair,
11821 :
11822 : .get_rr_interval = get_rr_interval_fair,
11823 :
11824 : .update_curr = update_curr_fair,
11825 :
11826 : #ifdef CONFIG_FAIR_GROUP_SCHED
11827 : .task_change_group = task_change_group_fair,
11828 : #endif
11829 :
11830 : #ifdef CONFIG_UCLAMP_TASK
11831 : .uclamp_enabled = 1,
11832 : #endif
11833 : };
11834 :
11835 : #ifdef CONFIG_SCHED_DEBUG
11836 0 : void print_cfs_stats(struct seq_file *m, int cpu)
11837 : {
11838 : struct cfs_rq *cfs_rq, *pos;
11839 :
11840 : rcu_read_lock();
11841 0 : for_each_leaf_cfs_rq_safe(cpu_rq(cpu), cfs_rq, pos)
11842 0 : print_cfs_rq(m, cpu, cfs_rq);
11843 : rcu_read_unlock();
11844 0 : }
11845 :
11846 : #ifdef CONFIG_NUMA_BALANCING
11847 : void show_numa_stats(struct task_struct *p, struct seq_file *m)
11848 : {
11849 : int node;
11850 : unsigned long tsf = 0, tpf = 0, gsf = 0, gpf = 0;
11851 : struct numa_group *ng;
11852 :
11853 : rcu_read_lock();
11854 : ng = rcu_dereference(p->numa_group);
11855 : for_each_online_node(node) {
11856 : if (p->numa_faults) {
11857 : tsf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 0)];
11858 : tpf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 1)];
11859 : }
11860 : if (ng) {
11861 : gsf = ng->faults[task_faults_idx(NUMA_MEM, node, 0)],
11862 : gpf = ng->faults[task_faults_idx(NUMA_MEM, node, 1)];
11863 : }
11864 : print_numa_stats(m, node, tsf, tpf, gsf, gpf);
11865 : }
11866 : rcu_read_unlock();
11867 : }
11868 : #endif /* CONFIG_NUMA_BALANCING */
11869 : #endif /* CONFIG_SCHED_DEBUG */
11870 :
11871 1 : __init void init_sched_fair_class(void)
11872 : {
11873 : #ifdef CONFIG_SMP
11874 : open_softirq(SCHED_SOFTIRQ, run_rebalance_domains);
11875 :
11876 : #ifdef CONFIG_NO_HZ_COMMON
11877 : nohz.next_balance = jiffies;
11878 : nohz.next_blocked = jiffies;
11879 : zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT);
11880 : #endif
11881 : #endif /* SMP */
11882 :
11883 1 : }
11884 :
11885 : /*
11886 : * Helper functions to facilitate extracting info from tracepoints.
11887 : */
11888 :
11889 0 : const struct sched_avg *sched_trace_cfs_rq_avg(struct cfs_rq *cfs_rq)
11890 : {
11891 : #ifdef CONFIG_SMP
11892 : return cfs_rq ? &cfs_rq->avg : NULL;
11893 : #else
11894 0 : return NULL;
11895 : #endif
11896 : }
11897 : EXPORT_SYMBOL_GPL(sched_trace_cfs_rq_avg);
11898 :
11899 0 : char *sched_trace_cfs_rq_path(struct cfs_rq *cfs_rq, char *str, int len)
11900 : {
11901 0 : if (!cfs_rq) {
11902 0 : if (str)
11903 0 : strlcpy(str, "(null)", len);
11904 : else
11905 : return NULL;
11906 : }
11907 :
11908 0 : cfs_rq_tg_path(cfs_rq, str, len);
11909 : return str;
11910 : }
11911 : EXPORT_SYMBOL_GPL(sched_trace_cfs_rq_path);
11912 :
11913 0 : int sched_trace_cfs_rq_cpu(struct cfs_rq *cfs_rq)
11914 : {
11915 0 : return cfs_rq ? cpu_of(rq_of(cfs_rq)) : -1;
11916 : }
11917 : EXPORT_SYMBOL_GPL(sched_trace_cfs_rq_cpu);
11918 :
11919 0 : const struct sched_avg *sched_trace_rq_avg_rt(struct rq *rq)
11920 : {
11921 : #ifdef CONFIG_SMP
11922 : return rq ? &rq->avg_rt : NULL;
11923 : #else
11924 0 : return NULL;
11925 : #endif
11926 : }
11927 : EXPORT_SYMBOL_GPL(sched_trace_rq_avg_rt);
11928 :
11929 0 : const struct sched_avg *sched_trace_rq_avg_dl(struct rq *rq)
11930 : {
11931 : #ifdef CONFIG_SMP
11932 : return rq ? &rq->avg_dl : NULL;
11933 : #else
11934 0 : return NULL;
11935 : #endif
11936 : }
11937 : EXPORT_SYMBOL_GPL(sched_trace_rq_avg_dl);
11938 :
11939 0 : const struct sched_avg *sched_trace_rq_avg_irq(struct rq *rq)
11940 : {
11941 : #if defined(CONFIG_SMP) && defined(CONFIG_HAVE_SCHED_AVG_IRQ)
11942 : return rq ? &rq->avg_irq : NULL;
11943 : #else
11944 0 : return NULL;
11945 : #endif
11946 : }
11947 : EXPORT_SYMBOL_GPL(sched_trace_rq_avg_irq);
11948 :
11949 0 : int sched_trace_rq_cpu(struct rq *rq)
11950 : {
11951 0 : return rq ? cpu_of(rq) : -1;
11952 : }
11953 : EXPORT_SYMBOL_GPL(sched_trace_rq_cpu);
11954 :
11955 0 : int sched_trace_rq_cpu_capacity(struct rq *rq)
11956 : {
11957 : return rq ?
11958 : #ifdef CONFIG_SMP
11959 : rq->cpu_capacity
11960 : #else
11961 : SCHED_CAPACITY_SCALE
11962 : #endif
11963 0 : : -1;
11964 : }
11965 : EXPORT_SYMBOL_GPL(sched_trace_rq_cpu_capacity);
11966 :
11967 0 : const struct cpumask *sched_trace_rd_span(struct root_domain *rd)
11968 : {
11969 : #ifdef CONFIG_SMP
11970 : return rd ? rd->span : NULL;
11971 : #else
11972 0 : return NULL;
11973 : #endif
11974 : }
11975 : EXPORT_SYMBOL_GPL(sched_trace_rd_span);
11976 :
11977 0 : int sched_trace_rq_nr_running(struct rq *rq)
11978 : {
11979 0 : return rq ? rq->nr_running : -1;
11980 : }
11981 : EXPORT_SYMBOL_GPL(sched_trace_rq_nr_running);
|
