Line data Source code
1 : /* SPDX-License-Identifier: GPL-2.0 */
2 : #ifndef _LINUX_SCHED_H
3 : #define _LINUX_SCHED_H
4 :
5 : /*
6 : * Define 'struct task_struct' and provide the main scheduler
7 : * APIs (schedule(), wakeup variants, etc.)
8 : */
9 :
10 : #include <uapi/linux/sched.h>
11 :
12 : #include <asm/current.h>
13 :
14 : #include <linux/pid.h>
15 : #include <linux/sem.h>
16 : #include <linux/shm.h>
17 : #include <linux/mutex.h>
18 : #include <linux/plist.h>
19 : #include <linux/hrtimer.h>
20 : #include <linux/irqflags.h>
21 : #include <linux/seccomp.h>
22 : #include <linux/nodemask.h>
23 : #include <linux/rcupdate.h>
24 : #include <linux/refcount.h>
25 : #include <linux/resource.h>
26 : #include <linux/latencytop.h>
27 : #include <linux/sched/prio.h>
28 : #include <linux/sched/types.h>
29 : #include <linux/signal_types.h>
30 : #include <linux/syscall_user_dispatch.h>
31 : #include <linux/mm_types_task.h>
32 : #include <linux/task_io_accounting.h>
33 : #include <linux/posix-timers.h>
34 : #include <linux/rseq.h>
35 : #include <linux/seqlock.h>
36 : #include <linux/kcsan.h>
37 : #include <asm/kmap_size.h>
38 :
39 : /* task_struct member predeclarations (sorted alphabetically): */
40 : struct audit_context;
41 : struct backing_dev_info;
42 : struct bio_list;
43 : struct blk_plug;
44 : struct bpf_local_storage;
45 : struct bpf_run_ctx;
46 : struct capture_control;
47 : struct cfs_rq;
48 : struct fs_struct;
49 : struct futex_pi_state;
50 : struct io_context;
51 : struct io_uring_task;
52 : struct mempolicy;
53 : struct nameidata;
54 : struct nsproxy;
55 : struct perf_event_context;
56 : struct pid_namespace;
57 : struct pipe_inode_info;
58 : struct rcu_node;
59 : struct reclaim_state;
60 : struct robust_list_head;
61 : struct root_domain;
62 : struct rq;
63 : struct sched_attr;
64 : struct sched_param;
65 : struct seq_file;
66 : struct sighand_struct;
67 : struct signal_struct;
68 : struct task_delay_info;
69 : struct task_group;
70 :
71 : /*
72 : * Task state bitmask. NOTE! These bits are also
73 : * encoded in fs/proc/array.c: get_task_state().
74 : *
75 : * We have two separate sets of flags: task->state
76 : * is about runnability, while task->exit_state are
77 : * about the task exiting. Confusing, but this way
78 : * modifying one set can't modify the other one by
79 : * mistake.
80 : */
81 :
82 : /* Used in tsk->state: */
83 : #define TASK_RUNNING 0x0000
84 : #define TASK_INTERRUPTIBLE 0x0001
85 : #define TASK_UNINTERRUPTIBLE 0x0002
86 : #define __TASK_STOPPED 0x0004
87 : #define __TASK_TRACED 0x0008
88 : /* Used in tsk->exit_state: */
89 : #define EXIT_DEAD 0x0010
90 : #define EXIT_ZOMBIE 0x0020
91 : #define EXIT_TRACE (EXIT_ZOMBIE | EXIT_DEAD)
92 : /* Used in tsk->state again: */
93 : #define TASK_PARKED 0x0040
94 : #define TASK_DEAD 0x0080
95 : #define TASK_WAKEKILL 0x0100
96 : #define TASK_WAKING 0x0200
97 : #define TASK_NOLOAD 0x0400
98 : #define TASK_NEW 0x0800
99 : /* RT specific auxilliary flag to mark RT lock waiters */
100 : #define TASK_RTLOCK_WAIT 0x1000
101 : #define TASK_STATE_MAX 0x2000
102 :
103 : /* Convenience macros for the sake of set_current_state: */
104 : #define TASK_KILLABLE (TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)
105 : #define TASK_STOPPED (TASK_WAKEKILL | __TASK_STOPPED)
106 : #define TASK_TRACED (TASK_WAKEKILL | __TASK_TRACED)
107 :
108 : #define TASK_IDLE (TASK_UNINTERRUPTIBLE | TASK_NOLOAD)
109 :
110 : /* Convenience macros for the sake of wake_up(): */
111 : #define TASK_NORMAL (TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE)
112 :
113 : /* get_task_state(): */
114 : #define TASK_REPORT (TASK_RUNNING | TASK_INTERRUPTIBLE | \
115 : TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \
116 : __TASK_TRACED | EXIT_DEAD | EXIT_ZOMBIE | \
117 : TASK_PARKED)
118 :
119 : #define task_is_running(task) (READ_ONCE((task)->__state) == TASK_RUNNING)
120 :
121 : #define task_is_traced(task) ((READ_ONCE(task->__state) & __TASK_TRACED) != 0)
122 :
123 : #define task_is_stopped(task) ((READ_ONCE(task->__state) & __TASK_STOPPED) != 0)
124 :
125 : #define task_is_stopped_or_traced(task) ((READ_ONCE(task->__state) & (__TASK_STOPPED | __TASK_TRACED)) != 0)
126 :
127 : /*
128 : * Special states are those that do not use the normal wait-loop pattern. See
129 : * the comment with set_special_state().
130 : */
131 : #define is_special_task_state(state) \
132 : ((state) & (__TASK_STOPPED | __TASK_TRACED | TASK_PARKED | TASK_DEAD))
133 :
134 : #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
135 : # define debug_normal_state_change(state_value) \
136 : do { \
137 : WARN_ON_ONCE(is_special_task_state(state_value)); \
138 : current->task_state_change = _THIS_IP_; \
139 : } while (0)
140 :
141 : # define debug_special_state_change(state_value) \
142 : do { \
143 : WARN_ON_ONCE(!is_special_task_state(state_value)); \
144 : current->task_state_change = _THIS_IP_; \
145 : } while (0)
146 :
147 : # define debug_rtlock_wait_set_state() \
148 : do { \
149 : current->saved_state_change = current->task_state_change;\
150 : current->task_state_change = _THIS_IP_; \
151 : } while (0)
152 :
153 : # define debug_rtlock_wait_restore_state() \
154 : do { \
155 : current->task_state_change = current->saved_state_change;\
156 : } while (0)
157 :
158 : #else
159 : # define debug_normal_state_change(cond) do { } while (0)
160 : # define debug_special_state_change(cond) do { } while (0)
161 : # define debug_rtlock_wait_set_state() do { } while (0)
162 : # define debug_rtlock_wait_restore_state() do { } while (0)
163 : #endif
164 :
165 : /*
166 : * set_current_state() includes a barrier so that the write of current->state
167 : * is correctly serialised wrt the caller's subsequent test of whether to
168 : * actually sleep:
169 : *
170 : * for (;;) {
171 : * set_current_state(TASK_UNINTERRUPTIBLE);
172 : * if (CONDITION)
173 : * break;
174 : *
175 : * schedule();
176 : * }
177 : * __set_current_state(TASK_RUNNING);
178 : *
179 : * If the caller does not need such serialisation (because, for instance, the
180 : * CONDITION test and condition change and wakeup are under the same lock) then
181 : * use __set_current_state().
182 : *
183 : * The above is typically ordered against the wakeup, which does:
184 : *
185 : * CONDITION = 1;
186 : * wake_up_state(p, TASK_UNINTERRUPTIBLE);
187 : *
188 : * where wake_up_state()/try_to_wake_up() executes a full memory barrier before
189 : * accessing p->state.
190 : *
191 : * Wakeup will do: if (@state & p->state) p->state = TASK_RUNNING, that is,
192 : * once it observes the TASK_UNINTERRUPTIBLE store the waking CPU can issue a
193 : * TASK_RUNNING store which can collide with __set_current_state(TASK_RUNNING).
194 : *
195 : * However, with slightly different timing the wakeup TASK_RUNNING store can
196 : * also collide with the TASK_UNINTERRUPTIBLE store. Losing that store is not
197 : * a problem either because that will result in one extra go around the loop
198 : * and our @cond test will save the day.
199 : *
200 : * Also see the comments of try_to_wake_up().
201 : */
202 : #define __set_current_state(state_value) \
203 : do { \
204 : debug_normal_state_change((state_value)); \
205 : WRITE_ONCE(current->__state, (state_value)); \
206 : } while (0)
207 :
208 : #define set_current_state(state_value) \
209 : do { \
210 : debug_normal_state_change((state_value)); \
211 : smp_store_mb(current->__state, (state_value)); \
212 : } while (0)
213 :
214 : /*
215 : * set_special_state() should be used for those states when the blocking task
216 : * can not use the regular condition based wait-loop. In that case we must
217 : * serialize against wakeups such that any possible in-flight TASK_RUNNING
218 : * stores will not collide with our state change.
219 : */
220 : #define set_special_state(state_value) \
221 : do { \
222 : unsigned long flags; /* may shadow */ \
223 : \
224 : raw_spin_lock_irqsave(¤t->pi_lock, flags); \
225 : debug_special_state_change((state_value)); \
226 : WRITE_ONCE(current->__state, (state_value)); \
227 : raw_spin_unlock_irqrestore(¤t->pi_lock, flags); \
228 : } while (0)
229 :
230 : /*
231 : * PREEMPT_RT specific variants for "sleeping" spin/rwlocks
232 : *
233 : * RT's spin/rwlock substitutions are state preserving. The state of the
234 : * task when blocking on the lock is saved in task_struct::saved_state and
235 : * restored after the lock has been acquired. These operations are
236 : * serialized by task_struct::pi_lock against try_to_wake_up(). Any non RT
237 : * lock related wakeups while the task is blocked on the lock are
238 : * redirected to operate on task_struct::saved_state to ensure that these
239 : * are not dropped. On restore task_struct::saved_state is set to
240 : * TASK_RUNNING so any wakeup attempt redirected to saved_state will fail.
241 : *
242 : * The lock operation looks like this:
243 : *
244 : * current_save_and_set_rtlock_wait_state();
245 : * for (;;) {
246 : * if (try_lock())
247 : * break;
248 : * raw_spin_unlock_irq(&lock->wait_lock);
249 : * schedule_rtlock();
250 : * raw_spin_lock_irq(&lock->wait_lock);
251 : * set_current_state(TASK_RTLOCK_WAIT);
252 : * }
253 : * current_restore_rtlock_saved_state();
254 : */
255 : #define current_save_and_set_rtlock_wait_state() \
256 : do { \
257 : lockdep_assert_irqs_disabled(); \
258 : raw_spin_lock(¤t->pi_lock); \
259 : current->saved_state = current->__state; \
260 : debug_rtlock_wait_set_state(); \
261 : WRITE_ONCE(current->__state, TASK_RTLOCK_WAIT); \
262 : raw_spin_unlock(¤t->pi_lock); \
263 : } while (0);
264 :
265 : #define current_restore_rtlock_saved_state() \
266 : do { \
267 : lockdep_assert_irqs_disabled(); \
268 : raw_spin_lock(¤t->pi_lock); \
269 : debug_rtlock_wait_restore_state(); \
270 : WRITE_ONCE(current->__state, current->saved_state); \
271 : current->saved_state = TASK_RUNNING; \
272 : raw_spin_unlock(¤t->pi_lock); \
273 : } while (0);
274 :
275 : #define get_current_state() READ_ONCE(current->__state)
276 :
277 : /*
278 : * Define the task command name length as enum, then it can be visible to
279 : * BPF programs.
280 : */
281 : enum {
282 : TASK_COMM_LEN = 16,
283 : };
284 :
285 : extern void scheduler_tick(void);
286 :
287 : #define MAX_SCHEDULE_TIMEOUT LONG_MAX
288 :
289 : extern long schedule_timeout(long timeout);
290 : extern long schedule_timeout_interruptible(long timeout);
291 : extern long schedule_timeout_killable(long timeout);
292 : extern long schedule_timeout_uninterruptible(long timeout);
293 : extern long schedule_timeout_idle(long timeout);
294 : asmlinkage void schedule(void);
295 : extern void schedule_preempt_disabled(void);
296 : asmlinkage void preempt_schedule_irq(void);
297 : #ifdef CONFIG_PREEMPT_RT
298 : extern void schedule_rtlock(void);
299 : #endif
300 :
301 : extern int __must_check io_schedule_prepare(void);
302 : extern void io_schedule_finish(int token);
303 : extern long io_schedule_timeout(long timeout);
304 : extern void io_schedule(void);
305 :
306 : /**
307 : * struct prev_cputime - snapshot of system and user cputime
308 : * @utime: time spent in user mode
309 : * @stime: time spent in system mode
310 : * @lock: protects the above two fields
311 : *
312 : * Stores previous user/system time values such that we can guarantee
313 : * monotonicity.
314 : */
315 : struct prev_cputime {
316 : #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
317 : u64 utime;
318 : u64 stime;
319 : raw_spinlock_t lock;
320 : #endif
321 : };
322 :
323 : enum vtime_state {
324 : /* Task is sleeping or running in a CPU with VTIME inactive: */
325 : VTIME_INACTIVE = 0,
326 : /* Task is idle */
327 : VTIME_IDLE,
328 : /* Task runs in kernelspace in a CPU with VTIME active: */
329 : VTIME_SYS,
330 : /* Task runs in userspace in a CPU with VTIME active: */
331 : VTIME_USER,
332 : /* Task runs as guests in a CPU with VTIME active: */
333 : VTIME_GUEST,
334 : };
335 :
336 : struct vtime {
337 : seqcount_t seqcount;
338 : unsigned long long starttime;
339 : enum vtime_state state;
340 : unsigned int cpu;
341 : u64 utime;
342 : u64 stime;
343 : u64 gtime;
344 : };
345 :
346 : /*
347 : * Utilization clamp constraints.
348 : * @UCLAMP_MIN: Minimum utilization
349 : * @UCLAMP_MAX: Maximum utilization
350 : * @UCLAMP_CNT: Utilization clamp constraints count
351 : */
352 : enum uclamp_id {
353 : UCLAMP_MIN = 0,
354 : UCLAMP_MAX,
355 : UCLAMP_CNT
356 : };
357 :
358 : #ifdef CONFIG_SMP
359 : extern struct root_domain def_root_domain;
360 : extern struct mutex sched_domains_mutex;
361 : #endif
362 :
363 : struct sched_info {
364 : #ifdef CONFIG_SCHED_INFO
365 : /* Cumulative counters: */
366 :
367 : /* # of times we have run on this CPU: */
368 : unsigned long pcount;
369 :
370 : /* Time spent waiting on a runqueue: */
371 : unsigned long long run_delay;
372 :
373 : /* Timestamps: */
374 :
375 : /* When did we last run on a CPU? */
376 : unsigned long long last_arrival;
377 :
378 : /* When were we last queued to run? */
379 : unsigned long long last_queued;
380 :
381 : #endif /* CONFIG_SCHED_INFO */
382 : };
383 :
384 : /*
385 : * Integer metrics need fixed point arithmetic, e.g., sched/fair
386 : * has a few: load, load_avg, util_avg, freq, and capacity.
387 : *
388 : * We define a basic fixed point arithmetic range, and then formalize
389 : * all these metrics based on that basic range.
390 : */
391 : # define SCHED_FIXEDPOINT_SHIFT 10
392 : # define SCHED_FIXEDPOINT_SCALE (1L << SCHED_FIXEDPOINT_SHIFT)
393 :
394 : /* Increase resolution of cpu_capacity calculations */
395 : # define SCHED_CAPACITY_SHIFT SCHED_FIXEDPOINT_SHIFT
396 : # define SCHED_CAPACITY_SCALE (1L << SCHED_CAPACITY_SHIFT)
397 :
398 : struct load_weight {
399 : unsigned long weight;
400 : u32 inv_weight;
401 : };
402 :
403 : /**
404 : * struct util_est - Estimation utilization of FAIR tasks
405 : * @enqueued: instantaneous estimated utilization of a task/cpu
406 : * @ewma: the Exponential Weighted Moving Average (EWMA)
407 : * utilization of a task
408 : *
409 : * Support data structure to track an Exponential Weighted Moving Average
410 : * (EWMA) of a FAIR task's utilization. New samples are added to the moving
411 : * average each time a task completes an activation. Sample's weight is chosen
412 : * so that the EWMA will be relatively insensitive to transient changes to the
413 : * task's workload.
414 : *
415 : * The enqueued attribute has a slightly different meaning for tasks and cpus:
416 : * - task: the task's util_avg at last task dequeue time
417 : * - cfs_rq: the sum of util_est.enqueued for each RUNNABLE task on that CPU
418 : * Thus, the util_est.enqueued of a task represents the contribution on the
419 : * estimated utilization of the CPU where that task is currently enqueued.
420 : *
421 : * Only for tasks we track a moving average of the past instantaneous
422 : * estimated utilization. This allows to absorb sporadic drops in utilization
423 : * of an otherwise almost periodic task.
424 : *
425 : * The UTIL_AVG_UNCHANGED flag is used to synchronize util_est with util_avg
426 : * updates. When a task is dequeued, its util_est should not be updated if its
427 : * util_avg has not been updated in the meantime.
428 : * This information is mapped into the MSB bit of util_est.enqueued at dequeue
429 : * time. Since max value of util_est.enqueued for a task is 1024 (PELT util_avg
430 : * for a task) it is safe to use MSB.
431 : */
432 : struct util_est {
433 : unsigned int enqueued;
434 : unsigned int ewma;
435 : #define UTIL_EST_WEIGHT_SHIFT 2
436 : #define UTIL_AVG_UNCHANGED 0x80000000
437 : } __attribute__((__aligned__(sizeof(u64))));
438 :
439 : /*
440 : * The load/runnable/util_avg accumulates an infinite geometric series
441 : * (see __update_load_avg_cfs_rq() in kernel/sched/pelt.c).
442 : *
443 : * [load_avg definition]
444 : *
445 : * load_avg = runnable% * scale_load_down(load)
446 : *
447 : * [runnable_avg definition]
448 : *
449 : * runnable_avg = runnable% * SCHED_CAPACITY_SCALE
450 : *
451 : * [util_avg definition]
452 : *
453 : * util_avg = running% * SCHED_CAPACITY_SCALE
454 : *
455 : * where runnable% is the time ratio that a sched_entity is runnable and
456 : * running% the time ratio that a sched_entity is running.
457 : *
458 : * For cfs_rq, they are the aggregated values of all runnable and blocked
459 : * sched_entities.
460 : *
461 : * The load/runnable/util_avg doesn't directly factor frequency scaling and CPU
462 : * capacity scaling. The scaling is done through the rq_clock_pelt that is used
463 : * for computing those signals (see update_rq_clock_pelt())
464 : *
465 : * N.B., the above ratios (runnable% and running%) themselves are in the
466 : * range of [0, 1]. To do fixed point arithmetics, we therefore scale them
467 : * to as large a range as necessary. This is for example reflected by
468 : * util_avg's SCHED_CAPACITY_SCALE.
469 : *
470 : * [Overflow issue]
471 : *
472 : * The 64-bit load_sum can have 4353082796 (=2^64/47742/88761) entities
473 : * with the highest load (=88761), always runnable on a single cfs_rq,
474 : * and should not overflow as the number already hits PID_MAX_LIMIT.
475 : *
476 : * For all other cases (including 32-bit kernels), struct load_weight's
477 : * weight will overflow first before we do, because:
478 : *
479 : * Max(load_avg) <= Max(load.weight)
480 : *
481 : * Then it is the load_weight's responsibility to consider overflow
482 : * issues.
483 : */
484 : struct sched_avg {
485 : u64 last_update_time;
486 : u64 load_sum;
487 : u64 runnable_sum;
488 : u32 util_sum;
489 : u32 period_contrib;
490 : unsigned long load_avg;
491 : unsigned long runnable_avg;
492 : unsigned long util_avg;
493 : struct util_est util_est;
494 : } ____cacheline_aligned;
495 :
496 : struct sched_statistics {
497 : #ifdef CONFIG_SCHEDSTATS
498 : u64 wait_start;
499 : u64 wait_max;
500 : u64 wait_count;
501 : u64 wait_sum;
502 : u64 iowait_count;
503 : u64 iowait_sum;
504 :
505 : u64 sleep_start;
506 : u64 sleep_max;
507 : s64 sum_sleep_runtime;
508 :
509 : u64 block_start;
510 : u64 block_max;
511 : s64 sum_block_runtime;
512 :
513 : u64 exec_max;
514 : u64 slice_max;
515 :
516 : u64 nr_migrations_cold;
517 : u64 nr_failed_migrations_affine;
518 : u64 nr_failed_migrations_running;
519 : u64 nr_failed_migrations_hot;
520 : u64 nr_forced_migrations;
521 :
522 : u64 nr_wakeups;
523 : u64 nr_wakeups_sync;
524 : u64 nr_wakeups_migrate;
525 : u64 nr_wakeups_local;
526 : u64 nr_wakeups_remote;
527 : u64 nr_wakeups_affine;
528 : u64 nr_wakeups_affine_attempts;
529 : u64 nr_wakeups_passive;
530 : u64 nr_wakeups_idle;
531 :
532 : #ifdef CONFIG_SCHED_CORE
533 : u64 core_forceidle_sum;
534 : #endif
535 : #endif /* CONFIG_SCHEDSTATS */
536 : } ____cacheline_aligned;
537 :
538 : struct sched_entity {
539 : /* For load-balancing: */
540 : struct load_weight load;
541 : struct rb_node run_node;
542 : struct list_head group_node;
543 : unsigned int on_rq;
544 :
545 : u64 exec_start;
546 : u64 sum_exec_runtime;
547 : u64 vruntime;
548 : u64 prev_sum_exec_runtime;
549 :
550 : u64 nr_migrations;
551 :
552 : #ifdef CONFIG_FAIR_GROUP_SCHED
553 : int depth;
554 : struct sched_entity *parent;
555 : /* rq on which this entity is (to be) queued: */
556 : struct cfs_rq *cfs_rq;
557 : /* rq "owned" by this entity/group: */
558 : struct cfs_rq *my_q;
559 : /* cached value of my_q->h_nr_running */
560 : unsigned long runnable_weight;
561 : #endif
562 :
563 : #ifdef CONFIG_SMP
564 : /*
565 : * Per entity load average tracking.
566 : *
567 : * Put into separate cache line so it does not
568 : * collide with read-mostly values above.
569 : */
570 : struct sched_avg avg;
571 : #endif
572 : };
573 :
574 : struct sched_rt_entity {
575 : struct list_head run_list;
576 : unsigned long timeout;
577 : unsigned long watchdog_stamp;
578 : unsigned int time_slice;
579 : unsigned short on_rq;
580 : unsigned short on_list;
581 :
582 : struct sched_rt_entity *back;
583 : #ifdef CONFIG_RT_GROUP_SCHED
584 : struct sched_rt_entity *parent;
585 : /* rq on which this entity is (to be) queued: */
586 : struct rt_rq *rt_rq;
587 : /* rq "owned" by this entity/group: */
588 : struct rt_rq *my_q;
589 : #endif
590 : } __randomize_layout;
591 :
592 : struct sched_dl_entity {
593 : struct rb_node rb_node;
594 :
595 : /*
596 : * Original scheduling parameters. Copied here from sched_attr
597 : * during sched_setattr(), they will remain the same until
598 : * the next sched_setattr().
599 : */
600 : u64 dl_runtime; /* Maximum runtime for each instance */
601 : u64 dl_deadline; /* Relative deadline of each instance */
602 : u64 dl_period; /* Separation of two instances (period) */
603 : u64 dl_bw; /* dl_runtime / dl_period */
604 : u64 dl_density; /* dl_runtime / dl_deadline */
605 :
606 : /*
607 : * Actual scheduling parameters. Initialized with the values above,
608 : * they are continuously updated during task execution. Note that
609 : * the remaining runtime could be < 0 in case we are in overrun.
610 : */
611 : s64 runtime; /* Remaining runtime for this instance */
612 : u64 deadline; /* Absolute deadline for this instance */
613 : unsigned int flags; /* Specifying the scheduler behaviour */
614 :
615 : /*
616 : * Some bool flags:
617 : *
618 : * @dl_throttled tells if we exhausted the runtime. If so, the
619 : * task has to wait for a replenishment to be performed at the
620 : * next firing of dl_timer.
621 : *
622 : * @dl_yielded tells if task gave up the CPU before consuming
623 : * all its available runtime during the last job.
624 : *
625 : * @dl_non_contending tells if the task is inactive while still
626 : * contributing to the active utilization. In other words, it
627 : * indicates if the inactive timer has been armed and its handler
628 : * has not been executed yet. This flag is useful to avoid race
629 : * conditions between the inactive timer handler and the wakeup
630 : * code.
631 : *
632 : * @dl_overrun tells if the task asked to be informed about runtime
633 : * overruns.
634 : */
635 : unsigned int dl_throttled : 1;
636 : unsigned int dl_yielded : 1;
637 : unsigned int dl_non_contending : 1;
638 : unsigned int dl_overrun : 1;
639 :
640 : /*
641 : * Bandwidth enforcement timer. Each -deadline task has its
642 : * own bandwidth to be enforced, thus we need one timer per task.
643 : */
644 : struct hrtimer dl_timer;
645 :
646 : /*
647 : * Inactive timer, responsible for decreasing the active utilization
648 : * at the "0-lag time". When a -deadline task blocks, it contributes
649 : * to GRUB's active utilization until the "0-lag time", hence a
650 : * timer is needed to decrease the active utilization at the correct
651 : * time.
652 : */
653 : struct hrtimer inactive_timer;
654 :
655 : #ifdef CONFIG_RT_MUTEXES
656 : /*
657 : * Priority Inheritance. When a DEADLINE scheduling entity is boosted
658 : * pi_se points to the donor, otherwise points to the dl_se it belongs
659 : * to (the original one/itself).
660 : */
661 : struct sched_dl_entity *pi_se;
662 : #endif
663 : };
664 :
665 : #ifdef CONFIG_UCLAMP_TASK
666 : /* Number of utilization clamp buckets (shorter alias) */
667 : #define UCLAMP_BUCKETS CONFIG_UCLAMP_BUCKETS_COUNT
668 :
669 : /*
670 : * Utilization clamp for a scheduling entity
671 : * @value: clamp value "assigned" to a se
672 : * @bucket_id: bucket index corresponding to the "assigned" value
673 : * @active: the se is currently refcounted in a rq's bucket
674 : * @user_defined: the requested clamp value comes from user-space
675 : *
676 : * The bucket_id is the index of the clamp bucket matching the clamp value
677 : * which is pre-computed and stored to avoid expensive integer divisions from
678 : * the fast path.
679 : *
680 : * The active bit is set whenever a task has got an "effective" value assigned,
681 : * which can be different from the clamp value "requested" from user-space.
682 : * This allows to know a task is refcounted in the rq's bucket corresponding
683 : * to the "effective" bucket_id.
684 : *
685 : * The user_defined bit is set whenever a task has got a task-specific clamp
686 : * value requested from userspace, i.e. the system defaults apply to this task
687 : * just as a restriction. This allows to relax default clamps when a less
688 : * restrictive task-specific value has been requested, thus allowing to
689 : * implement a "nice" semantic. For example, a task running with a 20%
690 : * default boost can still drop its own boosting to 0%.
691 : */
692 : struct uclamp_se {
693 : unsigned int value : bits_per(SCHED_CAPACITY_SCALE);
694 : unsigned int bucket_id : bits_per(UCLAMP_BUCKETS);
695 : unsigned int active : 1;
696 : unsigned int user_defined : 1;
697 : };
698 : #endif /* CONFIG_UCLAMP_TASK */
699 :
700 : union rcu_special {
701 : struct {
702 : u8 blocked;
703 : u8 need_qs;
704 : u8 exp_hint; /* Hint for performance. */
705 : u8 need_mb; /* Readers need smp_mb(). */
706 : } b; /* Bits. */
707 : u32 s; /* Set of bits. */
708 : };
709 :
710 : enum perf_event_task_context {
711 : perf_invalid_context = -1,
712 : perf_hw_context = 0,
713 : perf_sw_context,
714 : perf_nr_task_contexts,
715 : };
716 :
717 : struct wake_q_node {
718 : struct wake_q_node *next;
719 : };
720 :
721 : struct kmap_ctrl {
722 : #ifdef CONFIG_KMAP_LOCAL
723 : int idx;
724 : pte_t pteval[KM_MAX_IDX];
725 : #endif
726 : };
727 :
728 : struct task_struct {
729 : #ifdef CONFIG_THREAD_INFO_IN_TASK
730 : /*
731 : * For reasons of header soup (see current_thread_info()), this
732 : * must be the first element of task_struct.
733 : */
734 : struct thread_info thread_info;
735 : #endif
736 : unsigned int __state;
737 :
738 : #ifdef CONFIG_PREEMPT_RT
739 : /* saved state for "spinlock sleepers" */
740 : unsigned int saved_state;
741 : #endif
742 :
743 : /*
744 : * This begins the randomizable portion of task_struct. Only
745 : * scheduling-critical items should be added above here.
746 : */
747 : randomized_struct_fields_start
748 :
749 : void *stack;
750 : refcount_t usage;
751 : /* Per task flags (PF_*), defined further below: */
752 : unsigned int flags;
753 : unsigned int ptrace;
754 :
755 : #ifdef CONFIG_SMP
756 : int on_cpu;
757 : struct __call_single_node wake_entry;
758 : unsigned int wakee_flips;
759 : unsigned long wakee_flip_decay_ts;
760 : struct task_struct *last_wakee;
761 :
762 : /*
763 : * recent_used_cpu is initially set as the last CPU used by a task
764 : * that wakes affine another task. Waker/wakee relationships can
765 : * push tasks around a CPU where each wakeup moves to the next one.
766 : * Tracking a recently used CPU allows a quick search for a recently
767 : * used CPU that may be idle.
768 : */
769 : int recent_used_cpu;
770 : int wake_cpu;
771 : #endif
772 : int on_rq;
773 :
774 : int prio;
775 : int static_prio;
776 : int normal_prio;
777 : unsigned int rt_priority;
778 :
779 : struct sched_entity se;
780 : struct sched_rt_entity rt;
781 : struct sched_dl_entity dl;
782 : const struct sched_class *sched_class;
783 :
784 : #ifdef CONFIG_SCHED_CORE
785 : struct rb_node core_node;
786 : unsigned long core_cookie;
787 : unsigned int core_occupation;
788 : #endif
789 :
790 : #ifdef CONFIG_CGROUP_SCHED
791 : struct task_group *sched_task_group;
792 : #endif
793 :
794 : #ifdef CONFIG_UCLAMP_TASK
795 : /*
796 : * Clamp values requested for a scheduling entity.
797 : * Must be updated with task_rq_lock() held.
798 : */
799 : struct uclamp_se uclamp_req[UCLAMP_CNT];
800 : /*
801 : * Effective clamp values used for a scheduling entity.
802 : * Must be updated with task_rq_lock() held.
803 : */
804 : struct uclamp_se uclamp[UCLAMP_CNT];
805 : #endif
806 :
807 : struct sched_statistics stats;
808 :
809 : #ifdef CONFIG_PREEMPT_NOTIFIERS
810 : /* List of struct preempt_notifier: */
811 : struct hlist_head preempt_notifiers;
812 : #endif
813 :
814 : #ifdef CONFIG_BLK_DEV_IO_TRACE
815 : unsigned int btrace_seq;
816 : #endif
817 :
818 : unsigned int policy;
819 : int nr_cpus_allowed;
820 : const cpumask_t *cpus_ptr;
821 : cpumask_t *user_cpus_ptr;
822 : cpumask_t cpus_mask;
823 : void *migration_pending;
824 : #ifdef CONFIG_SMP
825 : unsigned short migration_disabled;
826 : #endif
827 : unsigned short migration_flags;
828 :
829 : #ifdef CONFIG_PREEMPT_RCU
830 : int rcu_read_lock_nesting;
831 : union rcu_special rcu_read_unlock_special;
832 : struct list_head rcu_node_entry;
833 : struct rcu_node *rcu_blocked_node;
834 : #endif /* #ifdef CONFIG_PREEMPT_RCU */
835 :
836 : #ifdef CONFIG_TASKS_RCU
837 : unsigned long rcu_tasks_nvcsw;
838 : u8 rcu_tasks_holdout;
839 : u8 rcu_tasks_idx;
840 : int rcu_tasks_idle_cpu;
841 : struct list_head rcu_tasks_holdout_list;
842 : #endif /* #ifdef CONFIG_TASKS_RCU */
843 :
844 : #ifdef CONFIG_TASKS_TRACE_RCU
845 : int trc_reader_nesting;
846 : int trc_ipi_to_cpu;
847 : union rcu_special trc_reader_special;
848 : bool trc_reader_checked;
849 : struct list_head trc_holdout_list;
850 : #endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
851 :
852 : struct sched_info sched_info;
853 :
854 : struct list_head tasks;
855 : #ifdef CONFIG_SMP
856 : struct plist_node pushable_tasks;
857 : struct rb_node pushable_dl_tasks;
858 : #endif
859 :
860 : struct mm_struct *mm;
861 : struct mm_struct *active_mm;
862 :
863 : /* Per-thread vma caching: */
864 : struct vmacache vmacache;
865 :
866 : #ifdef SPLIT_RSS_COUNTING
867 : struct task_rss_stat rss_stat;
868 : #endif
869 : int exit_state;
870 : int exit_code;
871 : int exit_signal;
872 : /* The signal sent when the parent dies: */
873 : int pdeath_signal;
874 : /* JOBCTL_*, siglock protected: */
875 : unsigned long jobctl;
876 :
877 : /* Used for emulating ABI behavior of previous Linux versions: */
878 : unsigned int personality;
879 :
880 : /* Scheduler bits, serialized by scheduler locks: */
881 : unsigned sched_reset_on_fork:1;
882 : unsigned sched_contributes_to_load:1;
883 : unsigned sched_migrated:1;
884 : #ifdef CONFIG_PSI
885 : unsigned sched_psi_wake_requeue:1;
886 : #endif
887 :
888 : /* Force alignment to the next boundary: */
889 : unsigned :0;
890 :
891 : /* Unserialized, strictly 'current' */
892 :
893 : /*
894 : * This field must not be in the scheduler word above due to wakelist
895 : * queueing no longer being serialized by p->on_cpu. However:
896 : *
897 : * p->XXX = X; ttwu()
898 : * schedule() if (p->on_rq && ..) // false
899 : * smp_mb__after_spinlock(); if (smp_load_acquire(&p->on_cpu) && //true
900 : * deactivate_task() ttwu_queue_wakelist())
901 : * p->on_rq = 0; p->sched_remote_wakeup = Y;
902 : *
903 : * guarantees all stores of 'current' are visible before
904 : * ->sched_remote_wakeup gets used, so it can be in this word.
905 : */
906 : unsigned sched_remote_wakeup:1;
907 :
908 : /* Bit to tell LSMs we're in execve(): */
909 : unsigned in_execve:1;
910 : unsigned in_iowait:1;
911 : #ifndef TIF_RESTORE_SIGMASK
912 : unsigned restore_sigmask:1;
913 : #endif
914 : #ifdef CONFIG_MEMCG
915 : unsigned in_user_fault:1;
916 : #endif
917 : #ifdef CONFIG_COMPAT_BRK
918 : unsigned brk_randomized:1;
919 : #endif
920 : #ifdef CONFIG_CGROUPS
921 : /* disallow userland-initiated cgroup migration */
922 : unsigned no_cgroup_migration:1;
923 : /* task is frozen/stopped (used by the cgroup freezer) */
924 : unsigned frozen:1;
925 : #endif
926 : #ifdef CONFIG_BLK_CGROUP
927 : unsigned use_memdelay:1;
928 : #endif
929 : #ifdef CONFIG_PSI
930 : /* Stalled due to lack of memory */
931 : unsigned in_memstall:1;
932 : #endif
933 : #ifdef CONFIG_PAGE_OWNER
934 : /* Used by page_owner=on to detect recursion in page tracking. */
935 : unsigned in_page_owner:1;
936 : #endif
937 : #ifdef CONFIG_EVENTFD
938 : /* Recursion prevention for eventfd_signal() */
939 : unsigned in_eventfd_signal:1;
940 : #endif
941 : #ifdef CONFIG_IOMMU_SVA
942 : unsigned pasid_activated:1;
943 : #endif
944 :
945 : unsigned long atomic_flags; /* Flags requiring atomic access. */
946 :
947 : struct restart_block restart_block;
948 :
949 : pid_t pid;
950 : pid_t tgid;
951 :
952 : #ifdef CONFIG_STACKPROTECTOR
953 : /* Canary value for the -fstack-protector GCC feature: */
954 : unsigned long stack_canary;
955 : #endif
956 : /*
957 : * Pointers to the (original) parent process, youngest child, younger sibling,
958 : * older sibling, respectively. (p->father can be replaced with
959 : * p->real_parent->pid)
960 : */
961 :
962 : /* Real parent process: */
963 : struct task_struct __rcu *real_parent;
964 :
965 : /* Recipient of SIGCHLD, wait4() reports: */
966 : struct task_struct __rcu *parent;
967 :
968 : /*
969 : * Children/sibling form the list of natural children:
970 : */
971 : struct list_head children;
972 : struct list_head sibling;
973 : struct task_struct *group_leader;
974 :
975 : /*
976 : * 'ptraced' is the list of tasks this task is using ptrace() on.
977 : *
978 : * This includes both natural children and PTRACE_ATTACH targets.
979 : * 'ptrace_entry' is this task's link on the p->parent->ptraced list.
980 : */
981 : struct list_head ptraced;
982 : struct list_head ptrace_entry;
983 :
984 : /* PID/PID hash table linkage. */
985 : struct pid *thread_pid;
986 : struct hlist_node pid_links[PIDTYPE_MAX];
987 : struct list_head thread_group;
988 : struct list_head thread_node;
989 :
990 : struct completion *vfork_done;
991 :
992 : /* CLONE_CHILD_SETTID: */
993 : int __user *set_child_tid;
994 :
995 : /* CLONE_CHILD_CLEARTID: */
996 : int __user *clear_child_tid;
997 :
998 : /* PF_KTHREAD | PF_IO_WORKER */
999 : void *worker_private;
1000 :
1001 : u64 utime;
1002 : u64 stime;
1003 : #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
1004 : u64 utimescaled;
1005 : u64 stimescaled;
1006 : #endif
1007 : u64 gtime;
1008 : struct prev_cputime prev_cputime;
1009 : #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
1010 : struct vtime vtime;
1011 : #endif
1012 :
1013 : #ifdef CONFIG_NO_HZ_FULL
1014 : atomic_t tick_dep_mask;
1015 : #endif
1016 : /* Context switch counts: */
1017 : unsigned long nvcsw;
1018 : unsigned long nivcsw;
1019 :
1020 : /* Monotonic time in nsecs: */
1021 : u64 start_time;
1022 :
1023 : /* Boot based time in nsecs: */
1024 : u64 start_boottime;
1025 :
1026 : /* MM fault and swap info: this can arguably be seen as either mm-specific or thread-specific: */
1027 : unsigned long min_flt;
1028 : unsigned long maj_flt;
1029 :
1030 : /* Empty if CONFIG_POSIX_CPUTIMERS=n */
1031 : struct posix_cputimers posix_cputimers;
1032 :
1033 : #ifdef CONFIG_POSIX_CPU_TIMERS_TASK_WORK
1034 : struct posix_cputimers_work posix_cputimers_work;
1035 : #endif
1036 :
1037 : /* Process credentials: */
1038 :
1039 : /* Tracer's credentials at attach: */
1040 : const struct cred __rcu *ptracer_cred;
1041 :
1042 : /* Objective and real subjective task credentials (COW): */
1043 : const struct cred __rcu *real_cred;
1044 :
1045 : /* Effective (overridable) subjective task credentials (COW): */
1046 : const struct cred __rcu *cred;
1047 :
1048 : #ifdef CONFIG_KEYS
1049 : /* Cached requested key. */
1050 : struct key *cached_requested_key;
1051 : #endif
1052 :
1053 : /*
1054 : * executable name, excluding path.
1055 : *
1056 : * - normally initialized setup_new_exec()
1057 : * - access it with [gs]et_task_comm()
1058 : * - lock it with task_lock()
1059 : */
1060 : char comm[TASK_COMM_LEN];
1061 :
1062 : struct nameidata *nameidata;
1063 :
1064 : #ifdef CONFIG_SYSVIPC
1065 : struct sysv_sem sysvsem;
1066 : struct sysv_shm sysvshm;
1067 : #endif
1068 : #ifdef CONFIG_DETECT_HUNG_TASK
1069 : unsigned long last_switch_count;
1070 : unsigned long last_switch_time;
1071 : #endif
1072 : /* Filesystem information: */
1073 : struct fs_struct *fs;
1074 :
1075 : /* Open file information: */
1076 : struct files_struct *files;
1077 :
1078 : #ifdef CONFIG_IO_URING
1079 : struct io_uring_task *io_uring;
1080 : #endif
1081 :
1082 : /* Namespaces: */
1083 : struct nsproxy *nsproxy;
1084 :
1085 : /* Signal handlers: */
1086 : struct signal_struct *signal;
1087 : struct sighand_struct __rcu *sighand;
1088 : sigset_t blocked;
1089 : sigset_t real_blocked;
1090 : /* Restored if set_restore_sigmask() was used: */
1091 : sigset_t saved_sigmask;
1092 : struct sigpending pending;
1093 : unsigned long sas_ss_sp;
1094 : size_t sas_ss_size;
1095 : unsigned int sas_ss_flags;
1096 :
1097 : struct callback_head *task_works;
1098 :
1099 : #ifdef CONFIG_AUDIT
1100 : #ifdef CONFIG_AUDITSYSCALL
1101 : struct audit_context *audit_context;
1102 : #endif
1103 : kuid_t loginuid;
1104 : unsigned int sessionid;
1105 : #endif
1106 : struct seccomp seccomp;
1107 : struct syscall_user_dispatch syscall_dispatch;
1108 :
1109 : /* Thread group tracking: */
1110 : u64 parent_exec_id;
1111 : u64 self_exec_id;
1112 :
1113 : /* Protection against (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed, mempolicy: */
1114 : spinlock_t alloc_lock;
1115 :
1116 : /* Protection of the PI data structures: */
1117 : raw_spinlock_t pi_lock;
1118 :
1119 : struct wake_q_node wake_q;
1120 :
1121 : #ifdef CONFIG_RT_MUTEXES
1122 : /* PI waiters blocked on a rt_mutex held by this task: */
1123 : struct rb_root_cached pi_waiters;
1124 : /* Updated under owner's pi_lock and rq lock */
1125 : struct task_struct *pi_top_task;
1126 : /* Deadlock detection and priority inheritance handling: */
1127 : struct rt_mutex_waiter *pi_blocked_on;
1128 : #endif
1129 :
1130 : #ifdef CONFIG_DEBUG_MUTEXES
1131 : /* Mutex deadlock detection: */
1132 : struct mutex_waiter *blocked_on;
1133 : #endif
1134 :
1135 : #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
1136 : int non_block_count;
1137 : #endif
1138 :
1139 : #ifdef CONFIG_TRACE_IRQFLAGS
1140 : struct irqtrace_events irqtrace;
1141 : unsigned int hardirq_threaded;
1142 : u64 hardirq_chain_key;
1143 : int softirqs_enabled;
1144 : int softirq_context;
1145 : int irq_config;
1146 : #endif
1147 : #ifdef CONFIG_PREEMPT_RT
1148 : int softirq_disable_cnt;
1149 : #endif
1150 :
1151 : #ifdef CONFIG_LOCKDEP
1152 : # define MAX_LOCK_DEPTH 48UL
1153 : u64 curr_chain_key;
1154 : int lockdep_depth;
1155 : unsigned int lockdep_recursion;
1156 : struct held_lock held_locks[MAX_LOCK_DEPTH];
1157 : #endif
1158 :
1159 : #if defined(CONFIG_UBSAN) && !defined(CONFIG_UBSAN_TRAP)
1160 : unsigned int in_ubsan;
1161 : #endif
1162 :
1163 : /* Journalling filesystem info: */
1164 : void *journal_info;
1165 :
1166 : /* Stacked block device info: */
1167 : struct bio_list *bio_list;
1168 :
1169 : /* Stack plugging: */
1170 : struct blk_plug *plug;
1171 :
1172 : /* VM state: */
1173 : struct reclaim_state *reclaim_state;
1174 :
1175 : struct backing_dev_info *backing_dev_info;
1176 :
1177 : struct io_context *io_context;
1178 :
1179 : #ifdef CONFIG_COMPACTION
1180 : struct capture_control *capture_control;
1181 : #endif
1182 : /* Ptrace state: */
1183 : unsigned long ptrace_message;
1184 : kernel_siginfo_t *last_siginfo;
1185 :
1186 : struct task_io_accounting ioac;
1187 : #ifdef CONFIG_PSI
1188 : /* Pressure stall state */
1189 : unsigned int psi_flags;
1190 : #endif
1191 : #ifdef CONFIG_TASK_XACCT
1192 : /* Accumulated RSS usage: */
1193 : u64 acct_rss_mem1;
1194 : /* Accumulated virtual memory usage: */
1195 : u64 acct_vm_mem1;
1196 : /* stime + utime since last update: */
1197 : u64 acct_timexpd;
1198 : #endif
1199 : #ifdef CONFIG_CPUSETS
1200 : /* Protected by ->alloc_lock: */
1201 : nodemask_t mems_allowed;
1202 : /* Sequence number to catch updates: */
1203 : seqcount_spinlock_t mems_allowed_seq;
1204 : int cpuset_mem_spread_rotor;
1205 : int cpuset_slab_spread_rotor;
1206 : #endif
1207 : #ifdef CONFIG_CGROUPS
1208 : /* Control Group info protected by css_set_lock: */
1209 : struct css_set __rcu *cgroups;
1210 : /* cg_list protected by css_set_lock and tsk->alloc_lock: */
1211 : struct list_head cg_list;
1212 : #endif
1213 : #ifdef CONFIG_X86_CPU_RESCTRL
1214 : u32 closid;
1215 : u32 rmid;
1216 : #endif
1217 : #ifdef CONFIG_FUTEX
1218 : struct robust_list_head __user *robust_list;
1219 : #ifdef CONFIG_COMPAT
1220 : struct compat_robust_list_head __user *compat_robust_list;
1221 : #endif
1222 : struct list_head pi_state_list;
1223 : struct futex_pi_state *pi_state_cache;
1224 : struct mutex futex_exit_mutex;
1225 : unsigned int futex_state;
1226 : #endif
1227 : #ifdef CONFIG_PERF_EVENTS
1228 : struct perf_event_context *perf_event_ctxp[perf_nr_task_contexts];
1229 : struct mutex perf_event_mutex;
1230 : struct list_head perf_event_list;
1231 : #endif
1232 : #ifdef CONFIG_DEBUG_PREEMPT
1233 : unsigned long preempt_disable_ip;
1234 : #endif
1235 : #ifdef CONFIG_NUMA
1236 : /* Protected by alloc_lock: */
1237 : struct mempolicy *mempolicy;
1238 : short il_prev;
1239 : short pref_node_fork;
1240 : #endif
1241 : #ifdef CONFIG_NUMA_BALANCING
1242 : int numa_scan_seq;
1243 : unsigned int numa_scan_period;
1244 : unsigned int numa_scan_period_max;
1245 : int numa_preferred_nid;
1246 : unsigned long numa_migrate_retry;
1247 : /* Migration stamp: */
1248 : u64 node_stamp;
1249 : u64 last_task_numa_placement;
1250 : u64 last_sum_exec_runtime;
1251 : struct callback_head numa_work;
1252 :
1253 : /*
1254 : * This pointer is only modified for current in syscall and
1255 : * pagefault context (and for tasks being destroyed), so it can be read
1256 : * from any of the following contexts:
1257 : * - RCU read-side critical section
1258 : * - current->numa_group from everywhere
1259 : * - task's runqueue locked, task not running
1260 : */
1261 : struct numa_group __rcu *numa_group;
1262 :
1263 : /*
1264 : * numa_faults is an array split into four regions:
1265 : * faults_memory, faults_cpu, faults_memory_buffer, faults_cpu_buffer
1266 : * in this precise order.
1267 : *
1268 : * faults_memory: Exponential decaying average of faults on a per-node
1269 : * basis. Scheduling placement decisions are made based on these
1270 : * counts. The values remain static for the duration of a PTE scan.
1271 : * faults_cpu: Track the nodes the process was running on when a NUMA
1272 : * hinting fault was incurred.
1273 : * faults_memory_buffer and faults_cpu_buffer: Record faults per node
1274 : * during the current scan window. When the scan completes, the counts
1275 : * in faults_memory and faults_cpu decay and these values are copied.
1276 : */
1277 : unsigned long *numa_faults;
1278 : unsigned long total_numa_faults;
1279 :
1280 : /*
1281 : * numa_faults_locality tracks if faults recorded during the last
1282 : * scan window were remote/local or failed to migrate. The task scan
1283 : * period is adapted based on the locality of the faults with different
1284 : * weights depending on whether they were shared or private faults
1285 : */
1286 : unsigned long numa_faults_locality[3];
1287 :
1288 : unsigned long numa_pages_migrated;
1289 : #endif /* CONFIG_NUMA_BALANCING */
1290 :
1291 : #ifdef CONFIG_RSEQ
1292 : struct rseq __user *rseq;
1293 : u32 rseq_sig;
1294 : /*
1295 : * RmW on rseq_event_mask must be performed atomically
1296 : * with respect to preemption.
1297 : */
1298 : unsigned long rseq_event_mask;
1299 : #endif
1300 :
1301 : struct tlbflush_unmap_batch tlb_ubc;
1302 :
1303 : union {
1304 : refcount_t rcu_users;
1305 : struct rcu_head rcu;
1306 : };
1307 :
1308 : /* Cache last used pipe for splice(): */
1309 : struct pipe_inode_info *splice_pipe;
1310 :
1311 : struct page_frag task_frag;
1312 :
1313 : #ifdef CONFIG_TASK_DELAY_ACCT
1314 : struct task_delay_info *delays;
1315 : #endif
1316 :
1317 : #ifdef CONFIG_FAULT_INJECTION
1318 : int make_it_fail;
1319 : unsigned int fail_nth;
1320 : #endif
1321 : /*
1322 : * When (nr_dirtied >= nr_dirtied_pause), it's time to call
1323 : * balance_dirty_pages() for a dirty throttling pause:
1324 : */
1325 : int nr_dirtied;
1326 : int nr_dirtied_pause;
1327 : /* Start of a write-and-pause period: */
1328 : unsigned long dirty_paused_when;
1329 :
1330 : #ifdef CONFIG_LATENCYTOP
1331 : int latency_record_count;
1332 : struct latency_record latency_record[LT_SAVECOUNT];
1333 : #endif
1334 : /*
1335 : * Time slack values; these are used to round up poll() and
1336 : * select() etc timeout values. These are in nanoseconds.
1337 : */
1338 : u64 timer_slack_ns;
1339 : u64 default_timer_slack_ns;
1340 :
1341 : #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
1342 : unsigned int kasan_depth;
1343 : #endif
1344 :
1345 : #ifdef CONFIG_KCSAN
1346 : struct kcsan_ctx kcsan_ctx;
1347 : #ifdef CONFIG_TRACE_IRQFLAGS
1348 : struct irqtrace_events kcsan_save_irqtrace;
1349 : #endif
1350 : #ifdef CONFIG_KCSAN_WEAK_MEMORY
1351 : int kcsan_stack_depth;
1352 : #endif
1353 : #endif
1354 :
1355 : #if IS_ENABLED(CONFIG_KUNIT)
1356 : struct kunit *kunit_test;
1357 : #endif
1358 :
1359 : #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1360 : /* Index of current stored address in ret_stack: */
1361 : int curr_ret_stack;
1362 : int curr_ret_depth;
1363 :
1364 : /* Stack of return addresses for return function tracing: */
1365 : struct ftrace_ret_stack *ret_stack;
1366 :
1367 : /* Timestamp for last schedule: */
1368 : unsigned long long ftrace_timestamp;
1369 :
1370 : /*
1371 : * Number of functions that haven't been traced
1372 : * because of depth overrun:
1373 : */
1374 : atomic_t trace_overrun;
1375 :
1376 : /* Pause tracing: */
1377 : atomic_t tracing_graph_pause;
1378 : #endif
1379 :
1380 : #ifdef CONFIG_TRACING
1381 : /* State flags for use by tracers: */
1382 : unsigned long trace;
1383 :
1384 : /* Bitmask and counter of trace recursion: */
1385 : unsigned long trace_recursion;
1386 : #endif /* CONFIG_TRACING */
1387 :
1388 : #ifdef CONFIG_KCOV
1389 : /* See kernel/kcov.c for more details. */
1390 :
1391 : /* Coverage collection mode enabled for this task (0 if disabled): */
1392 : unsigned int kcov_mode;
1393 :
1394 : /* Size of the kcov_area: */
1395 : unsigned int kcov_size;
1396 :
1397 : /* Buffer for coverage collection: */
1398 : void *kcov_area;
1399 :
1400 : /* KCOV descriptor wired with this task or NULL: */
1401 : struct kcov *kcov;
1402 :
1403 : /* KCOV common handle for remote coverage collection: */
1404 : u64 kcov_handle;
1405 :
1406 : /* KCOV sequence number: */
1407 : int kcov_sequence;
1408 :
1409 : /* Collect coverage from softirq context: */
1410 : unsigned int kcov_softirq;
1411 : #endif
1412 :
1413 : #ifdef CONFIG_MEMCG
1414 : struct mem_cgroup *memcg_in_oom;
1415 : gfp_t memcg_oom_gfp_mask;
1416 : int memcg_oom_order;
1417 :
1418 : /* Number of pages to reclaim on returning to userland: */
1419 : unsigned int memcg_nr_pages_over_high;
1420 :
1421 : /* Used by memcontrol for targeted memcg charge: */
1422 : struct mem_cgroup *active_memcg;
1423 : #endif
1424 :
1425 : #ifdef CONFIG_BLK_CGROUP
1426 : struct request_queue *throttle_queue;
1427 : #endif
1428 :
1429 : #ifdef CONFIG_UPROBES
1430 : struct uprobe_task *utask;
1431 : #endif
1432 : #if defined(CONFIG_BCACHE) || defined(CONFIG_BCACHE_MODULE)
1433 : unsigned int sequential_io;
1434 : unsigned int sequential_io_avg;
1435 : #endif
1436 : struct kmap_ctrl kmap_ctrl;
1437 : #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
1438 : unsigned long task_state_change;
1439 : # ifdef CONFIG_PREEMPT_RT
1440 : unsigned long saved_state_change;
1441 : # endif
1442 : #endif
1443 : int pagefault_disabled;
1444 : #ifdef CONFIG_MMU
1445 : struct task_struct *oom_reaper_list;
1446 : struct timer_list oom_reaper_timer;
1447 : #endif
1448 : #ifdef CONFIG_VMAP_STACK
1449 : struct vm_struct *stack_vm_area;
1450 : #endif
1451 : #ifdef CONFIG_THREAD_INFO_IN_TASK
1452 : /* A live task holds one reference: */
1453 : refcount_t stack_refcount;
1454 : #endif
1455 : #ifdef CONFIG_LIVEPATCH
1456 : int patch_state;
1457 : #endif
1458 : #ifdef CONFIG_SECURITY
1459 : /* Used by LSM modules for access restriction: */
1460 : void *security;
1461 : #endif
1462 : #ifdef CONFIG_BPF_SYSCALL
1463 : /* Used by BPF task local storage */
1464 : struct bpf_local_storage __rcu *bpf_storage;
1465 : /* Used for BPF run context */
1466 : struct bpf_run_ctx *bpf_ctx;
1467 : #endif
1468 :
1469 : #ifdef CONFIG_GCC_PLUGIN_STACKLEAK
1470 : unsigned long lowest_stack;
1471 : unsigned long prev_lowest_stack;
1472 : #endif
1473 :
1474 : #ifdef CONFIG_X86_MCE
1475 : void __user *mce_vaddr;
1476 : __u64 mce_kflags;
1477 : u64 mce_addr;
1478 : __u64 mce_ripv : 1,
1479 : mce_whole_page : 1,
1480 : __mce_reserved : 62;
1481 : struct callback_head mce_kill_me;
1482 : int mce_count;
1483 : #endif
1484 :
1485 : #ifdef CONFIG_KRETPROBES
1486 : struct llist_head kretprobe_instances;
1487 : #endif
1488 : #ifdef CONFIG_RETHOOK
1489 : struct llist_head rethooks;
1490 : #endif
1491 :
1492 : #ifdef CONFIG_ARCH_HAS_PARANOID_L1D_FLUSH
1493 : /*
1494 : * If L1D flush is supported on mm context switch
1495 : * then we use this callback head to queue kill work
1496 : * to kill tasks that are not running on SMT disabled
1497 : * cores
1498 : */
1499 : struct callback_head l1d_flush_kill;
1500 : #endif
1501 :
1502 : /*
1503 : * New fields for task_struct should be added above here, so that
1504 : * they are included in the randomized portion of task_struct.
1505 : */
1506 : randomized_struct_fields_end
1507 :
1508 : /* CPU-specific state of this task: */
1509 : struct thread_struct thread;
1510 :
1511 : /*
1512 : * WARNING: on x86, 'thread_struct' contains a variable-sized
1513 : * structure. It *MUST* be at the end of 'task_struct'.
1514 : *
1515 : * Do not put anything below here!
1516 : */
1517 : };
1518 :
1519 : static inline struct pid *task_pid(struct task_struct *task)
1520 : {
1521 : return task->thread_pid;
1522 : }
1523 :
1524 : /*
1525 : * the helpers to get the task's different pids as they are seen
1526 : * from various namespaces
1527 : *
1528 : * task_xid_nr() : global id, i.e. the id seen from the init namespace;
1529 : * task_xid_vnr() : virtual id, i.e. the id seen from the pid namespace of
1530 : * current.
1531 : * task_xid_nr_ns() : id seen from the ns specified;
1532 : *
1533 : * see also pid_nr() etc in include/linux/pid.h
1534 : */
1535 : pid_t __task_pid_nr_ns(struct task_struct *task, enum pid_type type, struct pid_namespace *ns);
1536 :
1537 : static inline pid_t task_pid_nr(struct task_struct *tsk)
1538 : {
1539 : return tsk->pid;
1540 : }
1541 :
1542 : static inline pid_t task_pid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1543 : {
1544 93 : return __task_pid_nr_ns(tsk, PIDTYPE_PID, ns);
1545 : }
1546 :
1547 : static inline pid_t task_pid_vnr(struct task_struct *tsk)
1548 : {
1549 0 : return __task_pid_nr_ns(tsk, PIDTYPE_PID, NULL);
1550 : }
1551 :
1552 :
1553 : static inline pid_t task_tgid_nr(struct task_struct *tsk)
1554 : {
1555 : return tsk->tgid;
1556 : }
1557 :
1558 : /**
1559 : * pid_alive - check that a task structure is not stale
1560 : * @p: Task structure to be checked.
1561 : *
1562 : * Test if a process is not yet dead (at most zombie state)
1563 : * If pid_alive fails, then pointers within the task structure
1564 : * can be stale and must not be dereferenced.
1565 : *
1566 : * Return: 1 if the process is alive. 0 otherwise.
1567 : */
1568 : static inline int pid_alive(const struct task_struct *p)
1569 : {
1570 : return p->thread_pid != NULL;
1571 : }
1572 :
1573 : static inline pid_t task_pgrp_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1574 : {
1575 0 : return __task_pid_nr_ns(tsk, PIDTYPE_PGID, ns);
1576 : }
1577 :
1578 : static inline pid_t task_pgrp_vnr(struct task_struct *tsk)
1579 : {
1580 0 : return __task_pid_nr_ns(tsk, PIDTYPE_PGID, NULL);
1581 : }
1582 :
1583 :
1584 : static inline pid_t task_session_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1585 : {
1586 0 : return __task_pid_nr_ns(tsk, PIDTYPE_SID, ns);
1587 : }
1588 :
1589 : static inline pid_t task_session_vnr(struct task_struct *tsk)
1590 : {
1591 0 : return __task_pid_nr_ns(tsk, PIDTYPE_SID, NULL);
1592 : }
1593 :
1594 : static inline pid_t task_tgid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1595 : {
1596 0 : return __task_pid_nr_ns(tsk, PIDTYPE_TGID, ns);
1597 : }
1598 :
1599 : static inline pid_t task_tgid_vnr(struct task_struct *tsk)
1600 : {
1601 0 : return __task_pid_nr_ns(tsk, PIDTYPE_TGID, NULL);
1602 : }
1603 :
1604 : static inline pid_t task_ppid_nr_ns(const struct task_struct *tsk, struct pid_namespace *ns)
1605 : {
1606 : pid_t pid = 0;
1607 :
1608 : rcu_read_lock();
1609 : if (pid_alive(tsk))
1610 : pid = task_tgid_nr_ns(rcu_dereference(tsk->real_parent), ns);
1611 : rcu_read_unlock();
1612 :
1613 : return pid;
1614 : }
1615 :
1616 : static inline pid_t task_ppid_nr(const struct task_struct *tsk)
1617 : {
1618 : return task_ppid_nr_ns(tsk, &init_pid_ns);
1619 : }
1620 :
1621 : /* Obsolete, do not use: */
1622 : static inline pid_t task_pgrp_nr(struct task_struct *tsk)
1623 : {
1624 : return task_pgrp_nr_ns(tsk, &init_pid_ns);
1625 : }
1626 :
1627 : #define TASK_REPORT_IDLE (TASK_REPORT + 1)
1628 : #define TASK_REPORT_MAX (TASK_REPORT_IDLE << 1)
1629 :
1630 : static inline unsigned int __task_state_index(unsigned int tsk_state,
1631 : unsigned int tsk_exit_state)
1632 : {
1633 0 : unsigned int state = (tsk_state | tsk_exit_state) & TASK_REPORT;
1634 :
1635 : BUILD_BUG_ON_NOT_POWER_OF_2(TASK_REPORT_MAX);
1636 :
1637 0 : if (tsk_state == TASK_IDLE)
1638 0 : state = TASK_REPORT_IDLE;
1639 :
1640 : /*
1641 : * We're lying here, but rather than expose a completely new task state
1642 : * to userspace, we can make this appear as if the task has gone through
1643 : * a regular rt_mutex_lock() call.
1644 : */
1645 0 : if (tsk_state == TASK_RTLOCK_WAIT)
1646 0 : state = TASK_UNINTERRUPTIBLE;
1647 :
1648 0 : return fls(state);
1649 : }
1650 :
1651 : static inline unsigned int task_state_index(struct task_struct *tsk)
1652 : {
1653 0 : return __task_state_index(READ_ONCE(tsk->__state), tsk->exit_state);
1654 : }
1655 :
1656 : static inline char task_index_to_char(unsigned int state)
1657 : {
1658 : static const char state_char[] = "RSDTtXZPI";
1659 :
1660 : BUILD_BUG_ON(1 + ilog2(TASK_REPORT_MAX) != sizeof(state_char) - 1);
1661 :
1662 0 : return state_char[state];
1663 : }
1664 :
1665 : static inline char task_state_to_char(struct task_struct *tsk)
1666 : {
1667 0 : return task_index_to_char(task_state_index(tsk));
1668 : }
1669 :
1670 : /**
1671 : * is_global_init - check if a task structure is init. Since init
1672 : * is free to have sub-threads we need to check tgid.
1673 : * @tsk: Task structure to be checked.
1674 : *
1675 : * Check if a task structure is the first user space task the kernel created.
1676 : *
1677 : * Return: 1 if the task structure is init. 0 otherwise.
1678 : */
1679 : static inline int is_global_init(struct task_struct *tsk)
1680 : {
1681 93 : return task_tgid_nr(tsk) == 1;
1682 : }
1683 :
1684 : extern struct pid *cad_pid;
1685 :
1686 : /*
1687 : * Per process flags
1688 : */
1689 : #define PF_VCPU 0x00000001 /* I'm a virtual CPU */
1690 : #define PF_IDLE 0x00000002 /* I am an IDLE thread */
1691 : #define PF_EXITING 0x00000004 /* Getting shut down */
1692 : #define PF_POSTCOREDUMP 0x00000008 /* Coredumps should ignore this task */
1693 : #define PF_IO_WORKER 0x00000010 /* Task is an IO worker */
1694 : #define PF_WQ_WORKER 0x00000020 /* I'm a workqueue worker */
1695 : #define PF_FORKNOEXEC 0x00000040 /* Forked but didn't exec */
1696 : #define PF_MCE_PROCESS 0x00000080 /* Process policy on mce errors */
1697 : #define PF_SUPERPRIV 0x00000100 /* Used super-user privileges */
1698 : #define PF_DUMPCORE 0x00000200 /* Dumped core */
1699 : #define PF_SIGNALED 0x00000400 /* Killed by a signal */
1700 : #define PF_MEMALLOC 0x00000800 /* Allocating memory */
1701 : #define PF_NPROC_EXCEEDED 0x00001000 /* set_user() noticed that RLIMIT_NPROC was exceeded */
1702 : #define PF_USED_MATH 0x00002000 /* If unset the fpu must be initialized before use */
1703 : #define PF_NOFREEZE 0x00008000 /* This thread should not be frozen */
1704 : #define PF_FROZEN 0x00010000 /* Frozen for system suspend */
1705 : #define PF_KSWAPD 0x00020000 /* I am kswapd */
1706 : #define PF_MEMALLOC_NOFS 0x00040000 /* All allocation requests will inherit GFP_NOFS */
1707 : #define PF_MEMALLOC_NOIO 0x00080000 /* All allocation requests will inherit GFP_NOIO */
1708 : #define PF_LOCAL_THROTTLE 0x00100000 /* Throttle writes only against the bdi I write to,
1709 : * I am cleaning dirty pages from some other bdi. */
1710 : #define PF_KTHREAD 0x00200000 /* I am a kernel thread */
1711 : #define PF_RANDOMIZE 0x00400000 /* Randomize virtual address space */
1712 : #define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_mask */
1713 : #define PF_MCE_EARLY 0x08000000 /* Early kill for mce process policy */
1714 : #define PF_MEMALLOC_PIN 0x10000000 /* Allocation context constrained to zones which allow long term pinning. */
1715 : #define PF_FREEZER_SKIP 0x40000000 /* Freezer should not count it as freezable */
1716 : #define PF_SUSPEND_TASK 0x80000000 /* This thread called freeze_processes() and should not be frozen */
1717 :
1718 : /*
1719 : * Only the _current_ task can read/write to tsk->flags, but other
1720 : * tasks can access tsk->flags in readonly mode for example
1721 : * with tsk_used_math (like during threaded core dumping).
1722 : * There is however an exception to this rule during ptrace
1723 : * or during fork: the ptracer task is allowed to write to the
1724 : * child->flags of its traced child (same goes for fork, the parent
1725 : * can write to the child->flags), because we're guaranteed the
1726 : * child is not running and in turn not changing child->flags
1727 : * at the same time the parent does it.
1728 : */
1729 : #define clear_stopped_child_used_math(child) do { (child)->flags &= ~PF_USED_MATH; } while (0)
1730 : #define set_stopped_child_used_math(child) do { (child)->flags |= PF_USED_MATH; } while (0)
1731 : #define clear_used_math() clear_stopped_child_used_math(current)
1732 : #define set_used_math() set_stopped_child_used_math(current)
1733 :
1734 : #define conditional_stopped_child_used_math(condition, child) \
1735 : do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= (condition) ? PF_USED_MATH : 0; } while (0)
1736 :
1737 : #define conditional_used_math(condition) conditional_stopped_child_used_math(condition, current)
1738 :
1739 : #define copy_to_stopped_child_used_math(child) \
1740 : do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= current->flags & PF_USED_MATH; } while (0)
1741 :
1742 : /* NOTE: this will return 0 or PF_USED_MATH, it will never return 1 */
1743 : #define tsk_used_math(p) ((p)->flags & PF_USED_MATH)
1744 : #define used_math() tsk_used_math(current)
1745 :
1746 : static __always_inline bool is_percpu_thread(void)
1747 : {
1748 : #ifdef CONFIG_SMP
1749 : return (current->flags & PF_NO_SETAFFINITY) &&
1750 : (current->nr_cpus_allowed == 1);
1751 : #else
1752 : return true;
1753 : #endif
1754 : }
1755 :
1756 : /* Per-process atomic flags. */
1757 : #define PFA_NO_NEW_PRIVS 0 /* May not gain new privileges. */
1758 : #define PFA_SPREAD_PAGE 1 /* Spread page cache over cpuset */
1759 : #define PFA_SPREAD_SLAB 2 /* Spread some slab caches over cpuset */
1760 : #define PFA_SPEC_SSB_DISABLE 3 /* Speculative Store Bypass disabled */
1761 : #define PFA_SPEC_SSB_FORCE_DISABLE 4 /* Speculative Store Bypass force disabled*/
1762 : #define PFA_SPEC_IB_DISABLE 5 /* Indirect branch speculation restricted */
1763 : #define PFA_SPEC_IB_FORCE_DISABLE 6 /* Indirect branch speculation permanently restricted */
1764 : #define PFA_SPEC_SSB_NOEXEC 7 /* Speculative Store Bypass clear on execve() */
1765 :
1766 : #define TASK_PFA_TEST(name, func) \
1767 : static inline bool task_##func(struct task_struct *p) \
1768 : { return test_bit(PFA_##name, &p->atomic_flags); }
1769 :
1770 : #define TASK_PFA_SET(name, func) \
1771 : static inline void task_set_##func(struct task_struct *p) \
1772 : { set_bit(PFA_##name, &p->atomic_flags); }
1773 :
1774 : #define TASK_PFA_CLEAR(name, func) \
1775 : static inline void task_clear_##func(struct task_struct *p) \
1776 : { clear_bit(PFA_##name, &p->atomic_flags); }
1777 :
1778 214 : TASK_PFA_TEST(NO_NEW_PRIVS, no_new_privs)
1779 0 : TASK_PFA_SET(NO_NEW_PRIVS, no_new_privs)
1780 :
1781 : TASK_PFA_TEST(SPREAD_PAGE, spread_page)
1782 : TASK_PFA_SET(SPREAD_PAGE, spread_page)
1783 : TASK_PFA_CLEAR(SPREAD_PAGE, spread_page)
1784 :
1785 : TASK_PFA_TEST(SPREAD_SLAB, spread_slab)
1786 : TASK_PFA_SET(SPREAD_SLAB, spread_slab)
1787 : TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab)
1788 :
1789 : TASK_PFA_TEST(SPEC_SSB_DISABLE, spec_ssb_disable)
1790 : TASK_PFA_SET(SPEC_SSB_DISABLE, spec_ssb_disable)
1791 : TASK_PFA_CLEAR(SPEC_SSB_DISABLE, spec_ssb_disable)
1792 :
1793 : TASK_PFA_TEST(SPEC_SSB_NOEXEC, spec_ssb_noexec)
1794 : TASK_PFA_SET(SPEC_SSB_NOEXEC, spec_ssb_noexec)
1795 : TASK_PFA_CLEAR(SPEC_SSB_NOEXEC, spec_ssb_noexec)
1796 :
1797 : TASK_PFA_TEST(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)
1798 : TASK_PFA_SET(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)
1799 :
1800 : TASK_PFA_TEST(SPEC_IB_DISABLE, spec_ib_disable)
1801 : TASK_PFA_SET(SPEC_IB_DISABLE, spec_ib_disable)
1802 : TASK_PFA_CLEAR(SPEC_IB_DISABLE, spec_ib_disable)
1803 :
1804 : TASK_PFA_TEST(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable)
1805 : TASK_PFA_SET(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable)
1806 :
1807 : static inline void
1808 : current_restore_flags(unsigned long orig_flags, unsigned long flags)
1809 : {
1810 93 : current->flags &= ~flags;
1811 93 : current->flags |= orig_flags & flags;
1812 : }
1813 :
1814 : extern int cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial);
1815 : extern int task_can_attach(struct task_struct *p, const struct cpumask *cs_cpus_allowed);
1816 : #ifdef CONFIG_SMP
1817 : extern void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask);
1818 : extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask);
1819 : extern int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src, int node);
1820 : extern void release_user_cpus_ptr(struct task_struct *p);
1821 : extern int dl_task_check_affinity(struct task_struct *p, const struct cpumask *mask);
1822 : extern void force_compatible_cpus_allowed_ptr(struct task_struct *p);
1823 : extern void relax_compatible_cpus_allowed_ptr(struct task_struct *p);
1824 : #else
1825 : static inline void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
1826 : {
1827 : }
1828 : static inline int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
1829 : {
1830 110 : if (!cpumask_test_cpu(0, new_mask))
1831 : return -EINVAL;
1832 : return 0;
1833 : }
1834 : static inline int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src, int node)
1835 : {
1836 : if (src->user_cpus_ptr)
1837 : return -EINVAL;
1838 : return 0;
1839 : }
1840 : static inline void release_user_cpus_ptr(struct task_struct *p)
1841 : {
1842 92 : WARN_ON(p->user_cpus_ptr);
1843 : }
1844 :
1845 : static inline int dl_task_check_affinity(struct task_struct *p, const struct cpumask *mask)
1846 : {
1847 : return 0;
1848 : }
1849 : #endif
1850 :
1851 : extern int yield_to(struct task_struct *p, bool preempt);
1852 : extern void set_user_nice(struct task_struct *p, long nice);
1853 : extern int task_prio(const struct task_struct *p);
1854 :
1855 : /**
1856 : * task_nice - return the nice value of a given task.
1857 : * @p: the task in question.
1858 : *
1859 : * Return: The nice value [ -20 ... 0 ... 19 ].
1860 : */
1861 : static inline int task_nice(const struct task_struct *p)
1862 : {
1863 125 : return PRIO_TO_NICE((p)->static_prio);
1864 : }
1865 :
1866 : extern int can_nice(const struct task_struct *p, const int nice);
1867 : extern int task_curr(const struct task_struct *p);
1868 : extern int idle_cpu(int cpu);
1869 : extern int available_idle_cpu(int cpu);
1870 : extern int sched_setscheduler(struct task_struct *, int, const struct sched_param *);
1871 : extern int sched_setscheduler_nocheck(struct task_struct *, int, const struct sched_param *);
1872 : extern void sched_set_fifo(struct task_struct *p);
1873 : extern void sched_set_fifo_low(struct task_struct *p);
1874 : extern void sched_set_normal(struct task_struct *p, int nice);
1875 : extern int sched_setattr(struct task_struct *, const struct sched_attr *);
1876 : extern int sched_setattr_nocheck(struct task_struct *, const struct sched_attr *);
1877 : extern struct task_struct *idle_task(int cpu);
1878 :
1879 : /**
1880 : * is_idle_task - is the specified task an idle task?
1881 : * @p: the task in question.
1882 : *
1883 : * Return: 1 if @p is an idle task. 0 otherwise.
1884 : */
1885 : static __always_inline bool is_idle_task(const struct task_struct *p)
1886 : {
1887 473 : return !!(p->flags & PF_IDLE);
1888 : }
1889 :
1890 : extern struct task_struct *curr_task(int cpu);
1891 : extern void ia64_set_curr_task(int cpu, struct task_struct *p);
1892 :
1893 : void yield(void);
1894 :
1895 : union thread_union {
1896 : #ifndef CONFIG_ARCH_TASK_STRUCT_ON_STACK
1897 : struct task_struct task;
1898 : #endif
1899 : #ifndef CONFIG_THREAD_INFO_IN_TASK
1900 : struct thread_info thread_info;
1901 : #endif
1902 : unsigned long stack[THREAD_SIZE/sizeof(long)];
1903 : };
1904 :
1905 : #ifndef CONFIG_THREAD_INFO_IN_TASK
1906 : extern struct thread_info init_thread_info;
1907 : #endif
1908 :
1909 : extern unsigned long init_stack[THREAD_SIZE / sizeof(unsigned long)];
1910 :
1911 : #ifdef CONFIG_THREAD_INFO_IN_TASK
1912 : # define task_thread_info(task) (&(task)->thread_info)
1913 : #elif !defined(__HAVE_THREAD_FUNCTIONS)
1914 : # define task_thread_info(task) ((struct thread_info *)(task)->stack)
1915 : #endif
1916 :
1917 : /*
1918 : * find a task by one of its numerical ids
1919 : *
1920 : * find_task_by_pid_ns():
1921 : * finds a task by its pid in the specified namespace
1922 : * find_task_by_vpid():
1923 : * finds a task by its virtual pid
1924 : *
1925 : * see also find_vpid() etc in include/linux/pid.h
1926 : */
1927 :
1928 : extern struct task_struct *find_task_by_vpid(pid_t nr);
1929 : extern struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns);
1930 :
1931 : /*
1932 : * find a task by its virtual pid and get the task struct
1933 : */
1934 : extern struct task_struct *find_get_task_by_vpid(pid_t nr);
1935 :
1936 : extern int wake_up_state(struct task_struct *tsk, unsigned int state);
1937 : extern int wake_up_process(struct task_struct *tsk);
1938 : extern void wake_up_new_task(struct task_struct *tsk);
1939 :
1940 : #ifdef CONFIG_SMP
1941 : extern void kick_process(struct task_struct *tsk);
1942 : #else
1943 : static inline void kick_process(struct task_struct *tsk) { }
1944 : #endif
1945 :
1946 : extern void __set_task_comm(struct task_struct *tsk, const char *from, bool exec);
1947 :
1948 : static inline void set_task_comm(struct task_struct *tsk, const char *from)
1949 : {
1950 106 : __set_task_comm(tsk, from, false);
1951 : }
1952 :
1953 : extern char *__get_task_comm(char *to, size_t len, struct task_struct *tsk);
1954 : #define get_task_comm(buf, tsk) ({ \
1955 : BUILD_BUG_ON(sizeof(buf) != TASK_COMM_LEN); \
1956 : __get_task_comm(buf, sizeof(buf), tsk); \
1957 : })
1958 :
1959 : #ifdef CONFIG_SMP
1960 : static __always_inline void scheduler_ipi(void)
1961 : {
1962 : /*
1963 : * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
1964 : * TIF_NEED_RESCHED remotely (for the first time) will also send
1965 : * this IPI.
1966 : */
1967 : preempt_fold_need_resched();
1968 : }
1969 : extern unsigned long wait_task_inactive(struct task_struct *, unsigned int match_state);
1970 : #else
1971 : static inline void scheduler_ipi(void) { }
1972 : static inline unsigned long wait_task_inactive(struct task_struct *p, unsigned int match_state)
1973 : {
1974 : return 1;
1975 : }
1976 : #endif
1977 :
1978 : /*
1979 : * Set thread flags in other task's structures.
1980 : * See asm/thread_info.h for TIF_xxxx flags available:
1981 : */
1982 : static inline void set_tsk_thread_flag(struct task_struct *tsk, int flag)
1983 : {
1984 624 : set_ti_thread_flag(task_thread_info(tsk), flag);
1985 : }
1986 :
1987 : static inline void clear_tsk_thread_flag(struct task_struct *tsk, int flag)
1988 : {
1989 1638 : clear_ti_thread_flag(task_thread_info(tsk), flag);
1990 : }
1991 :
1992 : static inline void update_tsk_thread_flag(struct task_struct *tsk, int flag,
1993 : bool value)
1994 : {
1995 : update_ti_thread_flag(task_thread_info(tsk), flag, value);
1996 : }
1997 :
1998 : static inline int test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag)
1999 : {
2000 0 : return test_and_set_ti_thread_flag(task_thread_info(tsk), flag);
2001 : }
2002 :
2003 : static inline int test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag)
2004 : {
2005 : return test_and_clear_ti_thread_flag(task_thread_info(tsk), flag);
2006 : }
2007 :
2008 : static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag)
2009 : {
2010 4958 : return test_ti_thread_flag(task_thread_info(tsk), flag);
2011 : }
2012 :
2013 : static inline void set_tsk_need_resched(struct task_struct *tsk)
2014 : {
2015 410 : set_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
2016 : }
2017 :
2018 : static inline void clear_tsk_need_resched(struct task_struct *tsk)
2019 : {
2020 1450 : clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
2021 : }
2022 :
2023 : static inline int test_tsk_need_resched(struct task_struct *tsk)
2024 : {
2025 2874 : return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED));
2026 : }
2027 :
2028 : /*
2029 : * cond_resched() and cond_resched_lock(): latency reduction via
2030 : * explicit rescheduling in places that are safe. The return
2031 : * value indicates whether a reschedule was done in fact.
2032 : * cond_resched_lock() will drop the spinlock before scheduling,
2033 : */
2034 : #if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC)
2035 : extern int __cond_resched(void);
2036 :
2037 : #if defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)
2038 :
2039 : DECLARE_STATIC_CALL(cond_resched, __cond_resched);
2040 :
2041 : static __always_inline int _cond_resched(void)
2042 : {
2043 : return static_call_mod(cond_resched)();
2044 : }
2045 :
2046 : #elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
2047 : extern int dynamic_cond_resched(void);
2048 :
2049 : static __always_inline int _cond_resched(void)
2050 : {
2051 : return dynamic_cond_resched();
2052 : }
2053 :
2054 : #else
2055 :
2056 : static inline int _cond_resched(void)
2057 : {
2058 739 : return __cond_resched();
2059 : }
2060 :
2061 : #endif /* CONFIG_PREEMPT_DYNAMIC */
2062 :
2063 : #else
2064 :
2065 : static inline int _cond_resched(void) { return 0; }
2066 :
2067 : #endif /* !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) */
2068 :
2069 : #define cond_resched() ({ \
2070 : __might_resched(__FILE__, __LINE__, 0); \
2071 : _cond_resched(); \
2072 : })
2073 :
2074 : extern int __cond_resched_lock(spinlock_t *lock);
2075 : extern int __cond_resched_rwlock_read(rwlock_t *lock);
2076 : extern int __cond_resched_rwlock_write(rwlock_t *lock);
2077 :
2078 : #define MIGHT_RESCHED_RCU_SHIFT 8
2079 : #define MIGHT_RESCHED_PREEMPT_MASK ((1U << MIGHT_RESCHED_RCU_SHIFT) - 1)
2080 :
2081 : #ifndef CONFIG_PREEMPT_RT
2082 : /*
2083 : * Non RT kernels have an elevated preempt count due to the held lock,
2084 : * but are not allowed to be inside a RCU read side critical section
2085 : */
2086 : # define PREEMPT_LOCK_RESCHED_OFFSETS PREEMPT_LOCK_OFFSET
2087 : #else
2088 : /*
2089 : * spin/rw_lock() on RT implies rcu_read_lock(). The might_sleep() check in
2090 : * cond_resched*lock() has to take that into account because it checks for
2091 : * preempt_count() and rcu_preempt_depth().
2092 : */
2093 : # define PREEMPT_LOCK_RESCHED_OFFSETS \
2094 : (PREEMPT_LOCK_OFFSET + (1U << MIGHT_RESCHED_RCU_SHIFT))
2095 : #endif
2096 :
2097 : #define cond_resched_lock(lock) ({ \
2098 : __might_resched(__FILE__, __LINE__, PREEMPT_LOCK_RESCHED_OFFSETS); \
2099 : __cond_resched_lock(lock); \
2100 : })
2101 :
2102 : #define cond_resched_rwlock_read(lock) ({ \
2103 : __might_resched(__FILE__, __LINE__, PREEMPT_LOCK_RESCHED_OFFSETS); \
2104 : __cond_resched_rwlock_read(lock); \
2105 : })
2106 :
2107 : #define cond_resched_rwlock_write(lock) ({ \
2108 : __might_resched(__FILE__, __LINE__, PREEMPT_LOCK_RESCHED_OFFSETS); \
2109 : __cond_resched_rwlock_write(lock); \
2110 : })
2111 :
2112 : static inline void cond_resched_rcu(void)
2113 : {
2114 : #if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU)
2115 : rcu_read_unlock();
2116 0 : cond_resched();
2117 : rcu_read_lock();
2118 : #endif
2119 : }
2120 :
2121 : /*
2122 : * Does a critical section need to be broken due to another
2123 : * task waiting?: (technically does not depend on CONFIG_PREEMPTION,
2124 : * but a general need for low latency)
2125 : */
2126 : static inline int spin_needbreak(spinlock_t *lock)
2127 : {
2128 : #ifdef CONFIG_PREEMPTION
2129 : return spin_is_contended(lock);
2130 : #else
2131 : return 0;
2132 : #endif
2133 : }
2134 :
2135 : /*
2136 : * Check if a rwlock is contended.
2137 : * Returns non-zero if there is another task waiting on the rwlock.
2138 : * Returns zero if the lock is not contended or the system / underlying
2139 : * rwlock implementation does not support contention detection.
2140 : * Technically does not depend on CONFIG_PREEMPTION, but a general need
2141 : * for low latency.
2142 : */
2143 : static inline int rwlock_needbreak(rwlock_t *lock)
2144 : {
2145 : #ifdef CONFIG_PREEMPTION
2146 : return rwlock_is_contended(lock);
2147 : #else
2148 : return 0;
2149 : #endif
2150 : }
2151 :
2152 : static __always_inline bool need_resched(void)
2153 : {
2154 1206 : return unlikely(tif_need_resched());
2155 : }
2156 :
2157 : /*
2158 : * Wrappers for p->thread_info->cpu access. No-op on UP.
2159 : */
2160 : #ifdef CONFIG_SMP
2161 :
2162 : static inline unsigned int task_cpu(const struct task_struct *p)
2163 : {
2164 : return READ_ONCE(task_thread_info(p)->cpu);
2165 : }
2166 :
2167 : extern void set_task_cpu(struct task_struct *p, unsigned int cpu);
2168 :
2169 : #else
2170 :
2171 : static inline unsigned int task_cpu(const struct task_struct *p)
2172 : {
2173 : return 0;
2174 : }
2175 :
2176 : static inline void set_task_cpu(struct task_struct *p, unsigned int cpu)
2177 : {
2178 : }
2179 :
2180 : #endif /* CONFIG_SMP */
2181 :
2182 : extern bool sched_task_on_rq(struct task_struct *p);
2183 : extern unsigned long get_wchan(struct task_struct *p);
2184 :
2185 : /*
2186 : * In order to reduce various lock holder preemption latencies provide an
2187 : * interface to see if a vCPU is currently running or not.
2188 : *
2189 : * This allows us to terminate optimistic spin loops and block, analogous to
2190 : * the native optimistic spin heuristic of testing if the lock owner task is
2191 : * running or not.
2192 : */
2193 : #ifndef vcpu_is_preempted
2194 : static inline bool vcpu_is_preempted(int cpu)
2195 : {
2196 : return false;
2197 : }
2198 : #endif
2199 :
2200 : extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
2201 : extern long sched_getaffinity(pid_t pid, struct cpumask *mask);
2202 :
2203 : #ifndef TASK_SIZE_OF
2204 : #define TASK_SIZE_OF(tsk) TASK_SIZE
2205 : #endif
2206 :
2207 : #ifdef CONFIG_SMP
2208 : static inline bool owner_on_cpu(struct task_struct *owner)
2209 : {
2210 : /*
2211 : * As lock holder preemption issue, we both skip spinning if
2212 : * task is not on cpu or its cpu is preempted
2213 : */
2214 : return READ_ONCE(owner->on_cpu) && !vcpu_is_preempted(task_cpu(owner));
2215 : }
2216 :
2217 : /* Returns effective CPU energy utilization, as seen by the scheduler */
2218 : unsigned long sched_cpu_util(int cpu, unsigned long max);
2219 : #endif /* CONFIG_SMP */
2220 :
2221 : #ifdef CONFIG_RSEQ
2222 :
2223 : /*
2224 : * Map the event mask on the user-space ABI enum rseq_cs_flags
2225 : * for direct mask checks.
2226 : */
2227 : enum rseq_event_mask_bits {
2228 : RSEQ_EVENT_PREEMPT_BIT = RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT,
2229 : RSEQ_EVENT_SIGNAL_BIT = RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT,
2230 : RSEQ_EVENT_MIGRATE_BIT = RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT,
2231 : };
2232 :
2233 : enum rseq_event_mask {
2234 : RSEQ_EVENT_PREEMPT = (1U << RSEQ_EVENT_PREEMPT_BIT),
2235 : RSEQ_EVENT_SIGNAL = (1U << RSEQ_EVENT_SIGNAL_BIT),
2236 : RSEQ_EVENT_MIGRATE = (1U << RSEQ_EVENT_MIGRATE_BIT),
2237 : };
2238 :
2239 : static inline void rseq_set_notify_resume(struct task_struct *t)
2240 : {
2241 : if (t->rseq)
2242 : set_tsk_thread_flag(t, TIF_NOTIFY_RESUME);
2243 : }
2244 :
2245 : void __rseq_handle_notify_resume(struct ksignal *sig, struct pt_regs *regs);
2246 :
2247 : static inline void rseq_handle_notify_resume(struct ksignal *ksig,
2248 : struct pt_regs *regs)
2249 : {
2250 : if (current->rseq)
2251 : __rseq_handle_notify_resume(ksig, regs);
2252 : }
2253 :
2254 : static inline void rseq_signal_deliver(struct ksignal *ksig,
2255 : struct pt_regs *regs)
2256 : {
2257 : preempt_disable();
2258 : __set_bit(RSEQ_EVENT_SIGNAL_BIT, ¤t->rseq_event_mask);
2259 : preempt_enable();
2260 : rseq_handle_notify_resume(ksig, regs);
2261 : }
2262 :
2263 : /* rseq_preempt() requires preemption to be disabled. */
2264 : static inline void rseq_preempt(struct task_struct *t)
2265 : {
2266 : __set_bit(RSEQ_EVENT_PREEMPT_BIT, &t->rseq_event_mask);
2267 : rseq_set_notify_resume(t);
2268 : }
2269 :
2270 : /* rseq_migrate() requires preemption to be disabled. */
2271 : static inline void rseq_migrate(struct task_struct *t)
2272 : {
2273 : __set_bit(RSEQ_EVENT_MIGRATE_BIT, &t->rseq_event_mask);
2274 : rseq_set_notify_resume(t);
2275 : }
2276 :
2277 : /*
2278 : * If parent process has a registered restartable sequences area, the
2279 : * child inherits. Unregister rseq for a clone with CLONE_VM set.
2280 : */
2281 : static inline void rseq_fork(struct task_struct *t, unsigned long clone_flags)
2282 : {
2283 : if (clone_flags & CLONE_VM) {
2284 : t->rseq = NULL;
2285 : t->rseq_sig = 0;
2286 : t->rseq_event_mask = 0;
2287 : } else {
2288 : t->rseq = current->rseq;
2289 : t->rseq_sig = current->rseq_sig;
2290 : t->rseq_event_mask = current->rseq_event_mask;
2291 : }
2292 : }
2293 :
2294 : static inline void rseq_execve(struct task_struct *t)
2295 : {
2296 : t->rseq = NULL;
2297 : t->rseq_sig = 0;
2298 : t->rseq_event_mask = 0;
2299 : }
2300 :
2301 : #else
2302 :
2303 : static inline void rseq_set_notify_resume(struct task_struct *t)
2304 : {
2305 : }
2306 : static inline void rseq_handle_notify_resume(struct ksignal *ksig,
2307 : struct pt_regs *regs)
2308 : {
2309 : }
2310 : static inline void rseq_signal_deliver(struct ksignal *ksig,
2311 : struct pt_regs *regs)
2312 : {
2313 : }
2314 : static inline void rseq_preempt(struct task_struct *t)
2315 : {
2316 : }
2317 : static inline void rseq_migrate(struct task_struct *t)
2318 : {
2319 : }
2320 : static inline void rseq_fork(struct task_struct *t, unsigned long clone_flags)
2321 : {
2322 : }
2323 : static inline void rseq_execve(struct task_struct *t)
2324 : {
2325 : }
2326 :
2327 : #endif
2328 :
2329 : #ifdef CONFIG_DEBUG_RSEQ
2330 :
2331 : void rseq_syscall(struct pt_regs *regs);
2332 :
2333 : #else
2334 :
2335 : static inline void rseq_syscall(struct pt_regs *regs)
2336 : {
2337 : }
2338 :
2339 : #endif
2340 :
2341 : const struct sched_avg *sched_trace_cfs_rq_avg(struct cfs_rq *cfs_rq);
2342 : char *sched_trace_cfs_rq_path(struct cfs_rq *cfs_rq, char *str, int len);
2343 : int sched_trace_cfs_rq_cpu(struct cfs_rq *cfs_rq);
2344 :
2345 : const struct sched_avg *sched_trace_rq_avg_rt(struct rq *rq);
2346 : const struct sched_avg *sched_trace_rq_avg_dl(struct rq *rq);
2347 : const struct sched_avg *sched_trace_rq_avg_irq(struct rq *rq);
2348 :
2349 : int sched_trace_rq_cpu(struct rq *rq);
2350 : int sched_trace_rq_cpu_capacity(struct rq *rq);
2351 : int sched_trace_rq_nr_running(struct rq *rq);
2352 :
2353 : const struct cpumask *sched_trace_rd_span(struct root_domain *rd);
2354 :
2355 : #ifdef CONFIG_SCHED_CORE
2356 : extern void sched_core_free(struct task_struct *tsk);
2357 : extern void sched_core_fork(struct task_struct *p);
2358 : extern int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type,
2359 : unsigned long uaddr);
2360 : #else
2361 : static inline void sched_core_free(struct task_struct *tsk) { }
2362 : static inline void sched_core_fork(struct task_struct *p) { }
2363 : #endif
2364 :
2365 : #endif
|
