Line data Source code
1 : /* CPU control.
2 : * (C) 2001, 2002, 2003, 2004 Rusty Russell
3 : *
4 : * This code is licenced under the GPL.
5 : */
6 : #include <linux/sched/mm.h>
7 : #include <linux/proc_fs.h>
8 : #include <linux/smp.h>
9 : #include <linux/init.h>
10 : #include <linux/notifier.h>
11 : #include <linux/sched/signal.h>
12 : #include <linux/sched/hotplug.h>
13 : #include <linux/sched/isolation.h>
14 : #include <linux/sched/task.h>
15 : #include <linux/sched/smt.h>
16 : #include <linux/unistd.h>
17 : #include <linux/cpu.h>
18 : #include <linux/oom.h>
19 : #include <linux/rcupdate.h>
20 : #include <linux/export.h>
21 : #include <linux/bug.h>
22 : #include <linux/kthread.h>
23 : #include <linux/stop_machine.h>
24 : #include <linux/mutex.h>
25 : #include <linux/gfp.h>
26 : #include <linux/suspend.h>
27 : #include <linux/lockdep.h>
28 : #include <linux/tick.h>
29 : #include <linux/irq.h>
30 : #include <linux/nmi.h>
31 : #include <linux/smpboot.h>
32 : #include <linux/relay.h>
33 : #include <linux/slab.h>
34 : #include <linux/scs.h>
35 : #include <linux/percpu-rwsem.h>
36 : #include <linux/cpuset.h>
37 : #include <linux/random.h>
38 :
39 : #include <trace/events/power.h>
40 : #define CREATE_TRACE_POINTS
41 : #include <trace/events/cpuhp.h>
42 :
43 : #include "smpboot.h"
44 :
45 : /**
46 : * struct cpuhp_cpu_state - Per cpu hotplug state storage
47 : * @state: The current cpu state
48 : * @target: The target state
49 : * @fail: Current CPU hotplug callback state
50 : * @thread: Pointer to the hotplug thread
51 : * @should_run: Thread should execute
52 : * @rollback: Perform a rollback
53 : * @single: Single callback invocation
54 : * @bringup: Single callback bringup or teardown selector
55 : * @cpu: CPU number
56 : * @node: Remote CPU node; for multi-instance, do a
57 : * single entry callback for install/remove
58 : * @last: For multi-instance rollback, remember how far we got
59 : * @cb_state: The state for a single callback (install/uninstall)
60 : * @result: Result of the operation
61 : * @done_up: Signal completion to the issuer of the task for cpu-up
62 : * @done_down: Signal completion to the issuer of the task for cpu-down
63 : */
64 : struct cpuhp_cpu_state {
65 : enum cpuhp_state state;
66 : enum cpuhp_state target;
67 : enum cpuhp_state fail;
68 : #ifdef CONFIG_SMP
69 : struct task_struct *thread;
70 : bool should_run;
71 : bool rollback;
72 : bool single;
73 : bool bringup;
74 : struct hlist_node *node;
75 : struct hlist_node *last;
76 : enum cpuhp_state cb_state;
77 : int result;
78 : struct completion done_up;
79 : struct completion done_down;
80 : #endif
81 : };
82 :
83 : static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {
84 : .fail = CPUHP_INVALID,
85 : };
86 :
87 : #ifdef CONFIG_SMP
88 : cpumask_t cpus_booted_once_mask;
89 : #endif
90 :
91 : #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
92 : static struct lockdep_map cpuhp_state_up_map =
93 : STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
94 : static struct lockdep_map cpuhp_state_down_map =
95 : STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
96 :
97 :
98 : static inline void cpuhp_lock_acquire(bool bringup)
99 : {
100 : lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
101 : }
102 :
103 : static inline void cpuhp_lock_release(bool bringup)
104 : {
105 : lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
106 : }
107 : #else
108 :
109 : static inline void cpuhp_lock_acquire(bool bringup) { }
110 : static inline void cpuhp_lock_release(bool bringup) { }
111 :
112 : #endif
113 :
114 : /**
115 : * struct cpuhp_step - Hotplug state machine step
116 : * @name: Name of the step
117 : * @startup: Startup function of the step
118 : * @teardown: Teardown function of the step
119 : * @cant_stop: Bringup/teardown can't be stopped at this step
120 : * @multi_instance: State has multiple instances which get added afterwards
121 : */
122 : struct cpuhp_step {
123 : const char *name;
124 : union {
125 : int (*single)(unsigned int cpu);
126 : int (*multi)(unsigned int cpu,
127 : struct hlist_node *node);
128 : } startup;
129 : union {
130 : int (*single)(unsigned int cpu);
131 : int (*multi)(unsigned int cpu,
132 : struct hlist_node *node);
133 : } teardown;
134 : /* private: */
135 : struct hlist_head list;
136 : /* public: */
137 : bool cant_stop;
138 : bool multi_instance;
139 : };
140 :
141 : static DEFINE_MUTEX(cpuhp_state_mutex);
142 : static struct cpuhp_step cpuhp_hp_states[];
143 :
144 : static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
145 : {
146 26 : return cpuhp_hp_states + state;
147 : }
148 :
149 : static bool cpuhp_step_empty(bool bringup, struct cpuhp_step *step)
150 : {
151 6 : return bringup ? !step->startup.single : !step->teardown.single;
152 : }
153 :
154 : /**
155 : * cpuhp_invoke_callback - Invoke the callbacks for a given state
156 : * @cpu: The cpu for which the callback should be invoked
157 : * @state: The state to do callbacks for
158 : * @bringup: True if the bringup callback should be invoked
159 : * @node: For multi-instance, do a single entry callback for install/remove
160 : * @lastp: For multi-instance rollback, remember how far we got
161 : *
162 : * Called from cpu hotplug and from the state register machinery.
163 : *
164 : * Return: %0 on success or a negative errno code
165 : */
166 3 : static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
167 : bool bringup, struct hlist_node *node,
168 : struct hlist_node **lastp)
169 : {
170 3 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
171 3 : struct cpuhp_step *step = cpuhp_get_step(state);
172 : int (*cbm)(unsigned int cpu, struct hlist_node *node);
173 : int (*cb)(unsigned int cpu);
174 : int ret, cnt;
175 :
176 3 : if (st->fail == state) {
177 0 : st->fail = CPUHP_INVALID;
178 0 : return -EAGAIN;
179 : }
180 :
181 3 : if (cpuhp_step_empty(bringup, step)) {
182 0 : WARN_ON_ONCE(1);
183 : return 0;
184 : }
185 :
186 3 : if (!step->multi_instance) {
187 3 : WARN_ON_ONCE(lastp && *lastp);
188 3 : cb = bringup ? step->startup.single : step->teardown.single;
189 :
190 3 : trace_cpuhp_enter(cpu, st->target, state, cb);
191 3 : ret = cb(cpu);
192 3 : trace_cpuhp_exit(cpu, st->state, state, ret);
193 3 : return ret;
194 : }
195 0 : cbm = bringup ? step->startup.multi : step->teardown.multi;
196 :
197 : /* Single invocation for instance add/remove */
198 0 : if (node) {
199 0 : WARN_ON_ONCE(lastp && *lastp);
200 0 : trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
201 0 : ret = cbm(cpu, node);
202 0 : trace_cpuhp_exit(cpu, st->state, state, ret);
203 0 : return ret;
204 : }
205 :
206 : /* State transition. Invoke on all instances */
207 0 : cnt = 0;
208 0 : hlist_for_each(node, &step->list) {
209 0 : if (lastp && node == *lastp)
210 : break;
211 :
212 0 : trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
213 0 : ret = cbm(cpu, node);
214 0 : trace_cpuhp_exit(cpu, st->state, state, ret);
215 0 : if (ret) {
216 0 : if (!lastp)
217 : goto err;
218 :
219 0 : *lastp = node;
220 0 : return ret;
221 : }
222 0 : cnt++;
223 : }
224 0 : if (lastp)
225 0 : *lastp = NULL;
226 : return 0;
227 : err:
228 : /* Rollback the instances if one failed */
229 0 : cbm = !bringup ? step->startup.multi : step->teardown.multi;
230 0 : if (!cbm)
231 : return ret;
232 :
233 0 : hlist_for_each(node, &step->list) {
234 0 : if (!cnt--)
235 : break;
236 :
237 0 : trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
238 0 : ret = cbm(cpu, node);
239 0 : trace_cpuhp_exit(cpu, st->state, state, ret);
240 : /*
241 : * Rollback must not fail,
242 : */
243 0 : WARN_ON_ONCE(ret);
244 : }
245 : return ret;
246 : }
247 :
248 : #ifdef CONFIG_SMP
249 : static bool cpuhp_is_ap_state(enum cpuhp_state state)
250 : {
251 : /*
252 : * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
253 : * purposes as that state is handled explicitly in cpu_down.
254 : */
255 : return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
256 : }
257 :
258 : static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
259 : {
260 : struct completion *done = bringup ? &st->done_up : &st->done_down;
261 : wait_for_completion(done);
262 : }
263 :
264 : static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
265 : {
266 : struct completion *done = bringup ? &st->done_up : &st->done_down;
267 : complete(done);
268 : }
269 :
270 : /*
271 : * The former STARTING/DYING states, ran with IRQs disabled and must not fail.
272 : */
273 : static bool cpuhp_is_atomic_state(enum cpuhp_state state)
274 : {
275 : return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE;
276 : }
277 :
278 : /* Serializes the updates to cpu_online_mask, cpu_present_mask */
279 : static DEFINE_MUTEX(cpu_add_remove_lock);
280 : bool cpuhp_tasks_frozen;
281 : EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
282 :
283 : /*
284 : * The following two APIs (cpu_maps_update_begin/done) must be used when
285 : * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
286 : */
287 : void cpu_maps_update_begin(void)
288 : {
289 : mutex_lock(&cpu_add_remove_lock);
290 : }
291 :
292 : void cpu_maps_update_done(void)
293 : {
294 : mutex_unlock(&cpu_add_remove_lock);
295 : }
296 :
297 : /*
298 : * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
299 : * Should always be manipulated under cpu_add_remove_lock
300 : */
301 : static int cpu_hotplug_disabled;
302 :
303 : #ifdef CONFIG_HOTPLUG_CPU
304 :
305 : DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);
306 :
307 : void cpus_read_lock(void)
308 : {
309 : percpu_down_read(&cpu_hotplug_lock);
310 : }
311 : EXPORT_SYMBOL_GPL(cpus_read_lock);
312 :
313 : int cpus_read_trylock(void)
314 : {
315 : return percpu_down_read_trylock(&cpu_hotplug_lock);
316 : }
317 : EXPORT_SYMBOL_GPL(cpus_read_trylock);
318 :
319 : void cpus_read_unlock(void)
320 : {
321 : percpu_up_read(&cpu_hotplug_lock);
322 : }
323 : EXPORT_SYMBOL_GPL(cpus_read_unlock);
324 :
325 : void cpus_write_lock(void)
326 : {
327 : percpu_down_write(&cpu_hotplug_lock);
328 : }
329 :
330 : void cpus_write_unlock(void)
331 : {
332 : percpu_up_write(&cpu_hotplug_lock);
333 : }
334 :
335 : void lockdep_assert_cpus_held(void)
336 : {
337 : /*
338 : * We can't have hotplug operations before userspace starts running,
339 : * and some init codepaths will knowingly not take the hotplug lock.
340 : * This is all valid, so mute lockdep until it makes sense to report
341 : * unheld locks.
342 : */
343 : if (system_state < SYSTEM_RUNNING)
344 : return;
345 :
346 : percpu_rwsem_assert_held(&cpu_hotplug_lock);
347 : }
348 :
349 : #ifdef CONFIG_LOCKDEP
350 : int lockdep_is_cpus_held(void)
351 : {
352 : return percpu_rwsem_is_held(&cpu_hotplug_lock);
353 : }
354 : #endif
355 :
356 : static void lockdep_acquire_cpus_lock(void)
357 : {
358 : rwsem_acquire(&cpu_hotplug_lock.dep_map, 0, 0, _THIS_IP_);
359 : }
360 :
361 : static void lockdep_release_cpus_lock(void)
362 : {
363 : rwsem_release(&cpu_hotplug_lock.dep_map, _THIS_IP_);
364 : }
365 :
366 : /*
367 : * Wait for currently running CPU hotplug operations to complete (if any) and
368 : * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
369 : * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
370 : * hotplug path before performing hotplug operations. So acquiring that lock
371 : * guarantees mutual exclusion from any currently running hotplug operations.
372 : */
373 : void cpu_hotplug_disable(void)
374 : {
375 : cpu_maps_update_begin();
376 : cpu_hotplug_disabled++;
377 : cpu_maps_update_done();
378 : }
379 : EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
380 :
381 : static void __cpu_hotplug_enable(void)
382 : {
383 : if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))
384 : return;
385 : cpu_hotplug_disabled--;
386 : }
387 :
388 : void cpu_hotplug_enable(void)
389 : {
390 : cpu_maps_update_begin();
391 : __cpu_hotplug_enable();
392 : cpu_maps_update_done();
393 : }
394 : EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
395 :
396 : #else
397 :
398 : static void lockdep_acquire_cpus_lock(void)
399 : {
400 : }
401 :
402 : static void lockdep_release_cpus_lock(void)
403 : {
404 : }
405 :
406 : #endif /* CONFIG_HOTPLUG_CPU */
407 :
408 : /*
409 : * Architectures that need SMT-specific errata handling during SMT hotplug
410 : * should override this.
411 : */
412 : void __weak arch_smt_update(void) { }
413 :
414 : #ifdef CONFIG_HOTPLUG_SMT
415 : enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED;
416 :
417 : void __init cpu_smt_disable(bool force)
418 : {
419 : if (!cpu_smt_possible())
420 : return;
421 :
422 : if (force) {
423 : pr_info("SMT: Force disabled\n");
424 : cpu_smt_control = CPU_SMT_FORCE_DISABLED;
425 : } else {
426 : pr_info("SMT: disabled\n");
427 : cpu_smt_control = CPU_SMT_DISABLED;
428 : }
429 : }
430 :
431 : /*
432 : * The decision whether SMT is supported can only be done after the full
433 : * CPU identification. Called from architecture code.
434 : */
435 : void __init cpu_smt_check_topology(void)
436 : {
437 : if (!topology_smt_supported())
438 : cpu_smt_control = CPU_SMT_NOT_SUPPORTED;
439 : }
440 :
441 : static int __init smt_cmdline_disable(char *str)
442 : {
443 : cpu_smt_disable(str && !strcmp(str, "force"));
444 : return 0;
445 : }
446 : early_param("nosmt", smt_cmdline_disable);
447 :
448 : static inline bool cpu_smt_allowed(unsigned int cpu)
449 : {
450 : if (cpu_smt_control == CPU_SMT_ENABLED)
451 : return true;
452 :
453 : if (topology_is_primary_thread(cpu))
454 : return true;
455 :
456 : /*
457 : * On x86 it's required to boot all logical CPUs at least once so
458 : * that the init code can get a chance to set CR4.MCE on each
459 : * CPU. Otherwise, a broadcasted MCE observing CR4.MCE=0b on any
460 : * core will shutdown the machine.
461 : */
462 : return !cpumask_test_cpu(cpu, &cpus_booted_once_mask);
463 : }
464 :
465 : /* Returns true if SMT is not supported of forcefully (irreversibly) disabled */
466 : bool cpu_smt_possible(void)
467 : {
468 : return cpu_smt_control != CPU_SMT_FORCE_DISABLED &&
469 : cpu_smt_control != CPU_SMT_NOT_SUPPORTED;
470 : }
471 : EXPORT_SYMBOL_GPL(cpu_smt_possible);
472 : #else
473 : static inline bool cpu_smt_allowed(unsigned int cpu) { return true; }
474 : #endif
475 :
476 : static inline enum cpuhp_state
477 : cpuhp_set_state(int cpu, struct cpuhp_cpu_state *st, enum cpuhp_state target)
478 : {
479 : enum cpuhp_state prev_state = st->state;
480 : bool bringup = st->state < target;
481 :
482 : st->rollback = false;
483 : st->last = NULL;
484 :
485 : st->target = target;
486 : st->single = false;
487 : st->bringup = bringup;
488 : if (cpu_dying(cpu) != !bringup)
489 : set_cpu_dying(cpu, !bringup);
490 :
491 : return prev_state;
492 : }
493 :
494 : static inline void
495 : cpuhp_reset_state(int cpu, struct cpuhp_cpu_state *st,
496 : enum cpuhp_state prev_state)
497 : {
498 : bool bringup = !st->bringup;
499 :
500 : st->target = prev_state;
501 :
502 : /*
503 : * Already rolling back. No need invert the bringup value or to change
504 : * the current state.
505 : */
506 : if (st->rollback)
507 : return;
508 :
509 : st->rollback = true;
510 :
511 : /*
512 : * If we have st->last we need to undo partial multi_instance of this
513 : * state first. Otherwise start undo at the previous state.
514 : */
515 : if (!st->last) {
516 : if (st->bringup)
517 : st->state--;
518 : else
519 : st->state++;
520 : }
521 :
522 : st->bringup = bringup;
523 : if (cpu_dying(cpu) != !bringup)
524 : set_cpu_dying(cpu, !bringup);
525 : }
526 :
527 : /* Regular hotplug invocation of the AP hotplug thread */
528 : static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st)
529 : {
530 : if (!st->single && st->state == st->target)
531 : return;
532 :
533 : st->result = 0;
534 : /*
535 : * Make sure the above stores are visible before should_run becomes
536 : * true. Paired with the mb() above in cpuhp_thread_fun()
537 : */
538 : smp_mb();
539 : st->should_run = true;
540 : wake_up_process(st->thread);
541 : wait_for_ap_thread(st, st->bringup);
542 : }
543 :
544 : static int cpuhp_kick_ap(int cpu, struct cpuhp_cpu_state *st,
545 : enum cpuhp_state target)
546 : {
547 : enum cpuhp_state prev_state;
548 : int ret;
549 :
550 : prev_state = cpuhp_set_state(cpu, st, target);
551 : __cpuhp_kick_ap(st);
552 : if ((ret = st->result)) {
553 : cpuhp_reset_state(cpu, st, prev_state);
554 : __cpuhp_kick_ap(st);
555 : }
556 :
557 : return ret;
558 : }
559 :
560 : static int bringup_wait_for_ap(unsigned int cpu)
561 : {
562 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
563 :
564 : /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
565 : wait_for_ap_thread(st, true);
566 : if (WARN_ON_ONCE((!cpu_online(cpu))))
567 : return -ECANCELED;
568 :
569 : /* Unpark the hotplug thread of the target cpu */
570 : kthread_unpark(st->thread);
571 :
572 : /*
573 : * SMT soft disabling on X86 requires to bring the CPU out of the
574 : * BIOS 'wait for SIPI' state in order to set the CR4.MCE bit. The
575 : * CPU marked itself as booted_once in notify_cpu_starting() so the
576 : * cpu_smt_allowed() check will now return false if this is not the
577 : * primary sibling.
578 : */
579 : if (!cpu_smt_allowed(cpu))
580 : return -ECANCELED;
581 :
582 : if (st->target <= CPUHP_AP_ONLINE_IDLE)
583 : return 0;
584 :
585 : return cpuhp_kick_ap(cpu, st, st->target);
586 : }
587 :
588 : static int bringup_cpu(unsigned int cpu)
589 : {
590 : struct task_struct *idle = idle_thread_get(cpu);
591 : int ret;
592 :
593 : /*
594 : * Reset stale stack state from the last time this CPU was online.
595 : */
596 : scs_task_reset(idle);
597 : kasan_unpoison_task_stack(idle);
598 :
599 : /*
600 : * Some architectures have to walk the irq descriptors to
601 : * setup the vector space for the cpu which comes online.
602 : * Prevent irq alloc/free across the bringup.
603 : */
604 : irq_lock_sparse();
605 :
606 : /* Arch-specific enabling code. */
607 : ret = __cpu_up(cpu, idle);
608 : irq_unlock_sparse();
609 : if (ret)
610 : return ret;
611 : return bringup_wait_for_ap(cpu);
612 : }
613 :
614 : static int finish_cpu(unsigned int cpu)
615 : {
616 : struct task_struct *idle = idle_thread_get(cpu);
617 : struct mm_struct *mm = idle->active_mm;
618 :
619 : /*
620 : * idle_task_exit() will have switched to &init_mm, now
621 : * clean up any remaining active_mm state.
622 : */
623 : if (mm != &init_mm)
624 : idle->active_mm = &init_mm;
625 : mmdrop(mm);
626 : return 0;
627 : }
628 :
629 : /*
630 : * Hotplug state machine related functions
631 : */
632 :
633 : /*
634 : * Get the next state to run. Empty ones will be skipped. Returns true if a
635 : * state must be run.
636 : *
637 : * st->state will be modified ahead of time, to match state_to_run, as if it
638 : * has already ran.
639 : */
640 : static bool cpuhp_next_state(bool bringup,
641 : enum cpuhp_state *state_to_run,
642 : struct cpuhp_cpu_state *st,
643 : enum cpuhp_state target)
644 : {
645 : do {
646 : if (bringup) {
647 : if (st->state >= target)
648 : return false;
649 :
650 : *state_to_run = ++st->state;
651 : } else {
652 : if (st->state <= target)
653 : return false;
654 :
655 : *state_to_run = st->state--;
656 : }
657 :
658 : if (!cpuhp_step_empty(bringup, cpuhp_get_step(*state_to_run)))
659 : break;
660 : } while (true);
661 :
662 : return true;
663 : }
664 :
665 : static int cpuhp_invoke_callback_range(bool bringup,
666 : unsigned int cpu,
667 : struct cpuhp_cpu_state *st,
668 : enum cpuhp_state target)
669 : {
670 : enum cpuhp_state state;
671 : int err = 0;
672 :
673 : while (cpuhp_next_state(bringup, &state, st, target)) {
674 : err = cpuhp_invoke_callback(cpu, state, bringup, NULL, NULL);
675 : if (err)
676 : break;
677 : }
678 :
679 : return err;
680 : }
681 :
682 : static inline bool can_rollback_cpu(struct cpuhp_cpu_state *st)
683 : {
684 : if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
685 : return true;
686 : /*
687 : * When CPU hotplug is disabled, then taking the CPU down is not
688 : * possible because takedown_cpu() and the architecture and
689 : * subsystem specific mechanisms are not available. So the CPU
690 : * which would be completely unplugged again needs to stay around
691 : * in the current state.
692 : */
693 : return st->state <= CPUHP_BRINGUP_CPU;
694 : }
695 :
696 : static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
697 : enum cpuhp_state target)
698 : {
699 : enum cpuhp_state prev_state = st->state;
700 : int ret = 0;
701 :
702 : ret = cpuhp_invoke_callback_range(true, cpu, st, target);
703 : if (ret) {
704 : pr_debug("CPU UP failed (%d) CPU %u state %s (%d)\n",
705 : ret, cpu, cpuhp_get_step(st->state)->name,
706 : st->state);
707 :
708 : cpuhp_reset_state(cpu, st, prev_state);
709 : if (can_rollback_cpu(st))
710 : WARN_ON(cpuhp_invoke_callback_range(false, cpu, st,
711 : prev_state));
712 : }
713 : return ret;
714 : }
715 :
716 : /*
717 : * The cpu hotplug threads manage the bringup and teardown of the cpus
718 : */
719 : static void cpuhp_create(unsigned int cpu)
720 : {
721 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
722 :
723 : init_completion(&st->done_up);
724 : init_completion(&st->done_down);
725 : }
726 :
727 : static int cpuhp_should_run(unsigned int cpu)
728 : {
729 : struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
730 :
731 : return st->should_run;
732 : }
733 :
734 : /*
735 : * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
736 : * callbacks when a state gets [un]installed at runtime.
737 : *
738 : * Each invocation of this function by the smpboot thread does a single AP
739 : * state callback.
740 : *
741 : * It has 3 modes of operation:
742 : * - single: runs st->cb_state
743 : * - up: runs ++st->state, while st->state < st->target
744 : * - down: runs st->state--, while st->state > st->target
745 : *
746 : * When complete or on error, should_run is cleared and the completion is fired.
747 : */
748 : static void cpuhp_thread_fun(unsigned int cpu)
749 : {
750 : struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
751 : bool bringup = st->bringup;
752 : enum cpuhp_state state;
753 :
754 : if (WARN_ON_ONCE(!st->should_run))
755 : return;
756 :
757 : /*
758 : * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
759 : * that if we see ->should_run we also see the rest of the state.
760 : */
761 : smp_mb();
762 :
763 : /*
764 : * The BP holds the hotplug lock, but we're now running on the AP,
765 : * ensure that anybody asserting the lock is held, will actually find
766 : * it so.
767 : */
768 : lockdep_acquire_cpus_lock();
769 : cpuhp_lock_acquire(bringup);
770 :
771 : if (st->single) {
772 : state = st->cb_state;
773 : st->should_run = false;
774 : } else {
775 : st->should_run = cpuhp_next_state(bringup, &state, st, st->target);
776 : if (!st->should_run)
777 : goto end;
778 : }
779 :
780 : WARN_ON_ONCE(!cpuhp_is_ap_state(state));
781 :
782 : if (cpuhp_is_atomic_state(state)) {
783 : local_irq_disable();
784 : st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
785 : local_irq_enable();
786 :
787 : /*
788 : * STARTING/DYING must not fail!
789 : */
790 : WARN_ON_ONCE(st->result);
791 : } else {
792 : st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
793 : }
794 :
795 : if (st->result) {
796 : /*
797 : * If we fail on a rollback, we're up a creek without no
798 : * paddle, no way forward, no way back. We loose, thanks for
799 : * playing.
800 : */
801 : WARN_ON_ONCE(st->rollback);
802 : st->should_run = false;
803 : }
804 :
805 : end:
806 : cpuhp_lock_release(bringup);
807 : lockdep_release_cpus_lock();
808 :
809 : if (!st->should_run)
810 : complete_ap_thread(st, bringup);
811 : }
812 :
813 : /* Invoke a single callback on a remote cpu */
814 : static int
815 : cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
816 : struct hlist_node *node)
817 : {
818 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
819 : int ret;
820 :
821 : if (!cpu_online(cpu))
822 : return 0;
823 :
824 : cpuhp_lock_acquire(false);
825 : cpuhp_lock_release(false);
826 :
827 : cpuhp_lock_acquire(true);
828 : cpuhp_lock_release(true);
829 :
830 : /*
831 : * If we are up and running, use the hotplug thread. For early calls
832 : * we invoke the thread function directly.
833 : */
834 : if (!st->thread)
835 : return cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
836 :
837 : st->rollback = false;
838 : st->last = NULL;
839 :
840 : st->node = node;
841 : st->bringup = bringup;
842 : st->cb_state = state;
843 : st->single = true;
844 :
845 : __cpuhp_kick_ap(st);
846 :
847 : /*
848 : * If we failed and did a partial, do a rollback.
849 : */
850 : if ((ret = st->result) && st->last) {
851 : st->rollback = true;
852 : st->bringup = !bringup;
853 :
854 : __cpuhp_kick_ap(st);
855 : }
856 :
857 : /*
858 : * Clean up the leftovers so the next hotplug operation wont use stale
859 : * data.
860 : */
861 : st->node = st->last = NULL;
862 : return ret;
863 : }
864 :
865 : static int cpuhp_kick_ap_work(unsigned int cpu)
866 : {
867 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
868 : enum cpuhp_state prev_state = st->state;
869 : int ret;
870 :
871 : cpuhp_lock_acquire(false);
872 : cpuhp_lock_release(false);
873 :
874 : cpuhp_lock_acquire(true);
875 : cpuhp_lock_release(true);
876 :
877 : trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);
878 : ret = cpuhp_kick_ap(cpu, st, st->target);
879 : trace_cpuhp_exit(cpu, st->state, prev_state, ret);
880 :
881 : return ret;
882 : }
883 :
884 : static struct smp_hotplug_thread cpuhp_threads = {
885 : .store = &cpuhp_state.thread,
886 : .create = &cpuhp_create,
887 : .thread_should_run = cpuhp_should_run,
888 : .thread_fn = cpuhp_thread_fun,
889 : .thread_comm = "cpuhp/%u",
890 : .selfparking = true,
891 : };
892 :
893 : void __init cpuhp_threads_init(void)
894 : {
895 : BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
896 : kthread_unpark(this_cpu_read(cpuhp_state.thread));
897 : }
898 :
899 : /*
900 : *
901 : * Serialize hotplug trainwrecks outside of the cpu_hotplug_lock
902 : * protected region.
903 : *
904 : * The operation is still serialized against concurrent CPU hotplug via
905 : * cpu_add_remove_lock, i.e. CPU map protection. But it is _not_
906 : * serialized against other hotplug related activity like adding or
907 : * removing of state callbacks and state instances, which invoke either the
908 : * startup or the teardown callback of the affected state.
909 : *
910 : * This is required for subsystems which are unfixable vs. CPU hotplug and
911 : * evade lock inversion problems by scheduling work which has to be
912 : * completed _before_ cpu_up()/_cpu_down() returns.
913 : *
914 : * Don't even think about adding anything to this for any new code or even
915 : * drivers. It's only purpose is to keep existing lock order trainwrecks
916 : * working.
917 : *
918 : * For cpu_down() there might be valid reasons to finish cleanups which are
919 : * not required to be done under cpu_hotplug_lock, but that's a different
920 : * story and would be not invoked via this.
921 : */
922 : static void cpu_up_down_serialize_trainwrecks(bool tasks_frozen)
923 : {
924 : /*
925 : * cpusets delegate hotplug operations to a worker to "solve" the
926 : * lock order problems. Wait for the worker, but only if tasks are
927 : * _not_ frozen (suspend, hibernate) as that would wait forever.
928 : *
929 : * The wait is required because otherwise the hotplug operation
930 : * returns with inconsistent state, which could even be observed in
931 : * user space when a new CPU is brought up. The CPU plug uevent
932 : * would be delivered and user space reacting on it would fail to
933 : * move tasks to the newly plugged CPU up to the point where the
934 : * work has finished because up to that point the newly plugged CPU
935 : * is not assignable in cpusets/cgroups. On unplug that's not
936 : * necessarily a visible issue, but it is still inconsistent state,
937 : * which is the real problem which needs to be "fixed". This can't
938 : * prevent the transient state between scheduling the work and
939 : * returning from waiting for it.
940 : */
941 : if (!tasks_frozen)
942 : cpuset_wait_for_hotplug();
943 : }
944 :
945 : #ifdef CONFIG_HOTPLUG_CPU
946 : #ifndef arch_clear_mm_cpumask_cpu
947 : #define arch_clear_mm_cpumask_cpu(cpu, mm) cpumask_clear_cpu(cpu, mm_cpumask(mm))
948 : #endif
949 :
950 : /**
951 : * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
952 : * @cpu: a CPU id
953 : *
954 : * This function walks all processes, finds a valid mm struct for each one and
955 : * then clears a corresponding bit in mm's cpumask. While this all sounds
956 : * trivial, there are various non-obvious corner cases, which this function
957 : * tries to solve in a safe manner.
958 : *
959 : * Also note that the function uses a somewhat relaxed locking scheme, so it may
960 : * be called only for an already offlined CPU.
961 : */
962 : void clear_tasks_mm_cpumask(int cpu)
963 : {
964 : struct task_struct *p;
965 :
966 : /*
967 : * This function is called after the cpu is taken down and marked
968 : * offline, so its not like new tasks will ever get this cpu set in
969 : * their mm mask. -- Peter Zijlstra
970 : * Thus, we may use rcu_read_lock() here, instead of grabbing
971 : * full-fledged tasklist_lock.
972 : */
973 : WARN_ON(cpu_online(cpu));
974 : rcu_read_lock();
975 : for_each_process(p) {
976 : struct task_struct *t;
977 :
978 : /*
979 : * Main thread might exit, but other threads may still have
980 : * a valid mm. Find one.
981 : */
982 : t = find_lock_task_mm(p);
983 : if (!t)
984 : continue;
985 : arch_clear_mm_cpumask_cpu(cpu, t->mm);
986 : task_unlock(t);
987 : }
988 : rcu_read_unlock();
989 : }
990 :
991 : /* Take this CPU down. */
992 : static int take_cpu_down(void *_param)
993 : {
994 : struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
995 : enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
996 : int err, cpu = smp_processor_id();
997 : int ret;
998 :
999 : /* Ensure this CPU doesn't handle any more interrupts. */
1000 : err = __cpu_disable();
1001 : if (err < 0)
1002 : return err;
1003 :
1004 : /*
1005 : * Must be called from CPUHP_TEARDOWN_CPU, which means, as we are going
1006 : * down, that the current state is CPUHP_TEARDOWN_CPU - 1.
1007 : */
1008 : WARN_ON(st->state != (CPUHP_TEARDOWN_CPU - 1));
1009 :
1010 : /* Invoke the former CPU_DYING callbacks */
1011 : ret = cpuhp_invoke_callback_range(false, cpu, st, target);
1012 :
1013 : /*
1014 : * DYING must not fail!
1015 : */
1016 : WARN_ON_ONCE(ret);
1017 :
1018 : /* Give up timekeeping duties */
1019 : tick_handover_do_timer();
1020 : /* Remove CPU from timer broadcasting */
1021 : tick_offline_cpu(cpu);
1022 : /* Park the stopper thread */
1023 : stop_machine_park(cpu);
1024 : return 0;
1025 : }
1026 :
1027 : static int takedown_cpu(unsigned int cpu)
1028 : {
1029 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1030 : int err;
1031 :
1032 : /* Park the smpboot threads */
1033 : kthread_park(st->thread);
1034 :
1035 : /*
1036 : * Prevent irq alloc/free while the dying cpu reorganizes the
1037 : * interrupt affinities.
1038 : */
1039 : irq_lock_sparse();
1040 :
1041 : /*
1042 : * So now all preempt/rcu users must observe !cpu_active().
1043 : */
1044 : err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));
1045 : if (err) {
1046 : /* CPU refused to die */
1047 : irq_unlock_sparse();
1048 : /* Unpark the hotplug thread so we can rollback there */
1049 : kthread_unpark(st->thread);
1050 : return err;
1051 : }
1052 : BUG_ON(cpu_online(cpu));
1053 :
1054 : /*
1055 : * The teardown callback for CPUHP_AP_SCHED_STARTING will have removed
1056 : * all runnable tasks from the CPU, there's only the idle task left now
1057 : * that the migration thread is done doing the stop_machine thing.
1058 : *
1059 : * Wait for the stop thread to go away.
1060 : */
1061 : wait_for_ap_thread(st, false);
1062 : BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);
1063 :
1064 : /* Interrupts are moved away from the dying cpu, reenable alloc/free */
1065 : irq_unlock_sparse();
1066 :
1067 : hotplug_cpu__broadcast_tick_pull(cpu);
1068 : /* This actually kills the CPU. */
1069 : __cpu_die(cpu);
1070 :
1071 : tick_cleanup_dead_cpu(cpu);
1072 : rcutree_migrate_callbacks(cpu);
1073 : return 0;
1074 : }
1075 :
1076 : static void cpuhp_complete_idle_dead(void *arg)
1077 : {
1078 : struct cpuhp_cpu_state *st = arg;
1079 :
1080 : complete_ap_thread(st, false);
1081 : }
1082 :
1083 : void cpuhp_report_idle_dead(void)
1084 : {
1085 : struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1086 :
1087 : BUG_ON(st->state != CPUHP_AP_OFFLINE);
1088 : rcu_report_dead(smp_processor_id());
1089 : st->state = CPUHP_AP_IDLE_DEAD;
1090 : /*
1091 : * We cannot call complete after rcu_report_dead() so we delegate it
1092 : * to an online cpu.
1093 : */
1094 : smp_call_function_single(cpumask_first(cpu_online_mask),
1095 : cpuhp_complete_idle_dead, st, 0);
1096 : }
1097 :
1098 : static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
1099 : enum cpuhp_state target)
1100 : {
1101 : enum cpuhp_state prev_state = st->state;
1102 : int ret = 0;
1103 :
1104 : ret = cpuhp_invoke_callback_range(false, cpu, st, target);
1105 : if (ret) {
1106 : pr_debug("CPU DOWN failed (%d) CPU %u state %s (%d)\n",
1107 : ret, cpu, cpuhp_get_step(st->state)->name,
1108 : st->state);
1109 :
1110 : cpuhp_reset_state(cpu, st, prev_state);
1111 :
1112 : if (st->state < prev_state)
1113 : WARN_ON(cpuhp_invoke_callback_range(true, cpu, st,
1114 : prev_state));
1115 : }
1116 :
1117 : return ret;
1118 : }
1119 :
1120 : /* Requires cpu_add_remove_lock to be held */
1121 : static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
1122 : enum cpuhp_state target)
1123 : {
1124 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1125 : int prev_state, ret = 0;
1126 :
1127 : if (num_online_cpus() == 1)
1128 : return -EBUSY;
1129 :
1130 : if (!cpu_present(cpu))
1131 : return -EINVAL;
1132 :
1133 : cpus_write_lock();
1134 :
1135 : cpuhp_tasks_frozen = tasks_frozen;
1136 :
1137 : prev_state = cpuhp_set_state(cpu, st, target);
1138 : /*
1139 : * If the current CPU state is in the range of the AP hotplug thread,
1140 : * then we need to kick the thread.
1141 : */
1142 : if (st->state > CPUHP_TEARDOWN_CPU) {
1143 : st->target = max((int)target, CPUHP_TEARDOWN_CPU);
1144 : ret = cpuhp_kick_ap_work(cpu);
1145 : /*
1146 : * The AP side has done the error rollback already. Just
1147 : * return the error code..
1148 : */
1149 : if (ret)
1150 : goto out;
1151 :
1152 : /*
1153 : * We might have stopped still in the range of the AP hotplug
1154 : * thread. Nothing to do anymore.
1155 : */
1156 : if (st->state > CPUHP_TEARDOWN_CPU)
1157 : goto out;
1158 :
1159 : st->target = target;
1160 : }
1161 : /*
1162 : * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
1163 : * to do the further cleanups.
1164 : */
1165 : ret = cpuhp_down_callbacks(cpu, st, target);
1166 : if (ret && st->state < prev_state) {
1167 : if (st->state == CPUHP_TEARDOWN_CPU) {
1168 : cpuhp_reset_state(cpu, st, prev_state);
1169 : __cpuhp_kick_ap(st);
1170 : } else {
1171 : WARN(1, "DEAD callback error for CPU%d", cpu);
1172 : }
1173 : }
1174 :
1175 : out:
1176 : cpus_write_unlock();
1177 : /*
1178 : * Do post unplug cleanup. This is still protected against
1179 : * concurrent CPU hotplug via cpu_add_remove_lock.
1180 : */
1181 : lockup_detector_cleanup();
1182 : arch_smt_update();
1183 : cpu_up_down_serialize_trainwrecks(tasks_frozen);
1184 : return ret;
1185 : }
1186 :
1187 : static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
1188 : {
1189 : if (cpu_hotplug_disabled)
1190 : return -EBUSY;
1191 : return _cpu_down(cpu, 0, target);
1192 : }
1193 :
1194 : static int cpu_down(unsigned int cpu, enum cpuhp_state target)
1195 : {
1196 : int err;
1197 :
1198 : cpu_maps_update_begin();
1199 : err = cpu_down_maps_locked(cpu, target);
1200 : cpu_maps_update_done();
1201 : return err;
1202 : }
1203 :
1204 : /**
1205 : * cpu_device_down - Bring down a cpu device
1206 : * @dev: Pointer to the cpu device to offline
1207 : *
1208 : * This function is meant to be used by device core cpu subsystem only.
1209 : *
1210 : * Other subsystems should use remove_cpu() instead.
1211 : *
1212 : * Return: %0 on success or a negative errno code
1213 : */
1214 : int cpu_device_down(struct device *dev)
1215 : {
1216 : return cpu_down(dev->id, CPUHP_OFFLINE);
1217 : }
1218 :
1219 : int remove_cpu(unsigned int cpu)
1220 : {
1221 : int ret;
1222 :
1223 : lock_device_hotplug();
1224 : ret = device_offline(get_cpu_device(cpu));
1225 : unlock_device_hotplug();
1226 :
1227 : return ret;
1228 : }
1229 : EXPORT_SYMBOL_GPL(remove_cpu);
1230 :
1231 : void smp_shutdown_nonboot_cpus(unsigned int primary_cpu)
1232 : {
1233 : unsigned int cpu;
1234 : int error;
1235 :
1236 : cpu_maps_update_begin();
1237 :
1238 : /*
1239 : * Make certain the cpu I'm about to reboot on is online.
1240 : *
1241 : * This is inline to what migrate_to_reboot_cpu() already do.
1242 : */
1243 : if (!cpu_online(primary_cpu))
1244 : primary_cpu = cpumask_first(cpu_online_mask);
1245 :
1246 : for_each_online_cpu(cpu) {
1247 : if (cpu == primary_cpu)
1248 : continue;
1249 :
1250 : error = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
1251 : if (error) {
1252 : pr_err("Failed to offline CPU%d - error=%d",
1253 : cpu, error);
1254 : break;
1255 : }
1256 : }
1257 :
1258 : /*
1259 : * Ensure all but the reboot CPU are offline.
1260 : */
1261 : BUG_ON(num_online_cpus() > 1);
1262 :
1263 : /*
1264 : * Make sure the CPUs won't be enabled by someone else after this
1265 : * point. Kexec will reboot to a new kernel shortly resetting
1266 : * everything along the way.
1267 : */
1268 : cpu_hotplug_disabled++;
1269 :
1270 : cpu_maps_update_done();
1271 : }
1272 :
1273 : #else
1274 : #define takedown_cpu NULL
1275 : #endif /*CONFIG_HOTPLUG_CPU*/
1276 :
1277 : /**
1278 : * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1279 : * @cpu: cpu that just started
1280 : *
1281 : * It must be called by the arch code on the new cpu, before the new cpu
1282 : * enables interrupts and before the "boot" cpu returns from __cpu_up().
1283 : */
1284 : void notify_cpu_starting(unsigned int cpu)
1285 : {
1286 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1287 : enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
1288 : int ret;
1289 :
1290 : rcu_cpu_starting(cpu); /* Enables RCU usage on this CPU. */
1291 : cpumask_set_cpu(cpu, &cpus_booted_once_mask);
1292 : ret = cpuhp_invoke_callback_range(true, cpu, st, target);
1293 :
1294 : /*
1295 : * STARTING must not fail!
1296 : */
1297 : WARN_ON_ONCE(ret);
1298 : }
1299 :
1300 : /*
1301 : * Called from the idle task. Wake up the controlling task which brings the
1302 : * hotplug thread of the upcoming CPU up and then delegates the rest of the
1303 : * online bringup to the hotplug thread.
1304 : */
1305 : void cpuhp_online_idle(enum cpuhp_state state)
1306 : {
1307 : struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1308 :
1309 : /* Happens for the boot cpu */
1310 : if (state != CPUHP_AP_ONLINE_IDLE)
1311 : return;
1312 :
1313 : /*
1314 : * Unpart the stopper thread before we start the idle loop (and start
1315 : * scheduling); this ensures the stopper task is always available.
1316 : */
1317 : stop_machine_unpark(smp_processor_id());
1318 :
1319 : st->state = CPUHP_AP_ONLINE_IDLE;
1320 : complete_ap_thread(st, true);
1321 : }
1322 :
1323 : /* Requires cpu_add_remove_lock to be held */
1324 : static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
1325 : {
1326 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1327 : struct task_struct *idle;
1328 : int ret = 0;
1329 :
1330 : cpus_write_lock();
1331 :
1332 : if (!cpu_present(cpu)) {
1333 : ret = -EINVAL;
1334 : goto out;
1335 : }
1336 :
1337 : /*
1338 : * The caller of cpu_up() might have raced with another
1339 : * caller. Nothing to do.
1340 : */
1341 : if (st->state >= target)
1342 : goto out;
1343 :
1344 : if (st->state == CPUHP_OFFLINE) {
1345 : /* Let it fail before we try to bring the cpu up */
1346 : idle = idle_thread_get(cpu);
1347 : if (IS_ERR(idle)) {
1348 : ret = PTR_ERR(idle);
1349 : goto out;
1350 : }
1351 : }
1352 :
1353 : cpuhp_tasks_frozen = tasks_frozen;
1354 :
1355 : cpuhp_set_state(cpu, st, target);
1356 : /*
1357 : * If the current CPU state is in the range of the AP hotplug thread,
1358 : * then we need to kick the thread once more.
1359 : */
1360 : if (st->state > CPUHP_BRINGUP_CPU) {
1361 : ret = cpuhp_kick_ap_work(cpu);
1362 : /*
1363 : * The AP side has done the error rollback already. Just
1364 : * return the error code..
1365 : */
1366 : if (ret)
1367 : goto out;
1368 : }
1369 :
1370 : /*
1371 : * Try to reach the target state. We max out on the BP at
1372 : * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
1373 : * responsible for bringing it up to the target state.
1374 : */
1375 : target = min((int)target, CPUHP_BRINGUP_CPU);
1376 : ret = cpuhp_up_callbacks(cpu, st, target);
1377 : out:
1378 : cpus_write_unlock();
1379 : arch_smt_update();
1380 : cpu_up_down_serialize_trainwrecks(tasks_frozen);
1381 : return ret;
1382 : }
1383 :
1384 : static int cpu_up(unsigned int cpu, enum cpuhp_state target)
1385 : {
1386 : int err = 0;
1387 :
1388 : if (!cpu_possible(cpu)) {
1389 : pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
1390 : cpu);
1391 : #if defined(CONFIG_IA64)
1392 : pr_err("please check additional_cpus= boot parameter\n");
1393 : #endif
1394 : return -EINVAL;
1395 : }
1396 :
1397 : err = try_online_node(cpu_to_node(cpu));
1398 : if (err)
1399 : return err;
1400 :
1401 : cpu_maps_update_begin();
1402 :
1403 : if (cpu_hotplug_disabled) {
1404 : err = -EBUSY;
1405 : goto out;
1406 : }
1407 : if (!cpu_smt_allowed(cpu)) {
1408 : err = -EPERM;
1409 : goto out;
1410 : }
1411 :
1412 : err = _cpu_up(cpu, 0, target);
1413 : out:
1414 : cpu_maps_update_done();
1415 : return err;
1416 : }
1417 :
1418 : /**
1419 : * cpu_device_up - Bring up a cpu device
1420 : * @dev: Pointer to the cpu device to online
1421 : *
1422 : * This function is meant to be used by device core cpu subsystem only.
1423 : *
1424 : * Other subsystems should use add_cpu() instead.
1425 : *
1426 : * Return: %0 on success or a negative errno code
1427 : */
1428 : int cpu_device_up(struct device *dev)
1429 : {
1430 : return cpu_up(dev->id, CPUHP_ONLINE);
1431 : }
1432 :
1433 : int add_cpu(unsigned int cpu)
1434 : {
1435 : int ret;
1436 :
1437 : lock_device_hotplug();
1438 : ret = device_online(get_cpu_device(cpu));
1439 : unlock_device_hotplug();
1440 :
1441 : return ret;
1442 : }
1443 : EXPORT_SYMBOL_GPL(add_cpu);
1444 :
1445 : /**
1446 : * bringup_hibernate_cpu - Bring up the CPU that we hibernated on
1447 : * @sleep_cpu: The cpu we hibernated on and should be brought up.
1448 : *
1449 : * On some architectures like arm64, we can hibernate on any CPU, but on
1450 : * wake up the CPU we hibernated on might be offline as a side effect of
1451 : * using maxcpus= for example.
1452 : *
1453 : * Return: %0 on success or a negative errno code
1454 : */
1455 : int bringup_hibernate_cpu(unsigned int sleep_cpu)
1456 : {
1457 : int ret;
1458 :
1459 : if (!cpu_online(sleep_cpu)) {
1460 : pr_info("Hibernated on a CPU that is offline! Bringing CPU up.\n");
1461 : ret = cpu_up(sleep_cpu, CPUHP_ONLINE);
1462 : if (ret) {
1463 : pr_err("Failed to bring hibernate-CPU up!\n");
1464 : return ret;
1465 : }
1466 : }
1467 : return 0;
1468 : }
1469 :
1470 : void bringup_nonboot_cpus(unsigned int setup_max_cpus)
1471 : {
1472 : unsigned int cpu;
1473 :
1474 : for_each_present_cpu(cpu) {
1475 : if (num_online_cpus() >= setup_max_cpus)
1476 : break;
1477 : if (!cpu_online(cpu))
1478 : cpu_up(cpu, CPUHP_ONLINE);
1479 : }
1480 : }
1481 :
1482 : #ifdef CONFIG_PM_SLEEP_SMP
1483 : static cpumask_var_t frozen_cpus;
1484 :
1485 : int freeze_secondary_cpus(int primary)
1486 : {
1487 : int cpu, error = 0;
1488 :
1489 : cpu_maps_update_begin();
1490 : if (primary == -1) {
1491 : primary = cpumask_first(cpu_online_mask);
1492 : if (!housekeeping_cpu(primary, HK_TYPE_TIMER))
1493 : primary = housekeeping_any_cpu(HK_TYPE_TIMER);
1494 : } else {
1495 : if (!cpu_online(primary))
1496 : primary = cpumask_first(cpu_online_mask);
1497 : }
1498 :
1499 : /*
1500 : * We take down all of the non-boot CPUs in one shot to avoid races
1501 : * with the userspace trying to use the CPU hotplug at the same time
1502 : */
1503 : cpumask_clear(frozen_cpus);
1504 :
1505 : pr_info("Disabling non-boot CPUs ...\n");
1506 : for_each_online_cpu(cpu) {
1507 : if (cpu == primary)
1508 : continue;
1509 :
1510 : if (pm_wakeup_pending()) {
1511 : pr_info("Wakeup pending. Abort CPU freeze\n");
1512 : error = -EBUSY;
1513 : break;
1514 : }
1515 :
1516 : trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
1517 : error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
1518 : trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
1519 : if (!error)
1520 : cpumask_set_cpu(cpu, frozen_cpus);
1521 : else {
1522 : pr_err("Error taking CPU%d down: %d\n", cpu, error);
1523 : break;
1524 : }
1525 : }
1526 :
1527 : if (!error)
1528 : BUG_ON(num_online_cpus() > 1);
1529 : else
1530 : pr_err("Non-boot CPUs are not disabled\n");
1531 :
1532 : /*
1533 : * Make sure the CPUs won't be enabled by someone else. We need to do
1534 : * this even in case of failure as all freeze_secondary_cpus() users are
1535 : * supposed to do thaw_secondary_cpus() on the failure path.
1536 : */
1537 : cpu_hotplug_disabled++;
1538 :
1539 : cpu_maps_update_done();
1540 : return error;
1541 : }
1542 :
1543 : void __weak arch_thaw_secondary_cpus_begin(void)
1544 : {
1545 : }
1546 :
1547 : void __weak arch_thaw_secondary_cpus_end(void)
1548 : {
1549 : }
1550 :
1551 : void thaw_secondary_cpus(void)
1552 : {
1553 : int cpu, error;
1554 :
1555 : /* Allow everyone to use the CPU hotplug again */
1556 : cpu_maps_update_begin();
1557 : __cpu_hotplug_enable();
1558 : if (cpumask_empty(frozen_cpus))
1559 : goto out;
1560 :
1561 : pr_info("Enabling non-boot CPUs ...\n");
1562 :
1563 : arch_thaw_secondary_cpus_begin();
1564 :
1565 : for_each_cpu(cpu, frozen_cpus) {
1566 : trace_suspend_resume(TPS("CPU_ON"), cpu, true);
1567 : error = _cpu_up(cpu, 1, CPUHP_ONLINE);
1568 : trace_suspend_resume(TPS("CPU_ON"), cpu, false);
1569 : if (!error) {
1570 : pr_info("CPU%d is up\n", cpu);
1571 : continue;
1572 : }
1573 : pr_warn("Error taking CPU%d up: %d\n", cpu, error);
1574 : }
1575 :
1576 : arch_thaw_secondary_cpus_end();
1577 :
1578 : cpumask_clear(frozen_cpus);
1579 : out:
1580 : cpu_maps_update_done();
1581 : }
1582 :
1583 : static int __init alloc_frozen_cpus(void)
1584 : {
1585 : if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
1586 : return -ENOMEM;
1587 : return 0;
1588 : }
1589 : core_initcall(alloc_frozen_cpus);
1590 :
1591 : /*
1592 : * When callbacks for CPU hotplug notifications are being executed, we must
1593 : * ensure that the state of the system with respect to the tasks being frozen
1594 : * or not, as reported by the notification, remains unchanged *throughout the
1595 : * duration* of the execution of the callbacks.
1596 : * Hence we need to prevent the freezer from racing with regular CPU hotplug.
1597 : *
1598 : * This synchronization is implemented by mutually excluding regular CPU
1599 : * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
1600 : * Hibernate notifications.
1601 : */
1602 : static int
1603 : cpu_hotplug_pm_callback(struct notifier_block *nb,
1604 : unsigned long action, void *ptr)
1605 : {
1606 : switch (action) {
1607 :
1608 : case PM_SUSPEND_PREPARE:
1609 : case PM_HIBERNATION_PREPARE:
1610 : cpu_hotplug_disable();
1611 : break;
1612 :
1613 : case PM_POST_SUSPEND:
1614 : case PM_POST_HIBERNATION:
1615 : cpu_hotplug_enable();
1616 : break;
1617 :
1618 : default:
1619 : return NOTIFY_DONE;
1620 : }
1621 :
1622 : return NOTIFY_OK;
1623 : }
1624 :
1625 :
1626 : static int __init cpu_hotplug_pm_sync_init(void)
1627 : {
1628 : /*
1629 : * cpu_hotplug_pm_callback has higher priority than x86
1630 : * bsp_pm_callback which depends on cpu_hotplug_pm_callback
1631 : * to disable cpu hotplug to avoid cpu hotplug race.
1632 : */
1633 : pm_notifier(cpu_hotplug_pm_callback, 0);
1634 : return 0;
1635 : }
1636 : core_initcall(cpu_hotplug_pm_sync_init);
1637 :
1638 : #endif /* CONFIG_PM_SLEEP_SMP */
1639 :
1640 : int __boot_cpu_id;
1641 :
1642 : #endif /* CONFIG_SMP */
1643 :
1644 : /* Boot processor state steps */
1645 : static struct cpuhp_step cpuhp_hp_states[] = {
1646 : [CPUHP_OFFLINE] = {
1647 : .name = "offline",
1648 : .startup.single = NULL,
1649 : .teardown.single = NULL,
1650 : },
1651 : #ifdef CONFIG_SMP
1652 : [CPUHP_CREATE_THREADS]= {
1653 : .name = "threads:prepare",
1654 : .startup.single = smpboot_create_threads,
1655 : .teardown.single = NULL,
1656 : .cant_stop = true,
1657 : },
1658 : [CPUHP_PERF_PREPARE] = {
1659 : .name = "perf:prepare",
1660 : .startup.single = perf_event_init_cpu,
1661 : .teardown.single = perf_event_exit_cpu,
1662 : },
1663 : [CPUHP_RANDOM_PREPARE] = {
1664 : .name = "random:prepare",
1665 : .startup.single = random_prepare_cpu,
1666 : .teardown.single = NULL,
1667 : },
1668 : [CPUHP_WORKQUEUE_PREP] = {
1669 : .name = "workqueue:prepare",
1670 : .startup.single = workqueue_prepare_cpu,
1671 : .teardown.single = NULL,
1672 : },
1673 : [CPUHP_HRTIMERS_PREPARE] = {
1674 : .name = "hrtimers:prepare",
1675 : .startup.single = hrtimers_prepare_cpu,
1676 : .teardown.single = hrtimers_dead_cpu,
1677 : },
1678 : [CPUHP_SMPCFD_PREPARE] = {
1679 : .name = "smpcfd:prepare",
1680 : .startup.single = smpcfd_prepare_cpu,
1681 : .teardown.single = smpcfd_dead_cpu,
1682 : },
1683 : [CPUHP_RELAY_PREPARE] = {
1684 : .name = "relay:prepare",
1685 : .startup.single = relay_prepare_cpu,
1686 : .teardown.single = NULL,
1687 : },
1688 : [CPUHP_SLAB_PREPARE] = {
1689 : .name = "slab:prepare",
1690 : .startup.single = slab_prepare_cpu,
1691 : .teardown.single = slab_dead_cpu,
1692 : },
1693 : [CPUHP_RCUTREE_PREP] = {
1694 : .name = "RCU/tree:prepare",
1695 : .startup.single = rcutree_prepare_cpu,
1696 : .teardown.single = rcutree_dead_cpu,
1697 : },
1698 : /*
1699 : * On the tear-down path, timers_dead_cpu() must be invoked
1700 : * before blk_mq_queue_reinit_notify() from notify_dead(),
1701 : * otherwise a RCU stall occurs.
1702 : */
1703 : [CPUHP_TIMERS_PREPARE] = {
1704 : .name = "timers:prepare",
1705 : .startup.single = timers_prepare_cpu,
1706 : .teardown.single = timers_dead_cpu,
1707 : },
1708 : /* Kicks the plugged cpu into life */
1709 : [CPUHP_BRINGUP_CPU] = {
1710 : .name = "cpu:bringup",
1711 : .startup.single = bringup_cpu,
1712 : .teardown.single = finish_cpu,
1713 : .cant_stop = true,
1714 : },
1715 : /* Final state before CPU kills itself */
1716 : [CPUHP_AP_IDLE_DEAD] = {
1717 : .name = "idle:dead",
1718 : },
1719 : /*
1720 : * Last state before CPU enters the idle loop to die. Transient state
1721 : * for synchronization.
1722 : */
1723 : [CPUHP_AP_OFFLINE] = {
1724 : .name = "ap:offline",
1725 : .cant_stop = true,
1726 : },
1727 : /* First state is scheduler control. Interrupts are disabled */
1728 : [CPUHP_AP_SCHED_STARTING] = {
1729 : .name = "sched:starting",
1730 : .startup.single = sched_cpu_starting,
1731 : .teardown.single = sched_cpu_dying,
1732 : },
1733 : [CPUHP_AP_RCUTREE_DYING] = {
1734 : .name = "RCU/tree:dying",
1735 : .startup.single = NULL,
1736 : .teardown.single = rcutree_dying_cpu,
1737 : },
1738 : [CPUHP_AP_SMPCFD_DYING] = {
1739 : .name = "smpcfd:dying",
1740 : .startup.single = NULL,
1741 : .teardown.single = smpcfd_dying_cpu,
1742 : },
1743 : /* Entry state on starting. Interrupts enabled from here on. Transient
1744 : * state for synchronsization */
1745 : [CPUHP_AP_ONLINE] = {
1746 : .name = "ap:online",
1747 : },
1748 : /*
1749 : * Handled on control processor until the plugged processor manages
1750 : * this itself.
1751 : */
1752 : [CPUHP_TEARDOWN_CPU] = {
1753 : .name = "cpu:teardown",
1754 : .startup.single = NULL,
1755 : .teardown.single = takedown_cpu,
1756 : .cant_stop = true,
1757 : },
1758 :
1759 : [CPUHP_AP_SCHED_WAIT_EMPTY] = {
1760 : .name = "sched:waitempty",
1761 : .startup.single = NULL,
1762 : .teardown.single = sched_cpu_wait_empty,
1763 : },
1764 :
1765 : /* Handle smpboot threads park/unpark */
1766 : [CPUHP_AP_SMPBOOT_THREADS] = {
1767 : .name = "smpboot/threads:online",
1768 : .startup.single = smpboot_unpark_threads,
1769 : .teardown.single = smpboot_park_threads,
1770 : },
1771 : [CPUHP_AP_IRQ_AFFINITY_ONLINE] = {
1772 : .name = "irq/affinity:online",
1773 : .startup.single = irq_affinity_online_cpu,
1774 : .teardown.single = NULL,
1775 : },
1776 : [CPUHP_AP_PERF_ONLINE] = {
1777 : .name = "perf:online",
1778 : .startup.single = perf_event_init_cpu,
1779 : .teardown.single = perf_event_exit_cpu,
1780 : },
1781 : [CPUHP_AP_WATCHDOG_ONLINE] = {
1782 : .name = "lockup_detector:online",
1783 : .startup.single = lockup_detector_online_cpu,
1784 : .teardown.single = lockup_detector_offline_cpu,
1785 : },
1786 : [CPUHP_AP_WORKQUEUE_ONLINE] = {
1787 : .name = "workqueue:online",
1788 : .startup.single = workqueue_online_cpu,
1789 : .teardown.single = workqueue_offline_cpu,
1790 : },
1791 : [CPUHP_AP_RANDOM_ONLINE] = {
1792 : .name = "random:online",
1793 : .startup.single = random_online_cpu,
1794 : .teardown.single = NULL,
1795 : },
1796 : [CPUHP_AP_RCUTREE_ONLINE] = {
1797 : .name = "RCU/tree:online",
1798 : .startup.single = rcutree_online_cpu,
1799 : .teardown.single = rcutree_offline_cpu,
1800 : },
1801 : #endif
1802 : /*
1803 : * The dynamically registered state space is here
1804 : */
1805 :
1806 : #ifdef CONFIG_SMP
1807 : /* Last state is scheduler control setting the cpu active */
1808 : [CPUHP_AP_ACTIVE] = {
1809 : .name = "sched:active",
1810 : .startup.single = sched_cpu_activate,
1811 : .teardown.single = sched_cpu_deactivate,
1812 : },
1813 : #endif
1814 :
1815 : /* CPU is fully up and running. */
1816 : [CPUHP_ONLINE] = {
1817 : .name = "online",
1818 : .startup.single = NULL,
1819 : .teardown.single = NULL,
1820 : },
1821 : };
1822 :
1823 : /* Sanity check for callbacks */
1824 : static int cpuhp_cb_check(enum cpuhp_state state)
1825 : {
1826 18 : if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
1827 : return -EINVAL;
1828 : return 0;
1829 : }
1830 :
1831 : /*
1832 : * Returns a free for dynamic slot assignment of the Online state. The states
1833 : * are protected by the cpuhp_slot_states mutex and an empty slot is identified
1834 : * by having no name assigned.
1835 : */
1836 5 : static int cpuhp_reserve_state(enum cpuhp_state state)
1837 : {
1838 : enum cpuhp_state i, end;
1839 : struct cpuhp_step *step;
1840 :
1841 5 : switch (state) {
1842 : case CPUHP_AP_ONLINE_DYN:
1843 : step = cpuhp_hp_states + CPUHP_AP_ONLINE_DYN;
1844 : end = CPUHP_AP_ONLINE_DYN_END;
1845 : break;
1846 : case CPUHP_BP_PREPARE_DYN:
1847 1 : step = cpuhp_hp_states + CPUHP_BP_PREPARE_DYN;
1848 1 : end = CPUHP_BP_PREPARE_DYN_END;
1849 1 : break;
1850 : default:
1851 : return -EINVAL;
1852 : }
1853 :
1854 11 : for (i = state; i <= end; i++, step++) {
1855 11 : if (!step->name)
1856 : return i;
1857 : }
1858 0 : WARN(1, "No more dynamic states available for CPU hotplug\n");
1859 0 : return -ENOSPC;
1860 : }
1861 :
1862 18 : static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name,
1863 : int (*startup)(unsigned int cpu),
1864 : int (*teardown)(unsigned int cpu),
1865 : bool multi_instance)
1866 : {
1867 : /* (Un)Install the callbacks for further cpu hotplug operations */
1868 : struct cpuhp_step *sp;
1869 19 : int ret = 0;
1870 :
1871 : /*
1872 : * If name is NULL, then the state gets removed.
1873 : *
1874 : * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on
1875 : * the first allocation from these dynamic ranges, so the removal
1876 : * would trigger a new allocation and clear the wrong (already
1877 : * empty) state, leaving the callbacks of the to be cleared state
1878 : * dangling, which causes wreckage on the next hotplug operation.
1879 : */
1880 36 : if (name && (state == CPUHP_AP_ONLINE_DYN ||
1881 18 : state == CPUHP_BP_PREPARE_DYN)) {
1882 5 : ret = cpuhp_reserve_state(state);
1883 5 : if (ret < 0)
1884 : return ret;
1885 : state = ret;
1886 : }
1887 19 : sp = cpuhp_get_step(state);
1888 18 : if (name && sp->name)
1889 : return -EBUSY;
1890 :
1891 19 : sp->startup.single = startup;
1892 19 : sp->teardown.single = teardown;
1893 19 : sp->name = name;
1894 19 : sp->multi_instance = multi_instance;
1895 19 : INIT_HLIST_HEAD(&sp->list);
1896 18 : return ret;
1897 : }
1898 :
1899 : static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
1900 : {
1901 0 : return cpuhp_get_step(state)->teardown.single;
1902 : }
1903 :
1904 : /*
1905 : * Call the startup/teardown function for a step either on the AP or
1906 : * on the current CPU.
1907 : */
1908 3 : static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
1909 : struct hlist_node *node)
1910 : {
1911 3 : struct cpuhp_step *sp = cpuhp_get_step(state);
1912 : int ret;
1913 :
1914 : /*
1915 : * If there's nothing to do, we done.
1916 : * Relies on the union for multi_instance.
1917 : */
1918 6 : if (cpuhp_step_empty(bringup, sp))
1919 : return 0;
1920 : /*
1921 : * The non AP bound callbacks can fail on bringup. On teardown
1922 : * e.g. module removal we crash for now.
1923 : */
1924 : #ifdef CONFIG_SMP
1925 : if (cpuhp_is_ap_state(state))
1926 : ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
1927 : else
1928 : ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1929 : #else
1930 3 : ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1931 : #endif
1932 3 : BUG_ON(ret && !bringup);
1933 : return ret;
1934 : }
1935 :
1936 : /*
1937 : * Called from __cpuhp_setup_state on a recoverable failure.
1938 : *
1939 : * Note: The teardown callbacks for rollback are not allowed to fail!
1940 : */
1941 1 : static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
1942 : struct hlist_node *node)
1943 : {
1944 : int cpu;
1945 :
1946 : /* Roll back the already executed steps on the other cpus */
1947 1 : for_each_present_cpu(cpu) {
1948 1 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1949 1 : int cpustate = st->state;
1950 :
1951 1 : if (cpu >= failedcpu)
1952 : break;
1953 :
1954 : /* Did we invoke the startup call on that cpu ? */
1955 0 : if (cpustate >= state)
1956 0 : cpuhp_issue_call(cpu, state, false, node);
1957 : }
1958 1 : }
1959 :
1960 1 : int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state,
1961 : struct hlist_node *node,
1962 : bool invoke)
1963 : {
1964 : struct cpuhp_step *sp;
1965 : int cpu;
1966 : int ret;
1967 :
1968 1 : lockdep_assert_cpus_held();
1969 :
1970 1 : sp = cpuhp_get_step(state);
1971 1 : if (sp->multi_instance == false)
1972 : return -EINVAL;
1973 :
1974 1 : mutex_lock(&cpuhp_state_mutex);
1975 :
1976 1 : if (!invoke || !sp->startup.multi)
1977 : goto add_node;
1978 :
1979 : /*
1980 : * Try to call the startup callback for each present cpu
1981 : * depending on the hotplug state of the cpu.
1982 : */
1983 0 : for_each_present_cpu(cpu) {
1984 0 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1985 0 : int cpustate = st->state;
1986 :
1987 0 : if (cpustate < state)
1988 0 : continue;
1989 :
1990 0 : ret = cpuhp_issue_call(cpu, state, true, node);
1991 0 : if (ret) {
1992 0 : if (sp->teardown.multi)
1993 0 : cpuhp_rollback_install(cpu, state, node);
1994 : goto unlock;
1995 : }
1996 : }
1997 : add_node:
1998 1 : ret = 0;
1999 1 : hlist_add_head(node, &sp->list);
2000 : unlock:
2001 1 : mutex_unlock(&cpuhp_state_mutex);
2002 1 : return ret;
2003 : }
2004 :
2005 1 : int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
2006 : bool invoke)
2007 : {
2008 : int ret;
2009 :
2010 : cpus_read_lock();
2011 1 : ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke);
2012 : cpus_read_unlock();
2013 1 : return ret;
2014 : }
2015 : EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance);
2016 :
2017 : /**
2018 : * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
2019 : * @state: The state to setup
2020 : * @name: Name of the step
2021 : * @invoke: If true, the startup function is invoked for cpus where
2022 : * cpu state >= @state
2023 : * @startup: startup callback function
2024 : * @teardown: teardown callback function
2025 : * @multi_instance: State is set up for multiple instances which get
2026 : * added afterwards.
2027 : *
2028 : * The caller needs to hold cpus read locked while calling this function.
2029 : * Return:
2030 : * On success:
2031 : * Positive state number if @state is CPUHP_AP_ONLINE_DYN;
2032 : * 0 for all other states
2033 : * On failure: proper (negative) error code
2034 : */
2035 18 : int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state,
2036 : const char *name, bool invoke,
2037 : int (*startup)(unsigned int cpu),
2038 : int (*teardown)(unsigned int cpu),
2039 : bool multi_instance)
2040 : {
2041 18 : int cpu, ret = 0;
2042 : bool dynstate;
2043 :
2044 18 : lockdep_assert_cpus_held();
2045 :
2046 18 : if (cpuhp_cb_check(state) || !name)
2047 : return -EINVAL;
2048 :
2049 18 : mutex_lock(&cpuhp_state_mutex);
2050 :
2051 18 : ret = cpuhp_store_callbacks(state, name, startup, teardown,
2052 : multi_instance);
2053 :
2054 18 : dynstate = state == CPUHP_AP_ONLINE_DYN;
2055 18 : if (ret > 0 && dynstate) {
2056 4 : state = ret;
2057 4 : ret = 0;
2058 : }
2059 :
2060 18 : if (ret || !invoke || !startup)
2061 : goto out;
2062 :
2063 : /*
2064 : * Try to call the startup callback for each present cpu
2065 : * depending on the hotplug state of the cpu.
2066 : */
2067 2 : for_each_present_cpu(cpu) {
2068 3 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2069 3 : int cpustate = st->state;
2070 :
2071 3 : if (cpustate < state)
2072 0 : continue;
2073 :
2074 3 : ret = cpuhp_issue_call(cpu, state, true, NULL);
2075 3 : if (ret) {
2076 1 : if (teardown)
2077 1 : cpuhp_rollback_install(cpu, state, NULL);
2078 : cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
2079 : goto out;
2080 : }
2081 : }
2082 : out:
2083 18 : mutex_unlock(&cpuhp_state_mutex);
2084 : /*
2085 : * If the requested state is CPUHP_AP_ONLINE_DYN, return the
2086 : * dynamically allocated state in case of success.
2087 : */
2088 18 : if (!ret && dynstate)
2089 : return state;
2090 14 : return ret;
2091 : }
2092 : EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked);
2093 :
2094 18 : int __cpuhp_setup_state(enum cpuhp_state state,
2095 : const char *name, bool invoke,
2096 : int (*startup)(unsigned int cpu),
2097 : int (*teardown)(unsigned int cpu),
2098 : bool multi_instance)
2099 : {
2100 : int ret;
2101 :
2102 : cpus_read_lock();
2103 18 : ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup,
2104 : teardown, multi_instance);
2105 : cpus_read_unlock();
2106 18 : return ret;
2107 : }
2108 : EXPORT_SYMBOL(__cpuhp_setup_state);
2109 :
2110 0 : int __cpuhp_state_remove_instance(enum cpuhp_state state,
2111 : struct hlist_node *node, bool invoke)
2112 : {
2113 0 : struct cpuhp_step *sp = cpuhp_get_step(state);
2114 : int cpu;
2115 :
2116 0 : BUG_ON(cpuhp_cb_check(state));
2117 :
2118 0 : if (!sp->multi_instance)
2119 : return -EINVAL;
2120 :
2121 : cpus_read_lock();
2122 0 : mutex_lock(&cpuhp_state_mutex);
2123 :
2124 0 : if (!invoke || !cpuhp_get_teardown_cb(state))
2125 : goto remove;
2126 : /*
2127 : * Call the teardown callback for each present cpu depending
2128 : * on the hotplug state of the cpu. This function is not
2129 : * allowed to fail currently!
2130 : */
2131 0 : for_each_present_cpu(cpu) {
2132 0 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2133 0 : int cpustate = st->state;
2134 :
2135 0 : if (cpustate >= state)
2136 0 : cpuhp_issue_call(cpu, state, false, node);
2137 : }
2138 :
2139 : remove:
2140 0 : hlist_del(node);
2141 0 : mutex_unlock(&cpuhp_state_mutex);
2142 : cpus_read_unlock();
2143 :
2144 0 : return 0;
2145 : }
2146 : EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);
2147 :
2148 : /**
2149 : * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
2150 : * @state: The state to remove
2151 : * @invoke: If true, the teardown function is invoked for cpus where
2152 : * cpu state >= @state
2153 : *
2154 : * The caller needs to hold cpus read locked while calling this function.
2155 : * The teardown callback is currently not allowed to fail. Think
2156 : * about module removal!
2157 : */
2158 0 : void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke)
2159 : {
2160 0 : struct cpuhp_step *sp = cpuhp_get_step(state);
2161 : int cpu;
2162 :
2163 0 : BUG_ON(cpuhp_cb_check(state));
2164 :
2165 : lockdep_assert_cpus_held();
2166 :
2167 0 : mutex_lock(&cpuhp_state_mutex);
2168 0 : if (sp->multi_instance) {
2169 0 : WARN(!hlist_empty(&sp->list),
2170 : "Error: Removing state %d which has instances left.\n",
2171 : state);
2172 : goto remove;
2173 : }
2174 :
2175 0 : if (!invoke || !cpuhp_get_teardown_cb(state))
2176 : goto remove;
2177 :
2178 : /*
2179 : * Call the teardown callback for each present cpu depending
2180 : * on the hotplug state of the cpu. This function is not
2181 : * allowed to fail currently!
2182 : */
2183 0 : for_each_present_cpu(cpu) {
2184 0 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2185 0 : int cpustate = st->state;
2186 :
2187 0 : if (cpustate >= state)
2188 0 : cpuhp_issue_call(cpu, state, false, NULL);
2189 : }
2190 : remove:
2191 0 : cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
2192 0 : mutex_unlock(&cpuhp_state_mutex);
2193 0 : }
2194 : EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked);
2195 :
2196 0 : void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
2197 : {
2198 : cpus_read_lock();
2199 0 : __cpuhp_remove_state_cpuslocked(state, invoke);
2200 : cpus_read_unlock();
2201 0 : }
2202 : EXPORT_SYMBOL(__cpuhp_remove_state);
2203 :
2204 : #ifdef CONFIG_HOTPLUG_SMT
2205 : static void cpuhp_offline_cpu_device(unsigned int cpu)
2206 : {
2207 : struct device *dev = get_cpu_device(cpu);
2208 :
2209 : dev->offline = true;
2210 : /* Tell user space about the state change */
2211 : kobject_uevent(&dev->kobj, KOBJ_OFFLINE);
2212 : }
2213 :
2214 : static void cpuhp_online_cpu_device(unsigned int cpu)
2215 : {
2216 : struct device *dev = get_cpu_device(cpu);
2217 :
2218 : dev->offline = false;
2219 : /* Tell user space about the state change */
2220 : kobject_uevent(&dev->kobj, KOBJ_ONLINE);
2221 : }
2222 :
2223 : int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
2224 : {
2225 : int cpu, ret = 0;
2226 :
2227 : cpu_maps_update_begin();
2228 : for_each_online_cpu(cpu) {
2229 : if (topology_is_primary_thread(cpu))
2230 : continue;
2231 : ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
2232 : if (ret)
2233 : break;
2234 : /*
2235 : * As this needs to hold the cpu maps lock it's impossible
2236 : * to call device_offline() because that ends up calling
2237 : * cpu_down() which takes cpu maps lock. cpu maps lock
2238 : * needs to be held as this might race against in kernel
2239 : * abusers of the hotplug machinery (thermal management).
2240 : *
2241 : * So nothing would update device:offline state. That would
2242 : * leave the sysfs entry stale and prevent onlining after
2243 : * smt control has been changed to 'off' again. This is
2244 : * called under the sysfs hotplug lock, so it is properly
2245 : * serialized against the regular offline usage.
2246 : */
2247 : cpuhp_offline_cpu_device(cpu);
2248 : }
2249 : if (!ret)
2250 : cpu_smt_control = ctrlval;
2251 : cpu_maps_update_done();
2252 : return ret;
2253 : }
2254 :
2255 : int cpuhp_smt_enable(void)
2256 : {
2257 : int cpu, ret = 0;
2258 :
2259 : cpu_maps_update_begin();
2260 : cpu_smt_control = CPU_SMT_ENABLED;
2261 : for_each_present_cpu(cpu) {
2262 : /* Skip online CPUs and CPUs on offline nodes */
2263 : if (cpu_online(cpu) || !node_online(cpu_to_node(cpu)))
2264 : continue;
2265 : ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
2266 : if (ret)
2267 : break;
2268 : /* See comment in cpuhp_smt_disable() */
2269 : cpuhp_online_cpu_device(cpu);
2270 : }
2271 : cpu_maps_update_done();
2272 : return ret;
2273 : }
2274 : #endif
2275 :
2276 : #if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
2277 : static ssize_t state_show(struct device *dev,
2278 : struct device_attribute *attr, char *buf)
2279 : {
2280 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2281 :
2282 : return sprintf(buf, "%d\n", st->state);
2283 : }
2284 : static DEVICE_ATTR_RO(state);
2285 :
2286 : static ssize_t target_store(struct device *dev, struct device_attribute *attr,
2287 : const char *buf, size_t count)
2288 : {
2289 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2290 : struct cpuhp_step *sp;
2291 : int target, ret;
2292 :
2293 : ret = kstrtoint(buf, 10, &target);
2294 : if (ret)
2295 : return ret;
2296 :
2297 : #ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
2298 : if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
2299 : return -EINVAL;
2300 : #else
2301 : if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
2302 : return -EINVAL;
2303 : #endif
2304 :
2305 : ret = lock_device_hotplug_sysfs();
2306 : if (ret)
2307 : return ret;
2308 :
2309 : mutex_lock(&cpuhp_state_mutex);
2310 : sp = cpuhp_get_step(target);
2311 : ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
2312 : mutex_unlock(&cpuhp_state_mutex);
2313 : if (ret)
2314 : goto out;
2315 :
2316 : if (st->state < target)
2317 : ret = cpu_up(dev->id, target);
2318 : else
2319 : ret = cpu_down(dev->id, target);
2320 : out:
2321 : unlock_device_hotplug();
2322 : return ret ? ret : count;
2323 : }
2324 :
2325 : static ssize_t target_show(struct device *dev,
2326 : struct device_attribute *attr, char *buf)
2327 : {
2328 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2329 :
2330 : return sprintf(buf, "%d\n", st->target);
2331 : }
2332 : static DEVICE_ATTR_RW(target);
2333 :
2334 : static ssize_t fail_store(struct device *dev, struct device_attribute *attr,
2335 : const char *buf, size_t count)
2336 : {
2337 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2338 : struct cpuhp_step *sp;
2339 : int fail, ret;
2340 :
2341 : ret = kstrtoint(buf, 10, &fail);
2342 : if (ret)
2343 : return ret;
2344 :
2345 : if (fail == CPUHP_INVALID) {
2346 : st->fail = fail;
2347 : return count;
2348 : }
2349 :
2350 : if (fail < CPUHP_OFFLINE || fail > CPUHP_ONLINE)
2351 : return -EINVAL;
2352 :
2353 : /*
2354 : * Cannot fail STARTING/DYING callbacks.
2355 : */
2356 : if (cpuhp_is_atomic_state(fail))
2357 : return -EINVAL;
2358 :
2359 : /*
2360 : * DEAD callbacks cannot fail...
2361 : * ... neither can CPUHP_BRINGUP_CPU during hotunplug. The latter
2362 : * triggering STARTING callbacks, a failure in this state would
2363 : * hinder rollback.
2364 : */
2365 : if (fail <= CPUHP_BRINGUP_CPU && st->state > CPUHP_BRINGUP_CPU)
2366 : return -EINVAL;
2367 :
2368 : /*
2369 : * Cannot fail anything that doesn't have callbacks.
2370 : */
2371 : mutex_lock(&cpuhp_state_mutex);
2372 : sp = cpuhp_get_step(fail);
2373 : if (!sp->startup.single && !sp->teardown.single)
2374 : ret = -EINVAL;
2375 : mutex_unlock(&cpuhp_state_mutex);
2376 : if (ret)
2377 : return ret;
2378 :
2379 : st->fail = fail;
2380 :
2381 : return count;
2382 : }
2383 :
2384 : static ssize_t fail_show(struct device *dev,
2385 : struct device_attribute *attr, char *buf)
2386 : {
2387 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2388 :
2389 : return sprintf(buf, "%d\n", st->fail);
2390 : }
2391 :
2392 : static DEVICE_ATTR_RW(fail);
2393 :
2394 : static struct attribute *cpuhp_cpu_attrs[] = {
2395 : &dev_attr_state.attr,
2396 : &dev_attr_target.attr,
2397 : &dev_attr_fail.attr,
2398 : NULL
2399 : };
2400 :
2401 : static const struct attribute_group cpuhp_cpu_attr_group = {
2402 : .attrs = cpuhp_cpu_attrs,
2403 : .name = "hotplug",
2404 : NULL
2405 : };
2406 :
2407 : static ssize_t states_show(struct device *dev,
2408 : struct device_attribute *attr, char *buf)
2409 : {
2410 : ssize_t cur, res = 0;
2411 : int i;
2412 :
2413 : mutex_lock(&cpuhp_state_mutex);
2414 : for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
2415 : struct cpuhp_step *sp = cpuhp_get_step(i);
2416 :
2417 : if (sp->name) {
2418 : cur = sprintf(buf, "%3d: %s\n", i, sp->name);
2419 : buf += cur;
2420 : res += cur;
2421 : }
2422 : }
2423 : mutex_unlock(&cpuhp_state_mutex);
2424 : return res;
2425 : }
2426 : static DEVICE_ATTR_RO(states);
2427 :
2428 : static struct attribute *cpuhp_cpu_root_attrs[] = {
2429 : &dev_attr_states.attr,
2430 : NULL
2431 : };
2432 :
2433 : static const struct attribute_group cpuhp_cpu_root_attr_group = {
2434 : .attrs = cpuhp_cpu_root_attrs,
2435 : .name = "hotplug",
2436 : NULL
2437 : };
2438 :
2439 : #ifdef CONFIG_HOTPLUG_SMT
2440 :
2441 : static ssize_t
2442 : __store_smt_control(struct device *dev, struct device_attribute *attr,
2443 : const char *buf, size_t count)
2444 : {
2445 : int ctrlval, ret;
2446 :
2447 : if (sysfs_streq(buf, "on"))
2448 : ctrlval = CPU_SMT_ENABLED;
2449 : else if (sysfs_streq(buf, "off"))
2450 : ctrlval = CPU_SMT_DISABLED;
2451 : else if (sysfs_streq(buf, "forceoff"))
2452 : ctrlval = CPU_SMT_FORCE_DISABLED;
2453 : else
2454 : return -EINVAL;
2455 :
2456 : if (cpu_smt_control == CPU_SMT_FORCE_DISABLED)
2457 : return -EPERM;
2458 :
2459 : if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
2460 : return -ENODEV;
2461 :
2462 : ret = lock_device_hotplug_sysfs();
2463 : if (ret)
2464 : return ret;
2465 :
2466 : if (ctrlval != cpu_smt_control) {
2467 : switch (ctrlval) {
2468 : case CPU_SMT_ENABLED:
2469 : ret = cpuhp_smt_enable();
2470 : break;
2471 : case CPU_SMT_DISABLED:
2472 : case CPU_SMT_FORCE_DISABLED:
2473 : ret = cpuhp_smt_disable(ctrlval);
2474 : break;
2475 : }
2476 : }
2477 :
2478 : unlock_device_hotplug();
2479 : return ret ? ret : count;
2480 : }
2481 :
2482 : #else /* !CONFIG_HOTPLUG_SMT */
2483 : static ssize_t
2484 : __store_smt_control(struct device *dev, struct device_attribute *attr,
2485 : const char *buf, size_t count)
2486 : {
2487 : return -ENODEV;
2488 : }
2489 : #endif /* CONFIG_HOTPLUG_SMT */
2490 :
2491 : static const char *smt_states[] = {
2492 : [CPU_SMT_ENABLED] = "on",
2493 : [CPU_SMT_DISABLED] = "off",
2494 : [CPU_SMT_FORCE_DISABLED] = "forceoff",
2495 : [CPU_SMT_NOT_SUPPORTED] = "notsupported",
2496 : [CPU_SMT_NOT_IMPLEMENTED] = "notimplemented",
2497 : };
2498 :
2499 : static ssize_t control_show(struct device *dev,
2500 : struct device_attribute *attr, char *buf)
2501 : {
2502 : const char *state = smt_states[cpu_smt_control];
2503 :
2504 : return snprintf(buf, PAGE_SIZE - 2, "%s\n", state);
2505 : }
2506 :
2507 : static ssize_t control_store(struct device *dev, struct device_attribute *attr,
2508 : const char *buf, size_t count)
2509 : {
2510 : return __store_smt_control(dev, attr, buf, count);
2511 : }
2512 : static DEVICE_ATTR_RW(control);
2513 :
2514 : static ssize_t active_show(struct device *dev,
2515 : struct device_attribute *attr, char *buf)
2516 : {
2517 : return snprintf(buf, PAGE_SIZE - 2, "%d\n", sched_smt_active());
2518 : }
2519 : static DEVICE_ATTR_RO(active);
2520 :
2521 : static struct attribute *cpuhp_smt_attrs[] = {
2522 : &dev_attr_control.attr,
2523 : &dev_attr_active.attr,
2524 : NULL
2525 : };
2526 :
2527 : static const struct attribute_group cpuhp_smt_attr_group = {
2528 : .attrs = cpuhp_smt_attrs,
2529 : .name = "smt",
2530 : NULL
2531 : };
2532 :
2533 : static int __init cpu_smt_sysfs_init(void)
2534 : {
2535 : return sysfs_create_group(&cpu_subsys.dev_root->kobj,
2536 : &cpuhp_smt_attr_group);
2537 : }
2538 :
2539 : static int __init cpuhp_sysfs_init(void)
2540 : {
2541 : int cpu, ret;
2542 :
2543 : ret = cpu_smt_sysfs_init();
2544 : if (ret)
2545 : return ret;
2546 :
2547 : ret = sysfs_create_group(&cpu_subsys.dev_root->kobj,
2548 : &cpuhp_cpu_root_attr_group);
2549 : if (ret)
2550 : return ret;
2551 :
2552 : for_each_possible_cpu(cpu) {
2553 : struct device *dev = get_cpu_device(cpu);
2554 :
2555 : if (!dev)
2556 : continue;
2557 : ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
2558 : if (ret)
2559 : return ret;
2560 : }
2561 : return 0;
2562 : }
2563 : device_initcall(cpuhp_sysfs_init);
2564 : #endif /* CONFIG_SYSFS && CONFIG_HOTPLUG_CPU */
2565 :
2566 : /*
2567 : * cpu_bit_bitmap[] is a special, "compressed" data structure that
2568 : * represents all NR_CPUS bits binary values of 1<<nr.
2569 : *
2570 : * It is used by cpumask_of() to get a constant address to a CPU
2571 : * mask value that has a single bit set only.
2572 : */
2573 :
2574 : /* cpu_bit_bitmap[0] is empty - so we can back into it */
2575 : #define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x))
2576 : #define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
2577 : #define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
2578 : #define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
2579 :
2580 : const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
2581 :
2582 : MASK_DECLARE_8(0), MASK_DECLARE_8(8),
2583 : MASK_DECLARE_8(16), MASK_DECLARE_8(24),
2584 : #if BITS_PER_LONG > 32
2585 : MASK_DECLARE_8(32), MASK_DECLARE_8(40),
2586 : MASK_DECLARE_8(48), MASK_DECLARE_8(56),
2587 : #endif
2588 : };
2589 : EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
2590 :
2591 : const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
2592 : EXPORT_SYMBOL(cpu_all_bits);
2593 :
2594 : #ifdef CONFIG_INIT_ALL_POSSIBLE
2595 : struct cpumask __cpu_possible_mask __read_mostly
2596 : = {CPU_BITS_ALL};
2597 : #else
2598 : struct cpumask __cpu_possible_mask __read_mostly;
2599 : #endif
2600 : EXPORT_SYMBOL(__cpu_possible_mask);
2601 :
2602 : struct cpumask __cpu_online_mask __read_mostly;
2603 : EXPORT_SYMBOL(__cpu_online_mask);
2604 :
2605 : struct cpumask __cpu_present_mask __read_mostly;
2606 : EXPORT_SYMBOL(__cpu_present_mask);
2607 :
2608 : struct cpumask __cpu_active_mask __read_mostly;
2609 : EXPORT_SYMBOL(__cpu_active_mask);
2610 :
2611 : struct cpumask __cpu_dying_mask __read_mostly;
2612 : EXPORT_SYMBOL(__cpu_dying_mask);
2613 :
2614 : atomic_t __num_online_cpus __read_mostly;
2615 : EXPORT_SYMBOL(__num_online_cpus);
2616 :
2617 0 : void init_cpu_present(const struct cpumask *src)
2618 : {
2619 0 : cpumask_copy(&__cpu_present_mask, src);
2620 0 : }
2621 :
2622 0 : void init_cpu_possible(const struct cpumask *src)
2623 : {
2624 0 : cpumask_copy(&__cpu_possible_mask, src);
2625 0 : }
2626 :
2627 0 : void init_cpu_online(const struct cpumask *src)
2628 : {
2629 0 : cpumask_copy(&__cpu_online_mask, src);
2630 0 : }
2631 :
2632 0 : void set_cpu_online(unsigned int cpu, bool online)
2633 : {
2634 : /*
2635 : * atomic_inc/dec() is required to handle the horrid abuse of this
2636 : * function by the reboot and kexec code which invoke it from
2637 : * IPI/NMI broadcasts when shutting down CPUs. Invocation from
2638 : * regular CPU hotplug is properly serialized.
2639 : *
2640 : * Note, that the fact that __num_online_cpus is of type atomic_t
2641 : * does not protect readers which are not serialized against
2642 : * concurrent hotplug operations.
2643 : */
2644 0 : if (online) {
2645 2 : if (!cpumask_test_and_set_cpu(cpu, &__cpu_online_mask))
2646 : atomic_inc(&__num_online_cpus);
2647 : } else {
2648 0 : if (cpumask_test_and_clear_cpu(cpu, &__cpu_online_mask))
2649 : atomic_dec(&__num_online_cpus);
2650 : }
2651 0 : }
2652 :
2653 : /*
2654 : * Activate the first processor.
2655 : */
2656 1 : void __init boot_cpu_init(void)
2657 : {
2658 1 : int cpu = smp_processor_id();
2659 :
2660 : /* Mark the boot cpu "present", "online" etc for SMP and UP case */
2661 2 : set_cpu_online(cpu, true);
2662 2 : set_cpu_active(cpu, true);
2663 2 : set_cpu_present(cpu, true);
2664 2 : set_cpu_possible(cpu, true);
2665 :
2666 : #ifdef CONFIG_SMP
2667 : __boot_cpu_id = cpu;
2668 : #endif
2669 1 : }
2670 :
2671 : /*
2672 : * Must be called _AFTER_ setting up the per_cpu areas
2673 : */
2674 1 : void __init boot_cpu_hotplug_init(void)
2675 : {
2676 : #ifdef CONFIG_SMP
2677 : cpumask_set_cpu(smp_processor_id(), &cpus_booted_once_mask);
2678 : #endif
2679 3 : this_cpu_write(cpuhp_state.state, CPUHP_ONLINE);
2680 1 : }
2681 :
2682 : /*
2683 : * These are used for a global "mitigations=" cmdline option for toggling
2684 : * optional CPU mitigations.
2685 : */
2686 : enum cpu_mitigations {
2687 : CPU_MITIGATIONS_OFF,
2688 : CPU_MITIGATIONS_AUTO,
2689 : CPU_MITIGATIONS_AUTO_NOSMT,
2690 : };
2691 :
2692 : static enum cpu_mitigations cpu_mitigations __ro_after_init =
2693 : CPU_MITIGATIONS_AUTO;
2694 :
2695 0 : static int __init mitigations_parse_cmdline(char *arg)
2696 : {
2697 0 : if (!strcmp(arg, "off"))
2698 0 : cpu_mitigations = CPU_MITIGATIONS_OFF;
2699 0 : else if (!strcmp(arg, "auto"))
2700 0 : cpu_mitigations = CPU_MITIGATIONS_AUTO;
2701 0 : else if (!strcmp(arg, "auto,nosmt"))
2702 0 : cpu_mitigations = CPU_MITIGATIONS_AUTO_NOSMT;
2703 : else
2704 0 : pr_crit("Unsupported mitigations=%s, system may still be vulnerable\n",
2705 : arg);
2706 :
2707 0 : return 0;
2708 : }
2709 : early_param("mitigations", mitigations_parse_cmdline);
2710 :
2711 : /* mitigations=off */
2712 0 : bool cpu_mitigations_off(void)
2713 : {
2714 0 : return cpu_mitigations == CPU_MITIGATIONS_OFF;
2715 : }
2716 : EXPORT_SYMBOL_GPL(cpu_mitigations_off);
2717 :
2718 : /* mitigations=auto,nosmt */
2719 0 : bool cpu_mitigations_auto_nosmt(void)
2720 : {
2721 0 : return cpu_mitigations == CPU_MITIGATIONS_AUTO_NOSMT;
2722 : }
2723 : EXPORT_SYMBOL_GPL(cpu_mitigations_auto_nosmt);
|
