Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : /*
3 : * linux/kernel/softirq.c
4 : *
5 : * Copyright (C) 1992 Linus Torvalds
6 : *
7 : * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
8 : */
9 :
10 : #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
11 :
12 : #include <linux/export.h>
13 : #include <linux/kernel_stat.h>
14 : #include <linux/interrupt.h>
15 : #include <linux/init.h>
16 : #include <linux/local_lock.h>
17 : #include <linux/mm.h>
18 : #include <linux/notifier.h>
19 : #include <linux/percpu.h>
20 : #include <linux/cpu.h>
21 : #include <linux/freezer.h>
22 : #include <linux/kthread.h>
23 : #include <linux/rcupdate.h>
24 : #include <linux/ftrace.h>
25 : #include <linux/smp.h>
26 : #include <linux/smpboot.h>
27 : #include <linux/tick.h>
28 : #include <linux/irq.h>
29 : #include <linux/wait_bit.h>
30 :
31 : #include <asm/softirq_stack.h>
32 :
33 : #define CREATE_TRACE_POINTS
34 : #include <trace/events/irq.h>
35 :
36 : /*
37 : - No shared variables, all the data are CPU local.
38 : - If a softirq needs serialization, let it serialize itself
39 : by its own spinlocks.
40 : - Even if softirq is serialized, only local cpu is marked for
41 : execution. Hence, we get something sort of weak cpu binding.
42 : Though it is still not clear, will it result in better locality
43 : or will not.
44 :
45 : Examples:
46 : - NET RX softirq. It is multithreaded and does not require
47 : any global serialization.
48 : - NET TX softirq. It kicks software netdevice queues, hence
49 : it is logically serialized per device, but this serialization
50 : is invisible to common code.
51 : - Tasklets: serialized wrt itself.
52 : */
53 :
54 : #ifndef __ARCH_IRQ_STAT
55 : DEFINE_PER_CPU_ALIGNED(irq_cpustat_t, irq_stat);
56 : EXPORT_PER_CPU_SYMBOL(irq_stat);
57 : #endif
58 :
59 : static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
60 :
61 : DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
62 :
63 : const char * const softirq_to_name[NR_SOFTIRQS] = {
64 : "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "IRQ_POLL",
65 : "TASKLET", "SCHED", "HRTIMER", "RCU"
66 : };
67 :
68 : /*
69 : * we cannot loop indefinitely here to avoid userspace starvation,
70 : * but we also don't want to introduce a worst case 1/HZ latency
71 : * to the pending events, so lets the scheduler to balance
72 : * the softirq load for us.
73 : */
74 : static void wakeup_softirqd(void)
75 : {
76 : /* Interrupts are disabled: no need to stop preemption */
77 186 : struct task_struct *tsk = __this_cpu_read(ksoftirqd);
78 :
79 186 : if (tsk)
80 186 : wake_up_process(tsk);
81 : }
82 :
83 : /*
84 : * If ksoftirqd is scheduled, we do not want to process pending softirqs
85 : * right now. Let ksoftirqd handle this at its own rate, to get fairness,
86 : * unless we're doing some of the synchronous softirqs.
87 : */
88 : #define SOFTIRQ_NOW_MASK ((1 << HI_SOFTIRQ) | (1 << TASKLET_SOFTIRQ))
89 0 : static bool ksoftirqd_running(unsigned long pending)
90 : {
91 0 : struct task_struct *tsk = __this_cpu_read(ksoftirqd);
92 :
93 0 : if (pending & SOFTIRQ_NOW_MASK)
94 : return false;
95 0 : return tsk && task_is_running(tsk) && !__kthread_should_park(tsk);
96 : }
97 :
98 : #ifdef CONFIG_TRACE_IRQFLAGS
99 : DEFINE_PER_CPU(int, hardirqs_enabled);
100 : DEFINE_PER_CPU(int, hardirq_context);
101 : EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled);
102 : EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context);
103 : #endif
104 :
105 : /*
106 : * SOFTIRQ_OFFSET usage:
107 : *
108 : * On !RT kernels 'count' is the preempt counter, on RT kernels this applies
109 : * to a per CPU counter and to task::softirqs_disabled_cnt.
110 : *
111 : * - count is changed by SOFTIRQ_OFFSET on entering or leaving softirq
112 : * processing.
113 : *
114 : * - count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
115 : * on local_bh_disable or local_bh_enable.
116 : *
117 : * This lets us distinguish between whether we are currently processing
118 : * softirq and whether we just have bh disabled.
119 : */
120 : #ifdef CONFIG_PREEMPT_RT
121 :
122 : /*
123 : * RT accounts for BH disabled sections in task::softirqs_disabled_cnt and
124 : * also in per CPU softirq_ctrl::cnt. This is necessary to allow tasks in a
125 : * softirq disabled section to be preempted.
126 : *
127 : * The per task counter is used for softirq_count(), in_softirq() and
128 : * in_serving_softirqs() because these counts are only valid when the task
129 : * holding softirq_ctrl::lock is running.
130 : *
131 : * The per CPU counter prevents pointless wakeups of ksoftirqd in case that
132 : * the task which is in a softirq disabled section is preempted or blocks.
133 : */
134 : struct softirq_ctrl {
135 : local_lock_t lock;
136 : int cnt;
137 : };
138 :
139 : static DEFINE_PER_CPU(struct softirq_ctrl, softirq_ctrl) = {
140 : .lock = INIT_LOCAL_LOCK(softirq_ctrl.lock),
141 : };
142 :
143 : /**
144 : * local_bh_blocked() - Check for idle whether BH processing is blocked
145 : *
146 : * Returns false if the per CPU softirq::cnt is 0 otherwise true.
147 : *
148 : * This is invoked from the idle task to guard against false positive
149 : * softirq pending warnings, which would happen when the task which holds
150 : * softirq_ctrl::lock was the only running task on the CPU and blocks on
151 : * some other lock.
152 : */
153 : bool local_bh_blocked(void)
154 : {
155 : return __this_cpu_read(softirq_ctrl.cnt) != 0;
156 : }
157 :
158 : void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
159 : {
160 : unsigned long flags;
161 : int newcnt;
162 :
163 : WARN_ON_ONCE(in_hardirq());
164 :
165 : /* First entry of a task into a BH disabled section? */
166 : if (!current->softirq_disable_cnt) {
167 : if (preemptible()) {
168 : local_lock(&softirq_ctrl.lock);
169 : /* Required to meet the RCU bottomhalf requirements. */
170 : rcu_read_lock();
171 : } else {
172 : DEBUG_LOCKS_WARN_ON(this_cpu_read(softirq_ctrl.cnt));
173 : }
174 : }
175 :
176 : /*
177 : * Track the per CPU softirq disabled state. On RT this is per CPU
178 : * state to allow preemption of bottom half disabled sections.
179 : */
180 : newcnt = __this_cpu_add_return(softirq_ctrl.cnt, cnt);
181 : /*
182 : * Reflect the result in the task state to prevent recursion on the
183 : * local lock and to make softirq_count() & al work.
184 : */
185 : current->softirq_disable_cnt = newcnt;
186 :
187 : if (IS_ENABLED(CONFIG_TRACE_IRQFLAGS) && newcnt == cnt) {
188 : raw_local_irq_save(flags);
189 : lockdep_softirqs_off(ip);
190 : raw_local_irq_restore(flags);
191 : }
192 : }
193 : EXPORT_SYMBOL(__local_bh_disable_ip);
194 :
195 : static void __local_bh_enable(unsigned int cnt, bool unlock)
196 : {
197 : unsigned long flags;
198 : int newcnt;
199 :
200 : DEBUG_LOCKS_WARN_ON(current->softirq_disable_cnt !=
201 : this_cpu_read(softirq_ctrl.cnt));
202 :
203 : if (IS_ENABLED(CONFIG_TRACE_IRQFLAGS) && softirq_count() == cnt) {
204 : raw_local_irq_save(flags);
205 : lockdep_softirqs_on(_RET_IP_);
206 : raw_local_irq_restore(flags);
207 : }
208 :
209 : newcnt = __this_cpu_sub_return(softirq_ctrl.cnt, cnt);
210 : current->softirq_disable_cnt = newcnt;
211 :
212 : if (!newcnt && unlock) {
213 : rcu_read_unlock();
214 : local_unlock(&softirq_ctrl.lock);
215 : }
216 : }
217 :
218 : void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
219 : {
220 : bool preempt_on = preemptible();
221 : unsigned long flags;
222 : u32 pending;
223 : int curcnt;
224 :
225 : WARN_ON_ONCE(in_hardirq());
226 : lockdep_assert_irqs_enabled();
227 :
228 : local_irq_save(flags);
229 : curcnt = __this_cpu_read(softirq_ctrl.cnt);
230 :
231 : /*
232 : * If this is not reenabling soft interrupts, no point in trying to
233 : * run pending ones.
234 : */
235 : if (curcnt != cnt)
236 : goto out;
237 :
238 : pending = local_softirq_pending();
239 : if (!pending || ksoftirqd_running(pending))
240 : goto out;
241 :
242 : /*
243 : * If this was called from non preemptible context, wake up the
244 : * softirq daemon.
245 : */
246 : if (!preempt_on) {
247 : wakeup_softirqd();
248 : goto out;
249 : }
250 :
251 : /*
252 : * Adjust softirq count to SOFTIRQ_OFFSET which makes
253 : * in_serving_softirq() become true.
254 : */
255 : cnt = SOFTIRQ_OFFSET;
256 : __local_bh_enable(cnt, false);
257 : __do_softirq();
258 :
259 : out:
260 : __local_bh_enable(cnt, preempt_on);
261 : local_irq_restore(flags);
262 : }
263 : EXPORT_SYMBOL(__local_bh_enable_ip);
264 :
265 : /*
266 : * Invoked from ksoftirqd_run() outside of the interrupt disabled section
267 : * to acquire the per CPU local lock for reentrancy protection.
268 : */
269 : static inline void ksoftirqd_run_begin(void)
270 : {
271 : __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
272 : local_irq_disable();
273 : }
274 :
275 : /* Counterpart to ksoftirqd_run_begin() */
276 : static inline void ksoftirqd_run_end(void)
277 : {
278 : __local_bh_enable(SOFTIRQ_OFFSET, true);
279 : WARN_ON_ONCE(in_interrupt());
280 : local_irq_enable();
281 : }
282 :
283 : static inline void softirq_handle_begin(void) { }
284 : static inline void softirq_handle_end(void) { }
285 :
286 : static inline bool should_wake_ksoftirqd(void)
287 : {
288 : return !this_cpu_read(softirq_ctrl.cnt);
289 : }
290 :
291 : static inline void invoke_softirq(void)
292 : {
293 : if (should_wake_ksoftirqd())
294 : wakeup_softirqd();
295 : }
296 :
297 : #else /* CONFIG_PREEMPT_RT */
298 :
299 : /*
300 : * This one is for softirq.c-internal use, where hardirqs are disabled
301 : * legitimately:
302 : */
303 : #ifdef CONFIG_TRACE_IRQFLAGS
304 : void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
305 : {
306 : unsigned long flags;
307 :
308 : WARN_ON_ONCE(in_hardirq());
309 :
310 : raw_local_irq_save(flags);
311 : /*
312 : * The preempt tracer hooks into preempt_count_add and will break
313 : * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
314 : * is set and before current->softirq_enabled is cleared.
315 : * We must manually increment preempt_count here and manually
316 : * call the trace_preempt_off later.
317 : */
318 : __preempt_count_add(cnt);
319 : /*
320 : * Were softirqs turned off above:
321 : */
322 : if (softirq_count() == (cnt & SOFTIRQ_MASK))
323 : lockdep_softirqs_off(ip);
324 : raw_local_irq_restore(flags);
325 :
326 : if (preempt_count() == cnt) {
327 : #ifdef CONFIG_DEBUG_PREEMPT
328 : current->preempt_disable_ip = get_lock_parent_ip();
329 : #endif
330 : trace_preempt_off(CALLER_ADDR0, get_lock_parent_ip());
331 : }
332 : }
333 : EXPORT_SYMBOL(__local_bh_disable_ip);
334 : #endif /* CONFIG_TRACE_IRQFLAGS */
335 :
336 : static void __local_bh_enable(unsigned int cnt)
337 : {
338 : lockdep_assert_irqs_disabled();
339 :
340 : if (preempt_count() == cnt)
341 : trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip());
342 :
343 93 : if (softirq_count() == (cnt & SOFTIRQ_MASK))
344 93 : lockdep_softirqs_on(_RET_IP_);
345 :
346 186 : __preempt_count_sub(cnt);
347 : }
348 :
349 : /*
350 : * Special-case - softirqs can safely be enabled by __do_softirq(),
351 : * without processing still-pending softirqs:
352 : */
353 0 : void _local_bh_enable(void)
354 : {
355 0 : WARN_ON_ONCE(in_hardirq());
356 0 : __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
357 0 : }
358 : EXPORT_SYMBOL(_local_bh_enable);
359 :
360 456 : void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
361 : {
362 456 : WARN_ON_ONCE(in_hardirq());
363 : lockdep_assert_irqs_enabled();
364 : #ifdef CONFIG_TRACE_IRQFLAGS
365 : local_irq_disable();
366 : #endif
367 : /*
368 : * Are softirqs going to be turned on now:
369 : */
370 : if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
371 : lockdep_softirqs_on(ip);
372 : /*
373 : * Keep preemption disabled until we are done with
374 : * softirq processing:
375 : */
376 912 : __preempt_count_sub(cnt - 1);
377 :
378 456 : if (unlikely(!in_interrupt() && local_softirq_pending())) {
379 : /*
380 : * Run softirq if any pending. And do it in its own stack
381 : * as we may be calling this deep in a task call stack already.
382 : */
383 0 : do_softirq();
384 : }
385 :
386 456 : preempt_count_dec();
387 : #ifdef CONFIG_TRACE_IRQFLAGS
388 : local_irq_enable();
389 : #endif
390 : preempt_check_resched();
391 456 : }
392 : EXPORT_SYMBOL(__local_bh_enable_ip);
393 :
394 : static inline void softirq_handle_begin(void)
395 : {
396 186 : __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
397 : }
398 :
399 93 : static inline void softirq_handle_end(void)
400 : {
401 93 : __local_bh_enable(SOFTIRQ_OFFSET);
402 93 : WARN_ON_ONCE(in_interrupt());
403 93 : }
404 :
405 : static inline void ksoftirqd_run_begin(void)
406 : {
407 : local_irq_disable();
408 : }
409 :
410 : static inline void ksoftirqd_run_end(void)
411 : {
412 : local_irq_enable();
413 : }
414 :
415 : static inline bool should_wake_ksoftirqd(void)
416 : {
417 : return true;
418 : }
419 :
420 0 : static inline void invoke_softirq(void)
421 : {
422 0 : if (ksoftirqd_running(local_softirq_pending()))
423 : return;
424 :
425 : if (!force_irqthreads() || !__this_cpu_read(ksoftirqd)) {
426 : #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
427 : /*
428 : * We can safely execute softirq on the current stack if
429 : * it is the irq stack, because it should be near empty
430 : * at this stage.
431 : */
432 : __do_softirq();
433 : #else
434 : /*
435 : * Otherwise, irq_exit() is called on the task stack that can
436 : * be potentially deep already. So call softirq in its own stack
437 : * to prevent from any overrun.
438 : */
439 : do_softirq_own_stack();
440 : #endif
441 : } else {
442 : wakeup_softirqd();
443 : }
444 : }
445 :
446 0 : asmlinkage __visible void do_softirq(void)
447 : {
448 : __u32 pending;
449 : unsigned long flags;
450 :
451 0 : if (in_interrupt())
452 : return;
453 :
454 0 : local_irq_save(flags);
455 :
456 0 : pending = local_softirq_pending();
457 :
458 0 : if (pending && !ksoftirqd_running(pending))
459 : do_softirq_own_stack();
460 :
461 0 : local_irq_restore(flags);
462 : }
463 :
464 : #endif /* !CONFIG_PREEMPT_RT */
465 :
466 : /*
467 : * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
468 : * but break the loop if need_resched() is set or after 2 ms.
469 : * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
470 : * certain cases, such as stop_machine(), jiffies may cease to
471 : * increment and so we need the MAX_SOFTIRQ_RESTART limit as
472 : * well to make sure we eventually return from this method.
473 : *
474 : * These limits have been established via experimentation.
475 : * The two things to balance is latency against fairness -
476 : * we want to handle softirqs as soon as possible, but they
477 : * should not be able to lock up the box.
478 : */
479 : #define MAX_SOFTIRQ_TIME msecs_to_jiffies(2)
480 : #define MAX_SOFTIRQ_RESTART 10
481 :
482 : #ifdef CONFIG_TRACE_IRQFLAGS
483 : /*
484 : * When we run softirqs from irq_exit() and thus on the hardirq stack we need
485 : * to keep the lockdep irq context tracking as tight as possible in order to
486 : * not miss-qualify lock contexts and miss possible deadlocks.
487 : */
488 :
489 : static inline bool lockdep_softirq_start(void)
490 : {
491 : bool in_hardirq = false;
492 :
493 : if (lockdep_hardirq_context()) {
494 : in_hardirq = true;
495 : lockdep_hardirq_exit();
496 : }
497 :
498 : lockdep_softirq_enter();
499 :
500 : return in_hardirq;
501 : }
502 :
503 : static inline void lockdep_softirq_end(bool in_hardirq)
504 : {
505 : lockdep_softirq_exit();
506 :
507 : if (in_hardirq)
508 : lockdep_hardirq_enter();
509 : }
510 : #else
511 : static inline bool lockdep_softirq_start(void) { return false; }
512 : static inline void lockdep_softirq_end(bool in_hardirq) { }
513 : #endif
514 :
515 93 : asmlinkage __visible void __softirq_entry __do_softirq(void)
516 : {
517 93 : unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
518 93 : unsigned long old_flags = current->flags;
519 93 : int max_restart = MAX_SOFTIRQ_RESTART;
520 : struct softirq_action *h;
521 : bool in_hardirq;
522 : __u32 pending;
523 : int softirq_bit;
524 :
525 : /*
526 : * Mask out PF_MEMALLOC as the current task context is borrowed for the
527 : * softirq. A softirq handled, such as network RX, might set PF_MEMALLOC
528 : * again if the socket is related to swapping.
529 : */
530 93 : current->flags &= ~PF_MEMALLOC;
531 :
532 93 : pending = local_softirq_pending();
533 :
534 93 : softirq_handle_begin();
535 93 : in_hardirq = lockdep_softirq_start();
536 93 : account_softirq_enter(current);
537 :
538 : restart:
539 : /* Reset the pending bitmask before enabling irqs */
540 185 : set_softirq_pending(0);
541 :
542 : local_irq_enable();
543 :
544 185 : h = softirq_vec;
545 :
546 925 : while ((softirq_bit = ffs(pending))) {
547 : unsigned int vec_nr;
548 : int prev_count;
549 :
550 185 : h += softirq_bit - 1;
551 :
552 185 : vec_nr = h - softirq_vec;
553 185 : prev_count = preempt_count();
554 :
555 185 : kstat_incr_softirqs_this_cpu(vec_nr);
556 :
557 185 : trace_softirq_entry(vec_nr);
558 185 : h->action(h);
559 185 : trace_softirq_exit(vec_nr);
560 185 : if (unlikely(prev_count != preempt_count())) {
561 0 : pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
562 : vec_nr, softirq_to_name[vec_nr], h->action,
563 : prev_count, preempt_count());
564 : preempt_count_set(prev_count);
565 : }
566 185 : h++;
567 185 : pending >>= softirq_bit;
568 : }
569 :
570 185 : if (!IS_ENABLED(CONFIG_PREEMPT_RT) &&
571 370 : __this_cpu_read(ksoftirqd) == current)
572 : rcu_softirq_qs();
573 :
574 : local_irq_disable();
575 :
576 185 : pending = local_softirq_pending();
577 185 : if (pending) {
578 184 : if (time_before(jiffies, end) && !need_resched() &&
579 : --max_restart)
580 : goto restart;
581 :
582 : wakeup_softirqd();
583 : }
584 :
585 93 : account_softirq_exit(current);
586 93 : lockdep_softirq_end(in_hardirq);
587 93 : softirq_handle_end();
588 93 : current_restore_flags(old_flags, PF_MEMALLOC);
589 93 : }
590 :
591 : /**
592 : * irq_enter_rcu - Enter an interrupt context with RCU watching
593 : */
594 13 : void irq_enter_rcu(void)
595 : {
596 13 : __irq_enter_raw();
597 :
598 26 : if (tick_nohz_full_cpu(smp_processor_id()) ||
599 26 : (is_idle_task(current) && (irq_count() == HARDIRQ_OFFSET)))
600 : tick_irq_enter();
601 :
602 13 : account_hardirq_enter(current);
603 13 : }
604 :
605 : /**
606 : * irq_enter - Enter an interrupt context including RCU update
607 : */
608 13 : void irq_enter(void)
609 : {
610 : rcu_irq_enter();
611 13 : irq_enter_rcu();
612 13 : }
613 :
614 : static inline void tick_irq_exit(void)
615 : {
616 : #ifdef CONFIG_NO_HZ_COMMON
617 : int cpu = smp_processor_id();
618 :
619 : /* Make sure that timer wheel updates are propagated */
620 : if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
621 : if (!in_hardirq())
622 : tick_nohz_irq_exit();
623 : }
624 : #endif
625 : }
626 :
627 13 : static inline void __irq_exit_rcu(void)
628 : {
629 : #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
630 : local_irq_disable();
631 : #else
632 : lockdep_assert_irqs_disabled();
633 : #endif
634 26 : account_hardirq_exit(current);
635 13 : preempt_count_sub(HARDIRQ_OFFSET);
636 13 : if (!in_interrupt() && local_softirq_pending())
637 0 : invoke_softirq();
638 :
639 : tick_irq_exit();
640 13 : }
641 :
642 : /**
643 : * irq_exit_rcu() - Exit an interrupt context without updating RCU
644 : *
645 : * Also processes softirqs if needed and possible.
646 : */
647 0 : void irq_exit_rcu(void)
648 : {
649 0 : __irq_exit_rcu();
650 : /* must be last! */
651 : lockdep_hardirq_exit();
652 0 : }
653 :
654 : /**
655 : * irq_exit - Exit an interrupt context, update RCU and lockdep
656 : *
657 : * Also processes softirqs if needed and possible.
658 : */
659 13 : void irq_exit(void)
660 : {
661 13 : __irq_exit_rcu();
662 : rcu_irq_exit();
663 : /* must be last! */
664 : lockdep_hardirq_exit();
665 13 : }
666 :
667 : /*
668 : * This function must run with irqs disabled!
669 : */
670 278 : inline void raise_softirq_irqoff(unsigned int nr)
671 : {
672 278 : __raise_softirq_irqoff(nr);
673 :
674 : /*
675 : * If we're in an interrupt or softirq, we're done
676 : * (this also catches softirq-disabled code). We will
677 : * actually run the softirq once we return from
678 : * the irq or softirq.
679 : *
680 : * Otherwise we wake up ksoftirqd to make sure we
681 : * schedule the softirq soon.
682 : */
683 278 : if (!in_interrupt() && should_wake_ksoftirqd())
684 : wakeup_softirqd();
685 278 : }
686 :
687 0 : void raise_softirq(unsigned int nr)
688 : {
689 : unsigned long flags;
690 :
691 0 : local_irq_save(flags);
692 0 : raise_softirq_irqoff(nr);
693 0 : local_irq_restore(flags);
694 0 : }
695 :
696 0 : void __raise_softirq_irqoff(unsigned int nr)
697 : {
698 : lockdep_assert_irqs_disabled();
699 278 : trace_softirq_raise(nr);
700 278 : or_softirq_pending(1UL << nr);
701 0 : }
702 :
703 4 : void open_softirq(int nr, void (*action)(struct softirq_action *))
704 : {
705 6 : softirq_vec[nr].action = action;
706 4 : }
707 :
708 : /*
709 : * Tasklets
710 : */
711 : struct tasklet_head {
712 : struct tasklet_struct *head;
713 : struct tasklet_struct **tail;
714 : };
715 :
716 : static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
717 : static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
718 :
719 0 : static void __tasklet_schedule_common(struct tasklet_struct *t,
720 : struct tasklet_head __percpu *headp,
721 : unsigned int softirq_nr)
722 : {
723 : struct tasklet_head *head;
724 : unsigned long flags;
725 :
726 0 : local_irq_save(flags);
727 0 : head = this_cpu_ptr(headp);
728 0 : t->next = NULL;
729 0 : *head->tail = t;
730 0 : head->tail = &(t->next);
731 0 : raise_softirq_irqoff(softirq_nr);
732 0 : local_irq_restore(flags);
733 0 : }
734 :
735 0 : void __tasklet_schedule(struct tasklet_struct *t)
736 : {
737 0 : __tasklet_schedule_common(t, &tasklet_vec,
738 : TASKLET_SOFTIRQ);
739 0 : }
740 : EXPORT_SYMBOL(__tasklet_schedule);
741 :
742 0 : void __tasklet_hi_schedule(struct tasklet_struct *t)
743 : {
744 0 : __tasklet_schedule_common(t, &tasklet_hi_vec,
745 : HI_SOFTIRQ);
746 0 : }
747 : EXPORT_SYMBOL(__tasklet_hi_schedule);
748 :
749 0 : static bool tasklet_clear_sched(struct tasklet_struct *t)
750 : {
751 0 : if (test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) {
752 0 : wake_up_var(&t->state);
753 0 : return true;
754 : }
755 :
756 0 : WARN_ONCE(1, "tasklet SCHED state not set: %s %pS\n",
757 : t->use_callback ? "callback" : "func",
758 : t->use_callback ? (void *)t->callback : (void *)t->func);
759 :
760 : return false;
761 : }
762 :
763 0 : static void tasklet_action_common(struct softirq_action *a,
764 : struct tasklet_head *tl_head,
765 : unsigned int softirq_nr)
766 : {
767 : struct tasklet_struct *list;
768 :
769 : local_irq_disable();
770 0 : list = tl_head->head;
771 0 : tl_head->head = NULL;
772 0 : tl_head->tail = &tl_head->head;
773 : local_irq_enable();
774 :
775 0 : while (list) {
776 0 : struct tasklet_struct *t = list;
777 :
778 0 : list = list->next;
779 :
780 0 : if (tasklet_trylock(t)) {
781 0 : if (!atomic_read(&t->count)) {
782 0 : if (tasklet_clear_sched(t)) {
783 0 : if (t->use_callback)
784 0 : t->callback(t);
785 : else
786 0 : t->func(t->data);
787 : }
788 0 : tasklet_unlock(t);
789 0 : continue;
790 : }
791 : tasklet_unlock(t);
792 : }
793 :
794 : local_irq_disable();
795 0 : t->next = NULL;
796 0 : *tl_head->tail = t;
797 0 : tl_head->tail = &t->next;
798 0 : __raise_softirq_irqoff(softirq_nr);
799 : local_irq_enable();
800 : }
801 0 : }
802 :
803 0 : static __latent_entropy void tasklet_action(struct softirq_action *a)
804 : {
805 0 : tasklet_action_common(a, this_cpu_ptr(&tasklet_vec), TASKLET_SOFTIRQ);
806 0 : }
807 :
808 0 : static __latent_entropy void tasklet_hi_action(struct softirq_action *a)
809 : {
810 0 : tasklet_action_common(a, this_cpu_ptr(&tasklet_hi_vec), HI_SOFTIRQ);
811 0 : }
812 :
813 0 : void tasklet_setup(struct tasklet_struct *t,
814 : void (*callback)(struct tasklet_struct *))
815 : {
816 0 : t->next = NULL;
817 0 : t->state = 0;
818 0 : atomic_set(&t->count, 0);
819 0 : t->callback = callback;
820 0 : t->use_callback = true;
821 0 : t->data = 0;
822 0 : }
823 : EXPORT_SYMBOL(tasklet_setup);
824 :
825 0 : void tasklet_init(struct tasklet_struct *t,
826 : void (*func)(unsigned long), unsigned long data)
827 : {
828 0 : t->next = NULL;
829 0 : t->state = 0;
830 0 : atomic_set(&t->count, 0);
831 0 : t->func = func;
832 0 : t->use_callback = false;
833 0 : t->data = data;
834 0 : }
835 : EXPORT_SYMBOL(tasklet_init);
836 :
837 : #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)
838 : /*
839 : * Do not use in new code. Waiting for tasklets from atomic contexts is
840 : * error prone and should be avoided.
841 : */
842 : void tasklet_unlock_spin_wait(struct tasklet_struct *t)
843 : {
844 : while (test_bit(TASKLET_STATE_RUN, &(t)->state)) {
845 : if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
846 : /*
847 : * Prevent a live lock when current preempted soft
848 : * interrupt processing or prevents ksoftirqd from
849 : * running. If the tasklet runs on a different CPU
850 : * then this has no effect other than doing the BH
851 : * disable/enable dance for nothing.
852 : */
853 : local_bh_disable();
854 : local_bh_enable();
855 : } else {
856 : cpu_relax();
857 : }
858 : }
859 : }
860 : EXPORT_SYMBOL(tasklet_unlock_spin_wait);
861 : #endif
862 :
863 0 : void tasklet_kill(struct tasklet_struct *t)
864 : {
865 0 : if (in_interrupt())
866 0 : pr_notice("Attempt to kill tasklet from interrupt\n");
867 :
868 0 : while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
869 0 : wait_var_event(&t->state, !test_bit(TASKLET_STATE_SCHED, &t->state));
870 :
871 0 : tasklet_unlock_wait(t);
872 0 : tasklet_clear_sched(t);
873 0 : }
874 : EXPORT_SYMBOL(tasklet_kill);
875 :
876 : #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)
877 : void tasklet_unlock(struct tasklet_struct *t)
878 : {
879 : smp_mb__before_atomic();
880 : clear_bit(TASKLET_STATE_RUN, &t->state);
881 : smp_mb__after_atomic();
882 : wake_up_var(&t->state);
883 : }
884 : EXPORT_SYMBOL_GPL(tasklet_unlock);
885 :
886 : void tasklet_unlock_wait(struct tasklet_struct *t)
887 : {
888 : wait_var_event(&t->state, !test_bit(TASKLET_STATE_RUN, &t->state));
889 : }
890 : EXPORT_SYMBOL_GPL(tasklet_unlock_wait);
891 : #endif
892 :
893 1 : void __init softirq_init(void)
894 : {
895 : int cpu;
896 :
897 2 : for_each_possible_cpu(cpu) {
898 2 : per_cpu(tasklet_vec, cpu).tail =
899 1 : &per_cpu(tasklet_vec, cpu).head;
900 2 : per_cpu(tasklet_hi_vec, cpu).tail =
901 1 : &per_cpu(tasklet_hi_vec, cpu).head;
902 : }
903 :
904 1 : open_softirq(TASKLET_SOFTIRQ, tasklet_action);
905 1 : open_softirq(HI_SOFTIRQ, tasklet_hi_action);
906 1 : }
907 :
908 186 : static int ksoftirqd_should_run(unsigned int cpu)
909 : {
910 186 : return local_softirq_pending();
911 : }
912 :
913 93 : static void run_ksoftirqd(unsigned int cpu)
914 : {
915 : ksoftirqd_run_begin();
916 93 : if (local_softirq_pending()) {
917 : /*
918 : * We can safely run softirq on inline stack, as we are not deep
919 : * in the task stack here.
920 : */
921 93 : __do_softirq();
922 : ksoftirqd_run_end();
923 93 : cond_resched();
924 93 : return;
925 : }
926 : ksoftirqd_run_end();
927 : }
928 :
929 : #ifdef CONFIG_HOTPLUG_CPU
930 : static int takeover_tasklets(unsigned int cpu)
931 : {
932 : /* CPU is dead, so no lock needed. */
933 : local_irq_disable();
934 :
935 : /* Find end, append list for that CPU. */
936 : if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
937 : *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
938 : __this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
939 : per_cpu(tasklet_vec, cpu).head = NULL;
940 : per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
941 : }
942 : raise_softirq_irqoff(TASKLET_SOFTIRQ);
943 :
944 : if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
945 : *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
946 : __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
947 : per_cpu(tasklet_hi_vec, cpu).head = NULL;
948 : per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
949 : }
950 : raise_softirq_irqoff(HI_SOFTIRQ);
951 :
952 : local_irq_enable();
953 : return 0;
954 : }
955 : #else
956 : #define takeover_tasklets NULL
957 : #endif /* CONFIG_HOTPLUG_CPU */
958 :
959 : static struct smp_hotplug_thread softirq_threads = {
960 : .store = &ksoftirqd,
961 : .thread_should_run = ksoftirqd_should_run,
962 : .thread_fn = run_ksoftirqd,
963 : .thread_comm = "ksoftirqd/%u",
964 : };
965 :
966 1 : static __init int spawn_ksoftirqd(void)
967 : {
968 1 : cpuhp_setup_state_nocalls(CPUHP_SOFTIRQ_DEAD, "softirq:dead", NULL,
969 : takeover_tasklets);
970 1 : BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
971 :
972 1 : return 0;
973 : }
974 : early_initcall(spawn_ksoftirqd);
975 :
976 : /*
977 : * [ These __weak aliases are kept in a separate compilation unit, so that
978 : * GCC does not inline them incorrectly. ]
979 : */
980 :
981 0 : int __init __weak early_irq_init(void)
982 : {
983 0 : return 0;
984 : }
985 :
986 0 : int __init __weak arch_probe_nr_irqs(void)
987 : {
988 0 : return NR_IRQS_LEGACY;
989 : }
990 :
991 1 : int __init __weak arch_early_irq_init(void)
992 : {
993 1 : return 0;
994 : }
995 :
996 0 : unsigned int __weak arch_dynirq_lower_bound(unsigned int from)
997 : {
998 0 : return from;
999 : }
|
