Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : #include <linux/kdebug.h>
3 : #include <linux/kprobes.h>
4 : #include <linux/export.h>
5 : #include <linux/notifier.h>
6 : #include <linux/rcupdate.h>
7 : #include <linux/vmalloc.h>
8 : #include <linux/reboot.h>
9 :
10 : /*
11 : * Notifier list for kernel code which wants to be called
12 : * at shutdown. This is used to stop any idling DMA operations
13 : * and the like.
14 : */
15 : BLOCKING_NOTIFIER_HEAD(reboot_notifier_list);
16 :
17 : /*
18 : * Notifier chain core routines. The exported routines below
19 : * are layered on top of these, with appropriate locking added.
20 : */
21 :
22 9 : static int notifier_chain_register(struct notifier_block **nl,
23 : struct notifier_block *n)
24 : {
25 20 : while ((*nl) != NULL) {
26 3 : if (unlikely((*nl) == n)) {
27 0 : WARN(1, "notifier callback %ps already registered",
28 : n->notifier_call);
29 0 : return -EEXIST;
30 : }
31 3 : if (n->priority > (*nl)->priority)
32 : break;
33 2 : nl = &((*nl)->next);
34 : }
35 9 : n->next = *nl;
36 9 : rcu_assign_pointer(*nl, n);
37 9 : return 0;
38 : }
39 :
40 : static int notifier_chain_unregister(struct notifier_block **nl,
41 : struct notifier_block *n)
42 : {
43 0 : while ((*nl) != NULL) {
44 0 : if ((*nl) == n) {
45 0 : rcu_assign_pointer(*nl, n->next);
46 : return 0;
47 : }
48 0 : nl = &((*nl)->next);
49 : }
50 : return -ENOENT;
51 : }
52 :
53 : /**
54 : * notifier_call_chain - Informs the registered notifiers about an event.
55 : * @nl: Pointer to head of the blocking notifier chain
56 : * @val: Value passed unmodified to notifier function
57 : * @v: Pointer passed unmodified to notifier function
58 : * @nr_to_call: Number of notifier functions to be called. Don't care
59 : * value of this parameter is -1.
60 : * @nr_calls: Records the number of notifications sent. Don't care
61 : * value of this field is NULL.
62 : * @returns: notifier_call_chain returns the value returned by the
63 : * last notifier function called.
64 : */
65 : static int notifier_call_chain(struct notifier_block **nl,
66 : unsigned long val, void *v,
67 : int nr_to_call, int *nr_calls)
68 : {
69 15 : int ret = NOTIFY_DONE;
70 : struct notifier_block *nb, *next_nb;
71 :
72 15 : nb = rcu_dereference_raw(*nl);
73 :
74 17 : while (nb && nr_to_call) {
75 2 : next_nb = rcu_dereference_raw(nb->next);
76 :
77 : #ifdef CONFIG_DEBUG_NOTIFIERS
78 : if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) {
79 : WARN(1, "Invalid notifier called!");
80 : nb = next_nb;
81 : continue;
82 : }
83 : #endif
84 2 : ret = nb->notifier_call(nb, val, v);
85 :
86 : if (nr_calls)
87 0 : (*nr_calls)++;
88 :
89 2 : if (ret & NOTIFY_STOP_MASK)
90 : break;
91 2 : nb = next_nb;
92 2 : nr_to_call--;
93 : }
94 : return ret;
95 : }
96 : NOKPROBE_SYMBOL(notifier_call_chain);
97 :
98 : /**
99 : * notifier_call_chain_robust - Inform the registered notifiers about an event
100 : * and rollback on error.
101 : * @nl: Pointer to head of the blocking notifier chain
102 : * @val_up: Value passed unmodified to the notifier function
103 : * @val_down: Value passed unmodified to the notifier function when recovering
104 : * from an error on @val_up
105 : * @v Pointer passed unmodified to the notifier function
106 : *
107 : * NOTE: It is important the @nl chain doesn't change between the two
108 : * invocations of notifier_call_chain() such that we visit the
109 : * exact same notifier callbacks; this rules out any RCU usage.
110 : *
111 : * Returns: the return value of the @val_up call.
112 : */
113 0 : static int notifier_call_chain_robust(struct notifier_block **nl,
114 : unsigned long val_up, unsigned long val_down,
115 : void *v)
116 : {
117 0 : int ret, nr = 0;
118 :
119 0 : ret = notifier_call_chain(nl, val_up, v, -1, &nr);
120 0 : if (ret & NOTIFY_STOP_MASK)
121 0 : notifier_call_chain(nl, val_down, v, nr-1, NULL);
122 :
123 0 : return ret;
124 : }
125 :
126 : /*
127 : * Atomic notifier chain routines. Registration and unregistration
128 : * use a spinlock, and call_chain is synchronized by RCU (no locks).
129 : */
130 :
131 : /**
132 : * atomic_notifier_chain_register - Add notifier to an atomic notifier chain
133 : * @nh: Pointer to head of the atomic notifier chain
134 : * @n: New entry in notifier chain
135 : *
136 : * Adds a notifier to an atomic notifier chain.
137 : *
138 : * Returns 0 on success, %-EEXIST on error.
139 : */
140 2 : int atomic_notifier_chain_register(struct atomic_notifier_head *nh,
141 : struct notifier_block *n)
142 : {
143 : unsigned long flags;
144 : int ret;
145 :
146 2 : spin_lock_irqsave(&nh->lock, flags);
147 2 : ret = notifier_chain_register(&nh->head, n);
148 4 : spin_unlock_irqrestore(&nh->lock, flags);
149 2 : return ret;
150 : }
151 : EXPORT_SYMBOL_GPL(atomic_notifier_chain_register);
152 :
153 : /**
154 : * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain
155 : * @nh: Pointer to head of the atomic notifier chain
156 : * @n: Entry to remove from notifier chain
157 : *
158 : * Removes a notifier from an atomic notifier chain.
159 : *
160 : * Returns zero on success or %-ENOENT on failure.
161 : */
162 0 : int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
163 : struct notifier_block *n)
164 : {
165 : unsigned long flags;
166 : int ret;
167 :
168 0 : spin_lock_irqsave(&nh->lock, flags);
169 0 : ret = notifier_chain_unregister(&nh->head, n);
170 0 : spin_unlock_irqrestore(&nh->lock, flags);
171 0 : synchronize_rcu();
172 0 : return ret;
173 : }
174 : EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
175 :
176 : /**
177 : * atomic_notifier_call_chain - Call functions in an atomic notifier chain
178 : * @nh: Pointer to head of the atomic notifier chain
179 : * @val: Value passed unmodified to notifier function
180 : * @v: Pointer passed unmodified to notifier function
181 : *
182 : * Calls each function in a notifier chain in turn. The functions
183 : * run in an atomic context, so they must not block.
184 : * This routine uses RCU to synchronize with changes to the chain.
185 : *
186 : * If the return value of the notifier can be and'ed
187 : * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain()
188 : * will return immediately, with the return value of
189 : * the notifier function which halted execution.
190 : * Otherwise the return value is the return value
191 : * of the last notifier function called.
192 : */
193 0 : int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
194 : unsigned long val, void *v)
195 : {
196 : int ret;
197 :
198 : rcu_read_lock();
199 0 : ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
200 : rcu_read_unlock();
201 :
202 0 : return ret;
203 : }
204 : EXPORT_SYMBOL_GPL(atomic_notifier_call_chain);
205 : NOKPROBE_SYMBOL(atomic_notifier_call_chain);
206 :
207 : /*
208 : * Blocking notifier chain routines. All access to the chain is
209 : * synchronized by an rwsem.
210 : */
211 :
212 : /**
213 : * blocking_notifier_chain_register - Add notifier to a blocking notifier chain
214 : * @nh: Pointer to head of the blocking notifier chain
215 : * @n: New entry in notifier chain
216 : *
217 : * Adds a notifier to a blocking notifier chain.
218 : * Must be called in process context.
219 : *
220 : * Returns 0 on success, %-EEXIST on error.
221 : */
222 5 : int blocking_notifier_chain_register(struct blocking_notifier_head *nh,
223 : struct notifier_block *n)
224 : {
225 : int ret;
226 :
227 : /*
228 : * This code gets used during boot-up, when task switching is
229 : * not yet working and interrupts must remain disabled. At
230 : * such times we must not call down_write().
231 : */
232 5 : if (unlikely(system_state == SYSTEM_BOOTING))
233 0 : return notifier_chain_register(&nh->head, n);
234 :
235 5 : down_write(&nh->rwsem);
236 5 : ret = notifier_chain_register(&nh->head, n);
237 5 : up_write(&nh->rwsem);
238 5 : return ret;
239 : }
240 : EXPORT_SYMBOL_GPL(blocking_notifier_chain_register);
241 :
242 : /**
243 : * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain
244 : * @nh: Pointer to head of the blocking notifier chain
245 : * @n: Entry to remove from notifier chain
246 : *
247 : * Removes a notifier from a blocking notifier chain.
248 : * Must be called from process context.
249 : *
250 : * Returns zero on success or %-ENOENT on failure.
251 : */
252 0 : int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh,
253 : struct notifier_block *n)
254 : {
255 : int ret;
256 :
257 : /*
258 : * This code gets used during boot-up, when task switching is
259 : * not yet working and interrupts must remain disabled. At
260 : * such times we must not call down_write().
261 : */
262 0 : if (unlikely(system_state == SYSTEM_BOOTING))
263 0 : return notifier_chain_unregister(&nh->head, n);
264 :
265 0 : down_write(&nh->rwsem);
266 0 : ret = notifier_chain_unregister(&nh->head, n);
267 0 : up_write(&nh->rwsem);
268 0 : return ret;
269 : }
270 : EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister);
271 :
272 0 : int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh,
273 : unsigned long val_up, unsigned long val_down, void *v)
274 : {
275 0 : int ret = NOTIFY_DONE;
276 :
277 : /*
278 : * We check the head outside the lock, but if this access is
279 : * racy then it does not matter what the result of the test
280 : * is, we re-check the list after having taken the lock anyway:
281 : */
282 0 : if (rcu_access_pointer(nh->head)) {
283 0 : down_read(&nh->rwsem);
284 0 : ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v);
285 0 : up_read(&nh->rwsem);
286 : }
287 0 : return ret;
288 : }
289 : EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust);
290 :
291 : /**
292 : * blocking_notifier_call_chain - Call functions in a blocking notifier chain
293 : * @nh: Pointer to head of the blocking notifier chain
294 : * @val: Value passed unmodified to notifier function
295 : * @v: Pointer passed unmodified to notifier function
296 : *
297 : * Calls each function in a notifier chain in turn. The functions
298 : * run in a process context, so they are allowed to block.
299 : *
300 : * If the return value of the notifier can be and'ed
301 : * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain()
302 : * will return immediately, with the return value of
303 : * the notifier function which halted execution.
304 : * Otherwise the return value is the return value
305 : * of the last notifier function called.
306 : */
307 5 : int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
308 : unsigned long val, void *v)
309 : {
310 5 : int ret = NOTIFY_DONE;
311 :
312 : /*
313 : * We check the head outside the lock, but if this access is
314 : * racy then it does not matter what the result of the test
315 : * is, we re-check the list after having taken the lock anyway:
316 : */
317 5 : if (rcu_access_pointer(nh->head)) {
318 1 : down_read(&nh->rwsem);
319 2 : ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
320 1 : up_read(&nh->rwsem);
321 : }
322 5 : return ret;
323 : }
324 : EXPORT_SYMBOL_GPL(blocking_notifier_call_chain);
325 :
326 : /*
327 : * Raw notifier chain routines. There is no protection;
328 : * the caller must provide it. Use at your own risk!
329 : */
330 :
331 : /**
332 : * raw_notifier_chain_register - Add notifier to a raw notifier chain
333 : * @nh: Pointer to head of the raw notifier chain
334 : * @n: New entry in notifier chain
335 : *
336 : * Adds a notifier to a raw notifier chain.
337 : * All locking must be provided by the caller.
338 : *
339 : * Returns 0 on success, %-EEXIST on error.
340 : */
341 2 : int raw_notifier_chain_register(struct raw_notifier_head *nh,
342 : struct notifier_block *n)
343 : {
344 2 : return notifier_chain_register(&nh->head, n);
345 : }
346 : EXPORT_SYMBOL_GPL(raw_notifier_chain_register);
347 :
348 : /**
349 : * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain
350 : * @nh: Pointer to head of the raw notifier chain
351 : * @n: Entry to remove from notifier chain
352 : *
353 : * Removes a notifier from a raw notifier chain.
354 : * All locking must be provided by the caller.
355 : *
356 : * Returns zero on success or %-ENOENT on failure.
357 : */
358 0 : int raw_notifier_chain_unregister(struct raw_notifier_head *nh,
359 : struct notifier_block *n)
360 : {
361 0 : return notifier_chain_unregister(&nh->head, n);
362 : }
363 : EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister);
364 :
365 0 : int raw_notifier_call_chain_robust(struct raw_notifier_head *nh,
366 : unsigned long val_up, unsigned long val_down, void *v)
367 : {
368 0 : return notifier_call_chain_robust(&nh->head, val_up, val_down, v);
369 : }
370 : EXPORT_SYMBOL_GPL(raw_notifier_call_chain_robust);
371 :
372 : /**
373 : * raw_notifier_call_chain - Call functions in a raw notifier chain
374 : * @nh: Pointer to head of the raw notifier chain
375 : * @val: Value passed unmodified to notifier function
376 : * @v: Pointer passed unmodified to notifier function
377 : *
378 : * Calls each function in a notifier chain in turn. The functions
379 : * run in an undefined context.
380 : * All locking must be provided by the caller.
381 : *
382 : * If the return value of the notifier can be and'ed
383 : * with %NOTIFY_STOP_MASK then raw_notifier_call_chain()
384 : * will return immediately, with the return value of
385 : * the notifier function which halted execution.
386 : * Otherwise the return value is the return value
387 : * of the last notifier function called.
388 : */
389 14 : int raw_notifier_call_chain(struct raw_notifier_head *nh,
390 : unsigned long val, void *v)
391 : {
392 28 : return notifier_call_chain(&nh->head, val, v, -1, NULL);
393 : }
394 : EXPORT_SYMBOL_GPL(raw_notifier_call_chain);
395 :
396 : #ifdef CONFIG_SRCU
397 : /*
398 : * SRCU notifier chain routines. Registration and unregistration
399 : * use a mutex, and call_chain is synchronized by SRCU (no locks).
400 : */
401 :
402 : /**
403 : * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain
404 : * @nh: Pointer to head of the SRCU notifier chain
405 : * @n: New entry in notifier chain
406 : *
407 : * Adds a notifier to an SRCU notifier chain.
408 : * Must be called in process context.
409 : *
410 : * Returns 0 on success, %-EEXIST on error.
411 : */
412 0 : int srcu_notifier_chain_register(struct srcu_notifier_head *nh,
413 : struct notifier_block *n)
414 : {
415 : int ret;
416 :
417 : /*
418 : * This code gets used during boot-up, when task switching is
419 : * not yet working and interrupts must remain disabled. At
420 : * such times we must not call mutex_lock().
421 : */
422 0 : if (unlikely(system_state == SYSTEM_BOOTING))
423 0 : return notifier_chain_register(&nh->head, n);
424 :
425 0 : mutex_lock(&nh->mutex);
426 0 : ret = notifier_chain_register(&nh->head, n);
427 0 : mutex_unlock(&nh->mutex);
428 0 : return ret;
429 : }
430 : EXPORT_SYMBOL_GPL(srcu_notifier_chain_register);
431 :
432 : /**
433 : * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain
434 : * @nh: Pointer to head of the SRCU notifier chain
435 : * @n: Entry to remove from notifier chain
436 : *
437 : * Removes a notifier from an SRCU notifier chain.
438 : * Must be called from process context.
439 : *
440 : * Returns zero on success or %-ENOENT on failure.
441 : */
442 0 : int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh,
443 : struct notifier_block *n)
444 : {
445 : int ret;
446 :
447 : /*
448 : * This code gets used during boot-up, when task switching is
449 : * not yet working and interrupts must remain disabled. At
450 : * such times we must not call mutex_lock().
451 : */
452 0 : if (unlikely(system_state == SYSTEM_BOOTING))
453 0 : return notifier_chain_unregister(&nh->head, n);
454 :
455 0 : mutex_lock(&nh->mutex);
456 0 : ret = notifier_chain_unregister(&nh->head, n);
457 0 : mutex_unlock(&nh->mutex);
458 0 : synchronize_srcu(&nh->srcu);
459 0 : return ret;
460 : }
461 : EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister);
462 :
463 : /**
464 : * srcu_notifier_call_chain - Call functions in an SRCU notifier chain
465 : * @nh: Pointer to head of the SRCU notifier chain
466 : * @val: Value passed unmodified to notifier function
467 : * @v: Pointer passed unmodified to notifier function
468 : *
469 : * Calls each function in a notifier chain in turn. The functions
470 : * run in a process context, so they are allowed to block.
471 : *
472 : * If the return value of the notifier can be and'ed
473 : * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain()
474 : * will return immediately, with the return value of
475 : * the notifier function which halted execution.
476 : * Otherwise the return value is the return value
477 : * of the last notifier function called.
478 : */
479 0 : int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
480 : unsigned long val, void *v)
481 : {
482 : int ret;
483 : int idx;
484 :
485 0 : idx = srcu_read_lock(&nh->srcu);
486 0 : ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
487 0 : srcu_read_unlock(&nh->srcu, idx);
488 0 : return ret;
489 : }
490 : EXPORT_SYMBOL_GPL(srcu_notifier_call_chain);
491 :
492 : /**
493 : * srcu_init_notifier_head - Initialize an SRCU notifier head
494 : * @nh: Pointer to head of the srcu notifier chain
495 : *
496 : * Unlike other sorts of notifier heads, SRCU notifier heads require
497 : * dynamic initialization. Be sure to call this routine before
498 : * calling any of the other SRCU notifier routines for this head.
499 : *
500 : * If an SRCU notifier head is deallocated, it must first be cleaned
501 : * up by calling srcu_cleanup_notifier_head(). Otherwise the head's
502 : * per-cpu data (used by the SRCU mechanism) will leak.
503 : */
504 1 : void srcu_init_notifier_head(struct srcu_notifier_head *nh)
505 : {
506 1 : mutex_init(&nh->mutex);
507 1 : if (init_srcu_struct(&nh->srcu) < 0)
508 0 : BUG();
509 1 : nh->head = NULL;
510 1 : }
511 : EXPORT_SYMBOL_GPL(srcu_init_notifier_head);
512 :
513 : #endif /* CONFIG_SRCU */
514 :
515 : static ATOMIC_NOTIFIER_HEAD(die_chain);
516 :
517 0 : int notrace notify_die(enum die_val val, const char *str,
518 : struct pt_regs *regs, long err, int trap, int sig)
519 : {
520 0 : struct die_args args = {
521 : .regs = regs,
522 : .str = str,
523 : .err = err,
524 : .trapnr = trap,
525 : .signr = sig,
526 :
527 : };
528 : RCU_LOCKDEP_WARN(!rcu_is_watching(),
529 : "notify_die called but RCU thinks we're quiescent");
530 0 : return atomic_notifier_call_chain(&die_chain, val, &args);
531 : }
532 : NOKPROBE_SYMBOL(notify_die);
533 :
534 0 : int register_die_notifier(struct notifier_block *nb)
535 : {
536 0 : return atomic_notifier_chain_register(&die_chain, nb);
537 : }
538 : EXPORT_SYMBOL_GPL(register_die_notifier);
539 :
540 0 : int unregister_die_notifier(struct notifier_block *nb)
541 : {
542 0 : return atomic_notifier_chain_unregister(&die_chain, nb);
543 : }
544 : EXPORT_SYMBOL_GPL(unregister_die_notifier);
|
