Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
4 : * Copyright (C) 2005-2006, Thomas Gleixner, Russell King
5 : *
6 : * This file contains the core interrupt handling code, for irq-chip based
7 : * architectures. Detailed information is available in
8 : * Documentation/core-api/genericirq.rst
9 : */
10 :
11 : #include <linux/irq.h>
12 : #include <linux/msi.h>
13 : #include <linux/module.h>
14 : #include <linux/interrupt.h>
15 : #include <linux/kernel_stat.h>
16 : #include <linux/irqdomain.h>
17 :
18 : #include <trace/events/irq.h>
19 :
20 : #include "internals.h"
21 :
22 0 : static irqreturn_t bad_chained_irq(int irq, void *dev_id)
23 : {
24 0 : WARN_ONCE(1, "Chained irq %d should not call an action\n", irq);
25 0 : return IRQ_NONE;
26 : }
27 :
28 : /*
29 : * Chained handlers should never call action on their IRQ. This default
30 : * action will emit warning if such thing happens.
31 : */
32 : struct irqaction chained_action = {
33 : .handler = bad_chained_irq,
34 : };
35 :
36 : /**
37 : * irq_set_chip - set the irq chip for an irq
38 : * @irq: irq number
39 : * @chip: pointer to irq chip description structure
40 : */
41 64 : int irq_set_chip(unsigned int irq, const struct irq_chip *chip)
42 : {
43 : unsigned long flags;
44 64 : struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
45 :
46 64 : if (!desc)
47 : return -EINVAL;
48 :
49 64 : desc->irq_data.chip = (struct irq_chip *)(chip ?: &no_irq_chip);
50 128 : irq_put_desc_unlock(desc, flags);
51 : /*
52 : * For !CONFIG_SPARSE_IRQ make the irq show up in
53 : * allocated_irqs.
54 : */
55 64 : irq_mark_irq(irq);
56 64 : return 0;
57 : }
58 : EXPORT_SYMBOL(irq_set_chip);
59 :
60 : /**
61 : * irq_set_irq_type - set the irq trigger type for an irq
62 : * @irq: irq number
63 : * @type: IRQ_TYPE_{LEVEL,EDGE}_* value - see include/linux/irq.h
64 : */
65 0 : int irq_set_irq_type(unsigned int irq, unsigned int type)
66 : {
67 : unsigned long flags;
68 0 : struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
69 0 : int ret = 0;
70 :
71 0 : if (!desc)
72 : return -EINVAL;
73 :
74 0 : ret = __irq_set_trigger(desc, type);
75 0 : irq_put_desc_busunlock(desc, flags);
76 0 : return ret;
77 : }
78 : EXPORT_SYMBOL(irq_set_irq_type);
79 :
80 : /**
81 : * irq_set_handler_data - set irq handler data for an irq
82 : * @irq: Interrupt number
83 : * @data: Pointer to interrupt specific data
84 : *
85 : * Set the hardware irq controller data for an irq
86 : */
87 0 : int irq_set_handler_data(unsigned int irq, void *data)
88 : {
89 : unsigned long flags;
90 0 : struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
91 :
92 0 : if (!desc)
93 : return -EINVAL;
94 0 : desc->irq_common_data.handler_data = data;
95 0 : irq_put_desc_unlock(desc, flags);
96 0 : return 0;
97 : }
98 : EXPORT_SYMBOL(irq_set_handler_data);
99 :
100 : /**
101 : * irq_set_msi_desc_off - set MSI descriptor data for an irq at offset
102 : * @irq_base: Interrupt number base
103 : * @irq_offset: Interrupt number offset
104 : * @entry: Pointer to MSI descriptor data
105 : *
106 : * Set the MSI descriptor entry for an irq at offset
107 : */
108 0 : int irq_set_msi_desc_off(unsigned int irq_base, unsigned int irq_offset,
109 : struct msi_desc *entry)
110 : {
111 : unsigned long flags;
112 0 : struct irq_desc *desc = irq_get_desc_lock(irq_base + irq_offset, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
113 :
114 0 : if (!desc)
115 : return -EINVAL;
116 0 : desc->irq_common_data.msi_desc = entry;
117 0 : if (entry && !irq_offset)
118 0 : entry->irq = irq_base;
119 0 : irq_put_desc_unlock(desc, flags);
120 0 : return 0;
121 : }
122 :
123 : /**
124 : * irq_set_msi_desc - set MSI descriptor data for an irq
125 : * @irq: Interrupt number
126 : * @entry: Pointer to MSI descriptor data
127 : *
128 : * Set the MSI descriptor entry for an irq
129 : */
130 0 : int irq_set_msi_desc(unsigned int irq, struct msi_desc *entry)
131 : {
132 0 : return irq_set_msi_desc_off(irq, 0, entry);
133 : }
134 :
135 : /**
136 : * irq_set_chip_data - set irq chip data for an irq
137 : * @irq: Interrupt number
138 : * @data: Pointer to chip specific data
139 : *
140 : * Set the hardware irq chip data for an irq
141 : */
142 0 : int irq_set_chip_data(unsigned int irq, void *data)
143 : {
144 : unsigned long flags;
145 0 : struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
146 :
147 0 : if (!desc)
148 : return -EINVAL;
149 0 : desc->irq_data.chip_data = data;
150 0 : irq_put_desc_unlock(desc, flags);
151 0 : return 0;
152 : }
153 : EXPORT_SYMBOL(irq_set_chip_data);
154 :
155 0 : struct irq_data *irq_get_irq_data(unsigned int irq)
156 : {
157 0 : struct irq_desc *desc = irq_to_desc(irq);
158 :
159 0 : return desc ? &desc->irq_data : NULL;
160 : }
161 : EXPORT_SYMBOL_GPL(irq_get_irq_data);
162 :
163 : static void irq_state_clr_disabled(struct irq_desc *desc)
164 : {
165 4 : irqd_clear(&desc->irq_data, IRQD_IRQ_DISABLED);
166 : }
167 :
168 : static void irq_state_clr_masked(struct irq_desc *desc)
169 : {
170 4 : irqd_clear(&desc->irq_data, IRQD_IRQ_MASKED);
171 : }
172 :
173 : static void irq_state_clr_started(struct irq_desc *desc)
174 : {
175 0 : irqd_clear(&desc->irq_data, IRQD_IRQ_STARTED);
176 : }
177 :
178 : static void irq_state_set_started(struct irq_desc *desc)
179 : {
180 4 : irqd_set(&desc->irq_data, IRQD_IRQ_STARTED);
181 : }
182 :
183 : enum {
184 : IRQ_STARTUP_NORMAL,
185 : IRQ_STARTUP_MANAGED,
186 : IRQ_STARTUP_ABORT,
187 : };
188 :
189 : #ifdef CONFIG_SMP
190 : static int
191 : __irq_startup_managed(struct irq_desc *desc, struct cpumask *aff, bool force)
192 : {
193 : struct irq_data *d = irq_desc_get_irq_data(desc);
194 :
195 : if (!irqd_affinity_is_managed(d))
196 : return IRQ_STARTUP_NORMAL;
197 :
198 : irqd_clr_managed_shutdown(d);
199 :
200 : if (cpumask_any_and(aff, cpu_online_mask) >= nr_cpu_ids) {
201 : /*
202 : * Catch code which fiddles with enable_irq() on a managed
203 : * and potentially shutdown IRQ. Chained interrupt
204 : * installment or irq auto probing should not happen on
205 : * managed irqs either.
206 : */
207 : if (WARN_ON_ONCE(force))
208 : return IRQ_STARTUP_ABORT;
209 : /*
210 : * The interrupt was requested, but there is no online CPU
211 : * in it's affinity mask. Put it into managed shutdown
212 : * state and let the cpu hotplug mechanism start it up once
213 : * a CPU in the mask becomes available.
214 : */
215 : return IRQ_STARTUP_ABORT;
216 : }
217 : /*
218 : * Managed interrupts have reserved resources, so this should not
219 : * happen.
220 : */
221 : if (WARN_ON(irq_domain_activate_irq(d, false)))
222 : return IRQ_STARTUP_ABORT;
223 : return IRQ_STARTUP_MANAGED;
224 : }
225 : #else
226 : static __always_inline int
227 : __irq_startup_managed(struct irq_desc *desc, struct cpumask *aff, bool force)
228 : {
229 : return IRQ_STARTUP_NORMAL;
230 : }
231 : #endif
232 :
233 2 : static int __irq_startup(struct irq_desc *desc)
234 : {
235 2 : struct irq_data *d = irq_desc_get_irq_data(desc);
236 2 : int ret = 0;
237 :
238 : /* Warn if this interrupt is not activated but try nevertheless */
239 2 : WARN_ON_ONCE(!irqd_is_activated(d));
240 :
241 2 : if (d->chip->irq_startup) {
242 0 : ret = d->chip->irq_startup(d);
243 0 : irq_state_clr_disabled(desc);
244 : irq_state_clr_masked(desc);
245 : } else {
246 2 : irq_enable(desc);
247 : }
248 2 : irq_state_set_started(desc);
249 2 : return ret;
250 : }
251 :
252 2 : int irq_startup(struct irq_desc *desc, bool resend, bool force)
253 : {
254 2 : struct irq_data *d = irq_desc_get_irq_data(desc);
255 2 : struct cpumask *aff = irq_data_get_affinity_mask(d);
256 2 : int ret = 0;
257 :
258 2 : desc->depth = 0;
259 :
260 2 : if (irqd_is_started(d)) {
261 0 : irq_enable(desc);
262 : } else {
263 2 : switch (__irq_startup_managed(desc, aff, force)) {
264 : case IRQ_STARTUP_NORMAL:
265 : if (d->chip->flags & IRQCHIP_AFFINITY_PRE_STARTUP)
266 : irq_setup_affinity(desc);
267 2 : ret = __irq_startup(desc);
268 : if (!(d->chip->flags & IRQCHIP_AFFINITY_PRE_STARTUP))
269 : irq_setup_affinity(desc);
270 : break;
271 : case IRQ_STARTUP_MANAGED:
272 : irq_do_set_affinity(d, aff, false);
273 : ret = __irq_startup(desc);
274 : break;
275 : case IRQ_STARTUP_ABORT:
276 : irqd_set_managed_shutdown(d);
277 : return 0;
278 : }
279 : }
280 2 : if (resend)
281 2 : check_irq_resend(desc, false);
282 :
283 : return ret;
284 : }
285 :
286 2 : int irq_activate(struct irq_desc *desc)
287 : {
288 2 : struct irq_data *d = irq_desc_get_irq_data(desc);
289 :
290 2 : if (!irqd_affinity_is_managed(d))
291 2 : return irq_domain_activate_irq(d, false);
292 : return 0;
293 : }
294 :
295 0 : int irq_activate_and_startup(struct irq_desc *desc, bool resend)
296 : {
297 0 : if (WARN_ON(irq_activate(desc)))
298 : return 0;
299 0 : return irq_startup(desc, resend, IRQ_START_FORCE);
300 : }
301 :
302 : static void __irq_disable(struct irq_desc *desc, bool mask);
303 :
304 0 : void irq_shutdown(struct irq_desc *desc)
305 : {
306 0 : if (irqd_is_started(&desc->irq_data)) {
307 0 : desc->depth = 1;
308 0 : if (desc->irq_data.chip->irq_shutdown) {
309 0 : desc->irq_data.chip->irq_shutdown(&desc->irq_data);
310 0 : irq_state_set_disabled(desc);
311 : irq_state_set_masked(desc);
312 : } else {
313 0 : __irq_disable(desc, true);
314 : }
315 : irq_state_clr_started(desc);
316 : }
317 0 : }
318 :
319 :
320 0 : void irq_shutdown_and_deactivate(struct irq_desc *desc)
321 : {
322 0 : irq_shutdown(desc);
323 : /*
324 : * This must be called even if the interrupt was never started up,
325 : * because the activation can happen before the interrupt is
326 : * available for request/startup. It has it's own state tracking so
327 : * it's safe to call it unconditionally.
328 : */
329 0 : irq_domain_deactivate_irq(&desc->irq_data);
330 0 : }
331 :
332 2 : void irq_enable(struct irq_desc *desc)
333 : {
334 4 : if (!irqd_irq_disabled(&desc->irq_data)) {
335 : unmask_irq(desc);
336 : } else {
337 2 : irq_state_clr_disabled(desc);
338 2 : if (desc->irq_data.chip->irq_enable) {
339 2 : desc->irq_data.chip->irq_enable(&desc->irq_data);
340 : irq_state_clr_masked(desc);
341 : } else {
342 : unmask_irq(desc);
343 : }
344 : }
345 2 : }
346 :
347 0 : static void __irq_disable(struct irq_desc *desc, bool mask)
348 : {
349 0 : if (irqd_irq_disabled(&desc->irq_data)) {
350 0 : if (mask)
351 : mask_irq(desc);
352 : } else {
353 0 : irq_state_set_disabled(desc);
354 0 : if (desc->irq_data.chip->irq_disable) {
355 0 : desc->irq_data.chip->irq_disable(&desc->irq_data);
356 : irq_state_set_masked(desc);
357 0 : } else if (mask) {
358 : mask_irq(desc);
359 : }
360 : }
361 0 : }
362 :
363 : /**
364 : * irq_disable - Mark interrupt disabled
365 : * @desc: irq descriptor which should be disabled
366 : *
367 : * If the chip does not implement the irq_disable callback, we
368 : * use a lazy disable approach. That means we mark the interrupt
369 : * disabled, but leave the hardware unmasked. That's an
370 : * optimization because we avoid the hardware access for the
371 : * common case where no interrupt happens after we marked it
372 : * disabled. If an interrupt happens, then the interrupt flow
373 : * handler masks the line at the hardware level and marks it
374 : * pending.
375 : *
376 : * If the interrupt chip does not implement the irq_disable callback,
377 : * a driver can disable the lazy approach for a particular irq line by
378 : * calling 'irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY)'. This can
379 : * be used for devices which cannot disable the interrupt at the
380 : * device level under certain circumstances and have to use
381 : * disable_irq[_nosync] instead.
382 : */
383 0 : void irq_disable(struct irq_desc *desc)
384 : {
385 0 : __irq_disable(desc, irq_settings_disable_unlazy(desc));
386 0 : }
387 :
388 0 : void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu)
389 : {
390 0 : if (desc->irq_data.chip->irq_enable)
391 0 : desc->irq_data.chip->irq_enable(&desc->irq_data);
392 : else
393 0 : desc->irq_data.chip->irq_unmask(&desc->irq_data);
394 0 : cpumask_set_cpu(cpu, desc->percpu_enabled);
395 0 : }
396 :
397 0 : void irq_percpu_disable(struct irq_desc *desc, unsigned int cpu)
398 : {
399 0 : if (desc->irq_data.chip->irq_disable)
400 0 : desc->irq_data.chip->irq_disable(&desc->irq_data);
401 : else
402 0 : desc->irq_data.chip->irq_mask(&desc->irq_data);
403 0 : cpumask_clear_cpu(cpu, desc->percpu_enabled);
404 0 : }
405 :
406 0 : static inline void mask_ack_irq(struct irq_desc *desc)
407 : {
408 0 : if (desc->irq_data.chip->irq_mask_ack) {
409 0 : desc->irq_data.chip->irq_mask_ack(&desc->irq_data);
410 : irq_state_set_masked(desc);
411 : } else {
412 0 : mask_irq(desc);
413 0 : if (desc->irq_data.chip->irq_ack)
414 0 : desc->irq_data.chip->irq_ack(&desc->irq_data);
415 : }
416 0 : }
417 :
418 0 : void mask_irq(struct irq_desc *desc)
419 : {
420 0 : if (irqd_irq_masked(&desc->irq_data))
421 : return;
422 :
423 0 : if (desc->irq_data.chip->irq_mask) {
424 0 : desc->irq_data.chip->irq_mask(&desc->irq_data);
425 : irq_state_set_masked(desc);
426 : }
427 : }
428 :
429 0 : void unmask_irq(struct irq_desc *desc)
430 : {
431 0 : if (!irqd_irq_masked(&desc->irq_data))
432 : return;
433 :
434 0 : if (desc->irq_data.chip->irq_unmask) {
435 0 : desc->irq_data.chip->irq_unmask(&desc->irq_data);
436 : irq_state_clr_masked(desc);
437 : }
438 : }
439 :
440 0 : void unmask_threaded_irq(struct irq_desc *desc)
441 : {
442 0 : struct irq_chip *chip = desc->irq_data.chip;
443 :
444 0 : if (chip->flags & IRQCHIP_EOI_THREADED)
445 0 : chip->irq_eoi(&desc->irq_data);
446 :
447 0 : unmask_irq(desc);
448 0 : }
449 :
450 : /*
451 : * handle_nested_irq - Handle a nested irq from a irq thread
452 : * @irq: the interrupt number
453 : *
454 : * Handle interrupts which are nested into a threaded interrupt
455 : * handler. The handler function is called inside the calling
456 : * threads context.
457 : */
458 0 : void handle_nested_irq(unsigned int irq)
459 : {
460 0 : struct irq_desc *desc = irq_to_desc(irq);
461 : struct irqaction *action;
462 : irqreturn_t action_ret;
463 :
464 : might_sleep();
465 :
466 0 : raw_spin_lock_irq(&desc->lock);
467 :
468 0 : desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
469 :
470 0 : action = desc->action;
471 0 : if (unlikely(!action || irqd_irq_disabled(&desc->irq_data))) {
472 0 : desc->istate |= IRQS_PENDING;
473 0 : goto out_unlock;
474 : }
475 :
476 0 : kstat_incr_irqs_this_cpu(desc);
477 0 : irqd_set(&desc->irq_data, IRQD_IRQ_INPROGRESS);
478 0 : raw_spin_unlock_irq(&desc->lock);
479 :
480 0 : action_ret = IRQ_NONE;
481 0 : for_each_action_of_desc(desc, action)
482 0 : action_ret |= action->thread_fn(action->irq, action->dev_id);
483 :
484 0 : if (!irq_settings_no_debug(desc))
485 0 : note_interrupt(desc, action_ret);
486 :
487 0 : raw_spin_lock_irq(&desc->lock);
488 0 : irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
489 :
490 : out_unlock:
491 0 : raw_spin_unlock_irq(&desc->lock);
492 0 : }
493 : EXPORT_SYMBOL_GPL(handle_nested_irq);
494 :
495 : static bool irq_check_poll(struct irq_desc *desc)
496 : {
497 0 : if (!(desc->istate & IRQS_POLL_INPROGRESS))
498 : return false;
499 0 : return irq_wait_for_poll(desc);
500 : }
501 :
502 13 : static bool irq_may_run(struct irq_desc *desc)
503 : {
504 13 : unsigned int mask = IRQD_IRQ_INPROGRESS | IRQD_WAKEUP_ARMED;
505 :
506 : /*
507 : * If the interrupt is not in progress and is not an armed
508 : * wakeup interrupt, proceed.
509 : */
510 26 : if (!irqd_has_set(&desc->irq_data, mask))
511 : return true;
512 :
513 : /*
514 : * If the interrupt is an armed wakeup source, mark it pending
515 : * and suspended, disable it and notify the pm core about the
516 : * event.
517 : */
518 0 : if (irq_pm_check_wakeup(desc))
519 : return false;
520 :
521 : /*
522 : * Handle a potential concurrent poll on a different core.
523 : */
524 : return irq_check_poll(desc);
525 : }
526 :
527 : /**
528 : * handle_simple_irq - Simple and software-decoded IRQs.
529 : * @desc: the interrupt description structure for this irq
530 : *
531 : * Simple interrupts are either sent from a demultiplexing interrupt
532 : * handler or come from hardware, where no interrupt hardware control
533 : * is necessary.
534 : *
535 : * Note: The caller is expected to handle the ack, clear, mask and
536 : * unmask issues if necessary.
537 : */
538 0 : void handle_simple_irq(struct irq_desc *desc)
539 : {
540 0 : raw_spin_lock(&desc->lock);
541 :
542 0 : if (!irq_may_run(desc))
543 : goto out_unlock;
544 :
545 0 : desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
546 :
547 0 : if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
548 0 : desc->istate |= IRQS_PENDING;
549 0 : goto out_unlock;
550 : }
551 :
552 0 : kstat_incr_irqs_this_cpu(desc);
553 0 : handle_irq_event(desc);
554 :
555 : out_unlock:
556 0 : raw_spin_unlock(&desc->lock);
557 0 : }
558 : EXPORT_SYMBOL_GPL(handle_simple_irq);
559 :
560 : /**
561 : * handle_untracked_irq - Simple and software-decoded IRQs.
562 : * @desc: the interrupt description structure for this irq
563 : *
564 : * Untracked interrupts are sent from a demultiplexing interrupt
565 : * handler when the demultiplexer does not know which device it its
566 : * multiplexed irq domain generated the interrupt. IRQ's handled
567 : * through here are not subjected to stats tracking, randomness, or
568 : * spurious interrupt detection.
569 : *
570 : * Note: Like handle_simple_irq, the caller is expected to handle
571 : * the ack, clear, mask and unmask issues if necessary.
572 : */
573 0 : void handle_untracked_irq(struct irq_desc *desc)
574 : {
575 0 : raw_spin_lock(&desc->lock);
576 :
577 0 : if (!irq_may_run(desc))
578 : goto out_unlock;
579 :
580 0 : desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
581 :
582 0 : if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
583 0 : desc->istate |= IRQS_PENDING;
584 0 : goto out_unlock;
585 : }
586 :
587 0 : desc->istate &= ~IRQS_PENDING;
588 0 : irqd_set(&desc->irq_data, IRQD_IRQ_INPROGRESS);
589 0 : raw_spin_unlock(&desc->lock);
590 :
591 0 : __handle_irq_event_percpu(desc);
592 :
593 0 : raw_spin_lock(&desc->lock);
594 0 : irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
595 :
596 : out_unlock:
597 0 : raw_spin_unlock(&desc->lock);
598 0 : }
599 : EXPORT_SYMBOL_GPL(handle_untracked_irq);
600 :
601 : /*
602 : * Called unconditionally from handle_level_irq() and only for oneshot
603 : * interrupts from handle_fasteoi_irq()
604 : */
605 0 : static void cond_unmask_irq(struct irq_desc *desc)
606 : {
607 : /*
608 : * We need to unmask in the following cases:
609 : * - Standard level irq (IRQF_ONESHOT is not set)
610 : * - Oneshot irq which did not wake the thread (caused by a
611 : * spurious interrupt or a primary handler handling it
612 : * completely).
613 : */
614 0 : if (!irqd_irq_disabled(&desc->irq_data) &&
615 0 : irqd_irq_masked(&desc->irq_data) && !desc->threads_oneshot)
616 : unmask_irq(desc);
617 0 : }
618 :
619 : /**
620 : * handle_level_irq - Level type irq handler
621 : * @desc: the interrupt description structure for this irq
622 : *
623 : * Level type interrupts are active as long as the hardware line has
624 : * the active level. This may require to mask the interrupt and unmask
625 : * it after the associated handler has acknowledged the device, so the
626 : * interrupt line is back to inactive.
627 : */
628 0 : void handle_level_irq(struct irq_desc *desc)
629 : {
630 0 : raw_spin_lock(&desc->lock);
631 0 : mask_ack_irq(desc);
632 :
633 0 : if (!irq_may_run(desc))
634 : goto out_unlock;
635 :
636 0 : desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
637 :
638 : /*
639 : * If its disabled or no action available
640 : * keep it masked and get out of here
641 : */
642 0 : if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
643 0 : desc->istate |= IRQS_PENDING;
644 0 : goto out_unlock;
645 : }
646 :
647 0 : kstat_incr_irqs_this_cpu(desc);
648 0 : handle_irq_event(desc);
649 :
650 0 : cond_unmask_irq(desc);
651 :
652 : out_unlock:
653 0 : raw_spin_unlock(&desc->lock);
654 0 : }
655 : EXPORT_SYMBOL_GPL(handle_level_irq);
656 :
657 0 : static void cond_unmask_eoi_irq(struct irq_desc *desc, struct irq_chip *chip)
658 : {
659 0 : if (!(desc->istate & IRQS_ONESHOT)) {
660 0 : chip->irq_eoi(&desc->irq_data);
661 : return;
662 : }
663 : /*
664 : * We need to unmask in the following cases:
665 : * - Oneshot irq which did not wake the thread (caused by a
666 : * spurious interrupt or a primary handler handling it
667 : * completely).
668 : */
669 0 : if (!irqd_irq_disabled(&desc->irq_data) &&
670 0 : irqd_irq_masked(&desc->irq_data) && !desc->threads_oneshot) {
671 0 : chip->irq_eoi(&desc->irq_data);
672 : unmask_irq(desc);
673 0 : } else if (!(chip->flags & IRQCHIP_EOI_THREADED)) {
674 0 : chip->irq_eoi(&desc->irq_data);
675 : }
676 : }
677 :
678 : /**
679 : * handle_fasteoi_irq - irq handler for transparent controllers
680 : * @desc: the interrupt description structure for this irq
681 : *
682 : * Only a single callback will be issued to the chip: an ->eoi()
683 : * call when the interrupt has been serviced. This enables support
684 : * for modern forms of interrupt handlers, which handle the flow
685 : * details in hardware, transparently.
686 : */
687 0 : void handle_fasteoi_irq(struct irq_desc *desc)
688 : {
689 0 : struct irq_chip *chip = desc->irq_data.chip;
690 :
691 0 : raw_spin_lock(&desc->lock);
692 :
693 0 : if (!irq_may_run(desc))
694 : goto out;
695 :
696 0 : desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
697 :
698 : /*
699 : * If its disabled or no action available
700 : * then mask it and get out of here:
701 : */
702 0 : if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
703 0 : desc->istate |= IRQS_PENDING;
704 : mask_irq(desc);
705 : goto out;
706 : }
707 :
708 0 : kstat_incr_irqs_this_cpu(desc);
709 0 : if (desc->istate & IRQS_ONESHOT)
710 : mask_irq(desc);
711 :
712 0 : handle_irq_event(desc);
713 :
714 0 : cond_unmask_eoi_irq(desc, chip);
715 :
716 0 : raw_spin_unlock(&desc->lock);
717 0 : return;
718 : out:
719 0 : if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
720 0 : chip->irq_eoi(&desc->irq_data);
721 0 : raw_spin_unlock(&desc->lock);
722 : }
723 : EXPORT_SYMBOL_GPL(handle_fasteoi_irq);
724 :
725 : /**
726 : * handle_fasteoi_nmi - irq handler for NMI interrupt lines
727 : * @desc: the interrupt description structure for this irq
728 : *
729 : * A simple NMI-safe handler, considering the restrictions
730 : * from request_nmi.
731 : *
732 : * Only a single callback will be issued to the chip: an ->eoi()
733 : * call when the interrupt has been serviced. This enables support
734 : * for modern forms of interrupt handlers, which handle the flow
735 : * details in hardware, transparently.
736 : */
737 0 : void handle_fasteoi_nmi(struct irq_desc *desc)
738 : {
739 0 : struct irq_chip *chip = irq_desc_get_chip(desc);
740 0 : struct irqaction *action = desc->action;
741 0 : unsigned int irq = irq_desc_get_irq(desc);
742 : irqreturn_t res;
743 :
744 0 : __kstat_incr_irqs_this_cpu(desc);
745 :
746 0 : trace_irq_handler_entry(irq, action);
747 : /*
748 : * NMIs cannot be shared, there is only one action.
749 : */
750 0 : res = action->handler(irq, action->dev_id);
751 0 : trace_irq_handler_exit(irq, action, res);
752 :
753 0 : if (chip->irq_eoi)
754 0 : chip->irq_eoi(&desc->irq_data);
755 0 : }
756 : EXPORT_SYMBOL_GPL(handle_fasteoi_nmi);
757 :
758 : /**
759 : * handle_edge_irq - edge type IRQ handler
760 : * @desc: the interrupt description structure for this irq
761 : *
762 : * Interrupt occurs on the falling and/or rising edge of a hardware
763 : * signal. The occurrence is latched into the irq controller hardware
764 : * and must be acked in order to be reenabled. After the ack another
765 : * interrupt can happen on the same source even before the first one
766 : * is handled by the associated event handler. If this happens it
767 : * might be necessary to disable (mask) the interrupt depending on the
768 : * controller hardware. This requires to reenable the interrupt inside
769 : * of the loop which handles the interrupts which have arrived while
770 : * the handler was running. If all pending interrupts are handled, the
771 : * loop is left.
772 : */
773 13 : void handle_edge_irq(struct irq_desc *desc)
774 : {
775 13 : raw_spin_lock(&desc->lock);
776 :
777 13 : desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
778 :
779 13 : if (!irq_may_run(desc)) {
780 0 : desc->istate |= IRQS_PENDING;
781 0 : mask_ack_irq(desc);
782 0 : goto out_unlock;
783 : }
784 :
785 : /*
786 : * If its disabled or no action available then mask it and get
787 : * out of here.
788 : */
789 26 : if (irqd_irq_disabled(&desc->irq_data) || !desc->action) {
790 0 : desc->istate |= IRQS_PENDING;
791 0 : mask_ack_irq(desc);
792 0 : goto out_unlock;
793 : }
794 :
795 13 : kstat_incr_irqs_this_cpu(desc);
796 :
797 : /* Start handling the irq */
798 13 : desc->irq_data.chip->irq_ack(&desc->irq_data);
799 :
800 : do {
801 13 : if (unlikely(!desc->action)) {
802 : mask_irq(desc);
803 : goto out_unlock;
804 : }
805 :
806 : /*
807 : * When another irq arrived while we were handling
808 : * one, we could have masked the irq.
809 : * Reenable it, if it was not disabled in meantime.
810 : */
811 13 : if (unlikely(desc->istate & IRQS_PENDING)) {
812 0 : if (!irqd_irq_disabled(&desc->irq_data) &&
813 0 : irqd_irq_masked(&desc->irq_data))
814 : unmask_irq(desc);
815 : }
816 :
817 13 : handle_irq_event(desc);
818 :
819 13 : } while ((desc->istate & IRQS_PENDING) &&
820 13 : !irqd_irq_disabled(&desc->irq_data));
821 :
822 : out_unlock:
823 13 : raw_spin_unlock(&desc->lock);
824 13 : }
825 : EXPORT_SYMBOL(handle_edge_irq);
826 :
827 : #ifdef CONFIG_IRQ_EDGE_EOI_HANDLER
828 : /**
829 : * handle_edge_eoi_irq - edge eoi type IRQ handler
830 : * @desc: the interrupt description structure for this irq
831 : *
832 : * Similar as the above handle_edge_irq, but using eoi and w/o the
833 : * mask/unmask logic.
834 : */
835 : void handle_edge_eoi_irq(struct irq_desc *desc)
836 : {
837 : struct irq_chip *chip = irq_desc_get_chip(desc);
838 :
839 : raw_spin_lock(&desc->lock);
840 :
841 : desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
842 :
843 : if (!irq_may_run(desc)) {
844 : desc->istate |= IRQS_PENDING;
845 : goto out_eoi;
846 : }
847 :
848 : /*
849 : * If its disabled or no action available then mask it and get
850 : * out of here.
851 : */
852 : if (irqd_irq_disabled(&desc->irq_data) || !desc->action) {
853 : desc->istate |= IRQS_PENDING;
854 : goto out_eoi;
855 : }
856 :
857 : kstat_incr_irqs_this_cpu(desc);
858 :
859 : do {
860 : if (unlikely(!desc->action))
861 : goto out_eoi;
862 :
863 : handle_irq_event(desc);
864 :
865 : } while ((desc->istate & IRQS_PENDING) &&
866 : !irqd_irq_disabled(&desc->irq_data));
867 :
868 : out_eoi:
869 : chip->irq_eoi(&desc->irq_data);
870 : raw_spin_unlock(&desc->lock);
871 : }
872 : #endif
873 :
874 : /**
875 : * handle_percpu_irq - Per CPU local irq handler
876 : * @desc: the interrupt description structure for this irq
877 : *
878 : * Per CPU interrupts on SMP machines without locking requirements
879 : */
880 0 : void handle_percpu_irq(struct irq_desc *desc)
881 : {
882 0 : struct irq_chip *chip = irq_desc_get_chip(desc);
883 :
884 : /*
885 : * PER CPU interrupts are not serialized. Do not touch
886 : * desc->tot_count.
887 : */
888 0 : __kstat_incr_irqs_this_cpu(desc);
889 :
890 0 : if (chip->irq_ack)
891 0 : chip->irq_ack(&desc->irq_data);
892 :
893 0 : handle_irq_event_percpu(desc);
894 :
895 0 : if (chip->irq_eoi)
896 0 : chip->irq_eoi(&desc->irq_data);
897 0 : }
898 :
899 : /**
900 : * handle_percpu_devid_irq - Per CPU local irq handler with per cpu dev ids
901 : * @desc: the interrupt description structure for this irq
902 : *
903 : * Per CPU interrupts on SMP machines without locking requirements. Same as
904 : * handle_percpu_irq() above but with the following extras:
905 : *
906 : * action->percpu_dev_id is a pointer to percpu variables which
907 : * contain the real device id for the cpu on which this handler is
908 : * called
909 : */
910 0 : void handle_percpu_devid_irq(struct irq_desc *desc)
911 : {
912 0 : struct irq_chip *chip = irq_desc_get_chip(desc);
913 0 : struct irqaction *action = desc->action;
914 0 : unsigned int irq = irq_desc_get_irq(desc);
915 : irqreturn_t res;
916 :
917 : /*
918 : * PER CPU interrupts are not serialized. Do not touch
919 : * desc->tot_count.
920 : */
921 0 : __kstat_incr_irqs_this_cpu(desc);
922 :
923 0 : if (chip->irq_ack)
924 0 : chip->irq_ack(&desc->irq_data);
925 :
926 0 : if (likely(action)) {
927 0 : trace_irq_handler_entry(irq, action);
928 0 : res = action->handler(irq, raw_cpu_ptr(action->percpu_dev_id));
929 0 : trace_irq_handler_exit(irq, action, res);
930 : } else {
931 0 : unsigned int cpu = smp_processor_id();
932 0 : bool enabled = cpumask_test_cpu(cpu, desc->percpu_enabled);
933 :
934 0 : if (enabled)
935 0 : irq_percpu_disable(desc, cpu);
936 :
937 0 : pr_err_once("Spurious%s percpu IRQ%u on CPU%u\n",
938 : enabled ? " and unmasked" : "", irq, cpu);
939 : }
940 :
941 0 : if (chip->irq_eoi)
942 0 : chip->irq_eoi(&desc->irq_data);
943 0 : }
944 :
945 : /**
946 : * handle_percpu_devid_fasteoi_nmi - Per CPU local NMI handler with per cpu
947 : * dev ids
948 : * @desc: the interrupt description structure for this irq
949 : *
950 : * Similar to handle_fasteoi_nmi, but handling the dev_id cookie
951 : * as a percpu pointer.
952 : */
953 0 : void handle_percpu_devid_fasteoi_nmi(struct irq_desc *desc)
954 : {
955 0 : struct irq_chip *chip = irq_desc_get_chip(desc);
956 0 : struct irqaction *action = desc->action;
957 0 : unsigned int irq = irq_desc_get_irq(desc);
958 : irqreturn_t res;
959 :
960 0 : __kstat_incr_irqs_this_cpu(desc);
961 :
962 0 : trace_irq_handler_entry(irq, action);
963 0 : res = action->handler(irq, raw_cpu_ptr(action->percpu_dev_id));
964 0 : trace_irq_handler_exit(irq, action, res);
965 :
966 0 : if (chip->irq_eoi)
967 0 : chip->irq_eoi(&desc->irq_data);
968 0 : }
969 :
970 : static void
971 64 : __irq_do_set_handler(struct irq_desc *desc, irq_flow_handler_t handle,
972 : int is_chained, const char *name)
973 : {
974 64 : if (!handle) {
975 : handle = handle_bad_irq;
976 : } else {
977 64 : struct irq_data *irq_data = &desc->irq_data;
978 : #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
979 : /*
980 : * With hierarchical domains we might run into a
981 : * situation where the outermost chip is not yet set
982 : * up, but the inner chips are there. Instead of
983 : * bailing we install the handler, but obviously we
984 : * cannot enable/startup the interrupt at this point.
985 : */
986 128 : while (irq_data) {
987 64 : if (irq_data->chip != &no_irq_chip)
988 : break;
989 : /*
990 : * Bail out if the outer chip is not set up
991 : * and the interrupt supposed to be started
992 : * right away.
993 : */
994 0 : if (WARN_ON(is_chained))
995 : return;
996 : /* Try the parent */
997 0 : irq_data = irq_data->parent_data;
998 : }
999 : #endif
1000 64 : if (WARN_ON(!irq_data || irq_data->chip == &no_irq_chip))
1001 : return;
1002 : }
1003 :
1004 : /* Uninstall? */
1005 64 : if (handle == handle_bad_irq) {
1006 0 : if (desc->irq_data.chip != &no_irq_chip)
1007 0 : mask_ack_irq(desc);
1008 0 : irq_state_set_disabled(desc);
1009 0 : if (is_chained)
1010 0 : desc->action = NULL;
1011 0 : desc->depth = 1;
1012 : }
1013 64 : desc->handle_irq = handle;
1014 64 : desc->name = name;
1015 :
1016 64 : if (handle != handle_bad_irq && is_chained) {
1017 0 : unsigned int type = irqd_get_trigger_type(&desc->irq_data);
1018 :
1019 : /*
1020 : * We're about to start this interrupt immediately,
1021 : * hence the need to set the trigger configuration.
1022 : * But the .set_type callback may have overridden the
1023 : * flow handler, ignoring that we're dealing with a
1024 : * chained interrupt. Reset it immediately because we
1025 : * do know better.
1026 : */
1027 0 : if (type != IRQ_TYPE_NONE) {
1028 0 : __irq_set_trigger(desc, type);
1029 0 : desc->handle_irq = handle;
1030 : }
1031 :
1032 0 : irq_settings_set_noprobe(desc);
1033 0 : irq_settings_set_norequest(desc);
1034 0 : irq_settings_set_nothread(desc);
1035 0 : desc->action = &chained_action;
1036 0 : irq_activate_and_startup(desc, IRQ_RESEND);
1037 : }
1038 : }
1039 :
1040 : void
1041 64 : __irq_set_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
1042 : const char *name)
1043 : {
1044 : unsigned long flags;
1045 64 : struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, 0);
1046 :
1047 64 : if (!desc)
1048 0 : return;
1049 :
1050 64 : __irq_do_set_handler(desc, handle, is_chained, name);
1051 128 : irq_put_desc_busunlock(desc, flags);
1052 : }
1053 : EXPORT_SYMBOL_GPL(__irq_set_handler);
1054 :
1055 : void
1056 0 : irq_set_chained_handler_and_data(unsigned int irq, irq_flow_handler_t handle,
1057 : void *data)
1058 : {
1059 : unsigned long flags;
1060 0 : struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, 0);
1061 :
1062 0 : if (!desc)
1063 0 : return;
1064 :
1065 0 : desc->irq_common_data.handler_data = data;
1066 0 : __irq_do_set_handler(desc, handle, 1, NULL);
1067 :
1068 0 : irq_put_desc_busunlock(desc, flags);
1069 : }
1070 : EXPORT_SYMBOL_GPL(irq_set_chained_handler_and_data);
1071 :
1072 : void
1073 64 : irq_set_chip_and_handler_name(unsigned int irq, const struct irq_chip *chip,
1074 : irq_flow_handler_t handle, const char *name)
1075 : {
1076 64 : irq_set_chip(irq, chip);
1077 64 : __irq_set_handler(irq, handle, 0, name);
1078 64 : }
1079 : EXPORT_SYMBOL_GPL(irq_set_chip_and_handler_name);
1080 :
1081 0 : void irq_modify_status(unsigned int irq, unsigned long clr, unsigned long set)
1082 : {
1083 : unsigned long flags, trigger, tmp;
1084 0 : struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
1085 :
1086 0 : if (!desc)
1087 0 : return;
1088 :
1089 : /*
1090 : * Warn when a driver sets the no autoenable flag on an already
1091 : * active interrupt.
1092 : */
1093 0 : WARN_ON_ONCE(!desc->depth && (set & _IRQ_NOAUTOEN));
1094 :
1095 0 : irq_settings_clr_and_set(desc, clr, set);
1096 :
1097 0 : trigger = irqd_get_trigger_type(&desc->irq_data);
1098 :
1099 0 : irqd_clear(&desc->irq_data, IRQD_NO_BALANCING | IRQD_PER_CPU |
1100 : IRQD_TRIGGER_MASK | IRQD_LEVEL | IRQD_MOVE_PCNTXT);
1101 0 : if (irq_settings_has_no_balance_set(desc))
1102 0 : irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
1103 0 : if (irq_settings_is_per_cpu(desc))
1104 0 : irqd_set(&desc->irq_data, IRQD_PER_CPU);
1105 0 : if (irq_settings_can_move_pcntxt(desc))
1106 0 : irqd_set(&desc->irq_data, IRQD_MOVE_PCNTXT);
1107 0 : if (irq_settings_is_level(desc))
1108 0 : irqd_set(&desc->irq_data, IRQD_LEVEL);
1109 :
1110 0 : tmp = irq_settings_get_trigger_mask(desc);
1111 0 : if (tmp != IRQ_TYPE_NONE)
1112 0 : trigger = tmp;
1113 :
1114 0 : irqd_set(&desc->irq_data, trigger);
1115 :
1116 0 : irq_put_desc_unlock(desc, flags);
1117 : }
1118 : EXPORT_SYMBOL_GPL(irq_modify_status);
1119 :
1120 : #ifdef CONFIG_DEPRECATED_IRQ_CPU_ONOFFLINE
1121 : /**
1122 : * irq_cpu_online - Invoke all irq_cpu_online functions.
1123 : *
1124 : * Iterate through all irqs and invoke the chip.irq_cpu_online()
1125 : * for each.
1126 : */
1127 : void irq_cpu_online(void)
1128 : {
1129 : struct irq_desc *desc;
1130 : struct irq_chip *chip;
1131 : unsigned long flags;
1132 : unsigned int irq;
1133 :
1134 : for_each_active_irq(irq) {
1135 : desc = irq_to_desc(irq);
1136 : if (!desc)
1137 : continue;
1138 :
1139 : raw_spin_lock_irqsave(&desc->lock, flags);
1140 :
1141 : chip = irq_data_get_irq_chip(&desc->irq_data);
1142 : if (chip && chip->irq_cpu_online &&
1143 : (!(chip->flags & IRQCHIP_ONOFFLINE_ENABLED) ||
1144 : !irqd_irq_disabled(&desc->irq_data)))
1145 : chip->irq_cpu_online(&desc->irq_data);
1146 :
1147 : raw_spin_unlock_irqrestore(&desc->lock, flags);
1148 : }
1149 : }
1150 :
1151 : /**
1152 : * irq_cpu_offline - Invoke all irq_cpu_offline functions.
1153 : *
1154 : * Iterate through all irqs and invoke the chip.irq_cpu_offline()
1155 : * for each.
1156 : */
1157 : void irq_cpu_offline(void)
1158 : {
1159 : struct irq_desc *desc;
1160 : struct irq_chip *chip;
1161 : unsigned long flags;
1162 : unsigned int irq;
1163 :
1164 : for_each_active_irq(irq) {
1165 : desc = irq_to_desc(irq);
1166 : if (!desc)
1167 : continue;
1168 :
1169 : raw_spin_lock_irqsave(&desc->lock, flags);
1170 :
1171 : chip = irq_data_get_irq_chip(&desc->irq_data);
1172 : if (chip && chip->irq_cpu_offline &&
1173 : (!(chip->flags & IRQCHIP_ONOFFLINE_ENABLED) ||
1174 : !irqd_irq_disabled(&desc->irq_data)))
1175 : chip->irq_cpu_offline(&desc->irq_data);
1176 :
1177 : raw_spin_unlock_irqrestore(&desc->lock, flags);
1178 : }
1179 : }
1180 : #endif
1181 :
1182 : #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
1183 :
1184 : #ifdef CONFIG_IRQ_FASTEOI_HIERARCHY_HANDLERS
1185 : /**
1186 : * handle_fasteoi_ack_irq - irq handler for edge hierarchy
1187 : * stacked on transparent controllers
1188 : *
1189 : * @desc: the interrupt description structure for this irq
1190 : *
1191 : * Like handle_fasteoi_irq(), but for use with hierarchy where
1192 : * the irq_chip also needs to have its ->irq_ack() function
1193 : * called.
1194 : */
1195 : void handle_fasteoi_ack_irq(struct irq_desc *desc)
1196 : {
1197 : struct irq_chip *chip = desc->irq_data.chip;
1198 :
1199 : raw_spin_lock(&desc->lock);
1200 :
1201 : if (!irq_may_run(desc))
1202 : goto out;
1203 :
1204 : desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
1205 :
1206 : /*
1207 : * If its disabled or no action available
1208 : * then mask it and get out of here:
1209 : */
1210 : if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
1211 : desc->istate |= IRQS_PENDING;
1212 : mask_irq(desc);
1213 : goto out;
1214 : }
1215 :
1216 : kstat_incr_irqs_this_cpu(desc);
1217 : if (desc->istate & IRQS_ONESHOT)
1218 : mask_irq(desc);
1219 :
1220 : /* Start handling the irq */
1221 : desc->irq_data.chip->irq_ack(&desc->irq_data);
1222 :
1223 : handle_irq_event(desc);
1224 :
1225 : cond_unmask_eoi_irq(desc, chip);
1226 :
1227 : raw_spin_unlock(&desc->lock);
1228 : return;
1229 : out:
1230 : if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
1231 : chip->irq_eoi(&desc->irq_data);
1232 : raw_spin_unlock(&desc->lock);
1233 : }
1234 : EXPORT_SYMBOL_GPL(handle_fasteoi_ack_irq);
1235 :
1236 : /**
1237 : * handle_fasteoi_mask_irq - irq handler for level hierarchy
1238 : * stacked on transparent controllers
1239 : *
1240 : * @desc: the interrupt description structure for this irq
1241 : *
1242 : * Like handle_fasteoi_irq(), but for use with hierarchy where
1243 : * the irq_chip also needs to have its ->irq_mask_ack() function
1244 : * called.
1245 : */
1246 : void handle_fasteoi_mask_irq(struct irq_desc *desc)
1247 : {
1248 : struct irq_chip *chip = desc->irq_data.chip;
1249 :
1250 : raw_spin_lock(&desc->lock);
1251 : mask_ack_irq(desc);
1252 :
1253 : if (!irq_may_run(desc))
1254 : goto out;
1255 :
1256 : desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
1257 :
1258 : /*
1259 : * If its disabled or no action available
1260 : * then mask it and get out of here:
1261 : */
1262 : if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
1263 : desc->istate |= IRQS_PENDING;
1264 : mask_irq(desc);
1265 : goto out;
1266 : }
1267 :
1268 : kstat_incr_irqs_this_cpu(desc);
1269 : if (desc->istate & IRQS_ONESHOT)
1270 : mask_irq(desc);
1271 :
1272 : handle_irq_event(desc);
1273 :
1274 : cond_unmask_eoi_irq(desc, chip);
1275 :
1276 : raw_spin_unlock(&desc->lock);
1277 : return;
1278 : out:
1279 : if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
1280 : chip->irq_eoi(&desc->irq_data);
1281 : raw_spin_unlock(&desc->lock);
1282 : }
1283 : EXPORT_SYMBOL_GPL(handle_fasteoi_mask_irq);
1284 :
1285 : #endif /* CONFIG_IRQ_FASTEOI_HIERARCHY_HANDLERS */
1286 :
1287 : /**
1288 : * irq_chip_set_parent_state - set the state of a parent interrupt.
1289 : *
1290 : * @data: Pointer to interrupt specific data
1291 : * @which: State to be restored (one of IRQCHIP_STATE_*)
1292 : * @val: Value corresponding to @which
1293 : *
1294 : * Conditional success, if the underlying irqchip does not implement it.
1295 : */
1296 0 : int irq_chip_set_parent_state(struct irq_data *data,
1297 : enum irqchip_irq_state which,
1298 : bool val)
1299 : {
1300 0 : data = data->parent_data;
1301 :
1302 0 : if (!data || !data->chip->irq_set_irqchip_state)
1303 : return 0;
1304 :
1305 0 : return data->chip->irq_set_irqchip_state(data, which, val);
1306 : }
1307 : EXPORT_SYMBOL_GPL(irq_chip_set_parent_state);
1308 :
1309 : /**
1310 : * irq_chip_get_parent_state - get the state of a parent interrupt.
1311 : *
1312 : * @data: Pointer to interrupt specific data
1313 : * @which: one of IRQCHIP_STATE_* the caller wants to know
1314 : * @state: a pointer to a boolean where the state is to be stored
1315 : *
1316 : * Conditional success, if the underlying irqchip does not implement it.
1317 : */
1318 0 : int irq_chip_get_parent_state(struct irq_data *data,
1319 : enum irqchip_irq_state which,
1320 : bool *state)
1321 : {
1322 0 : data = data->parent_data;
1323 :
1324 0 : if (!data || !data->chip->irq_get_irqchip_state)
1325 : return 0;
1326 :
1327 0 : return data->chip->irq_get_irqchip_state(data, which, state);
1328 : }
1329 : EXPORT_SYMBOL_GPL(irq_chip_get_parent_state);
1330 :
1331 : /**
1332 : * irq_chip_enable_parent - Enable the parent interrupt (defaults to unmask if
1333 : * NULL)
1334 : * @data: Pointer to interrupt specific data
1335 : */
1336 0 : void irq_chip_enable_parent(struct irq_data *data)
1337 : {
1338 0 : data = data->parent_data;
1339 0 : if (data->chip->irq_enable)
1340 0 : data->chip->irq_enable(data);
1341 : else
1342 0 : data->chip->irq_unmask(data);
1343 0 : }
1344 : EXPORT_SYMBOL_GPL(irq_chip_enable_parent);
1345 :
1346 : /**
1347 : * irq_chip_disable_parent - Disable the parent interrupt (defaults to mask if
1348 : * NULL)
1349 : * @data: Pointer to interrupt specific data
1350 : */
1351 0 : void irq_chip_disable_parent(struct irq_data *data)
1352 : {
1353 0 : data = data->parent_data;
1354 0 : if (data->chip->irq_disable)
1355 0 : data->chip->irq_disable(data);
1356 : else
1357 0 : data->chip->irq_mask(data);
1358 0 : }
1359 : EXPORT_SYMBOL_GPL(irq_chip_disable_parent);
1360 :
1361 : /**
1362 : * irq_chip_ack_parent - Acknowledge the parent interrupt
1363 : * @data: Pointer to interrupt specific data
1364 : */
1365 0 : void irq_chip_ack_parent(struct irq_data *data)
1366 : {
1367 0 : data = data->parent_data;
1368 0 : data->chip->irq_ack(data);
1369 0 : }
1370 : EXPORT_SYMBOL_GPL(irq_chip_ack_parent);
1371 :
1372 : /**
1373 : * irq_chip_mask_parent - Mask the parent interrupt
1374 : * @data: Pointer to interrupt specific data
1375 : */
1376 0 : void irq_chip_mask_parent(struct irq_data *data)
1377 : {
1378 0 : data = data->parent_data;
1379 0 : data->chip->irq_mask(data);
1380 0 : }
1381 : EXPORT_SYMBOL_GPL(irq_chip_mask_parent);
1382 :
1383 : /**
1384 : * irq_chip_mask_ack_parent - Mask and acknowledge the parent interrupt
1385 : * @data: Pointer to interrupt specific data
1386 : */
1387 0 : void irq_chip_mask_ack_parent(struct irq_data *data)
1388 : {
1389 0 : data = data->parent_data;
1390 0 : data->chip->irq_mask_ack(data);
1391 0 : }
1392 : EXPORT_SYMBOL_GPL(irq_chip_mask_ack_parent);
1393 :
1394 : /**
1395 : * irq_chip_unmask_parent - Unmask the parent interrupt
1396 : * @data: Pointer to interrupt specific data
1397 : */
1398 0 : void irq_chip_unmask_parent(struct irq_data *data)
1399 : {
1400 0 : data = data->parent_data;
1401 0 : data->chip->irq_unmask(data);
1402 0 : }
1403 : EXPORT_SYMBOL_GPL(irq_chip_unmask_parent);
1404 :
1405 : /**
1406 : * irq_chip_eoi_parent - Invoke EOI on the parent interrupt
1407 : * @data: Pointer to interrupt specific data
1408 : */
1409 0 : void irq_chip_eoi_parent(struct irq_data *data)
1410 : {
1411 0 : data = data->parent_data;
1412 0 : data->chip->irq_eoi(data);
1413 0 : }
1414 : EXPORT_SYMBOL_GPL(irq_chip_eoi_parent);
1415 :
1416 : /**
1417 : * irq_chip_set_affinity_parent - Set affinity on the parent interrupt
1418 : * @data: Pointer to interrupt specific data
1419 : * @dest: The affinity mask to set
1420 : * @force: Flag to enforce setting (disable online checks)
1421 : *
1422 : * Conditional, as the underlying parent chip might not implement it.
1423 : */
1424 0 : int irq_chip_set_affinity_parent(struct irq_data *data,
1425 : const struct cpumask *dest, bool force)
1426 : {
1427 0 : data = data->parent_data;
1428 0 : if (data->chip->irq_set_affinity)
1429 0 : return data->chip->irq_set_affinity(data, dest, force);
1430 :
1431 : return -ENOSYS;
1432 : }
1433 : EXPORT_SYMBOL_GPL(irq_chip_set_affinity_parent);
1434 :
1435 : /**
1436 : * irq_chip_set_type_parent - Set IRQ type on the parent interrupt
1437 : * @data: Pointer to interrupt specific data
1438 : * @type: IRQ_TYPE_{LEVEL,EDGE}_* value - see include/linux/irq.h
1439 : *
1440 : * Conditional, as the underlying parent chip might not implement it.
1441 : */
1442 0 : int irq_chip_set_type_parent(struct irq_data *data, unsigned int type)
1443 : {
1444 0 : data = data->parent_data;
1445 :
1446 0 : if (data->chip->irq_set_type)
1447 0 : return data->chip->irq_set_type(data, type);
1448 :
1449 : return -ENOSYS;
1450 : }
1451 : EXPORT_SYMBOL_GPL(irq_chip_set_type_parent);
1452 :
1453 : /**
1454 : * irq_chip_retrigger_hierarchy - Retrigger an interrupt in hardware
1455 : * @data: Pointer to interrupt specific data
1456 : *
1457 : * Iterate through the domain hierarchy of the interrupt and check
1458 : * whether a hw retrigger function exists. If yes, invoke it.
1459 : */
1460 0 : int irq_chip_retrigger_hierarchy(struct irq_data *data)
1461 : {
1462 0 : for (data = data->parent_data; data; data = data->parent_data)
1463 0 : if (data->chip && data->chip->irq_retrigger)
1464 0 : return data->chip->irq_retrigger(data);
1465 :
1466 : return 0;
1467 : }
1468 : EXPORT_SYMBOL_GPL(irq_chip_retrigger_hierarchy);
1469 :
1470 : /**
1471 : * irq_chip_set_vcpu_affinity_parent - Set vcpu affinity on the parent interrupt
1472 : * @data: Pointer to interrupt specific data
1473 : * @vcpu_info: The vcpu affinity information
1474 : */
1475 0 : int irq_chip_set_vcpu_affinity_parent(struct irq_data *data, void *vcpu_info)
1476 : {
1477 0 : data = data->parent_data;
1478 0 : if (data->chip->irq_set_vcpu_affinity)
1479 0 : return data->chip->irq_set_vcpu_affinity(data, vcpu_info);
1480 :
1481 : return -ENOSYS;
1482 : }
1483 : EXPORT_SYMBOL_GPL(irq_chip_set_vcpu_affinity_parent);
1484 : /**
1485 : * irq_chip_set_wake_parent - Set/reset wake-up on the parent interrupt
1486 : * @data: Pointer to interrupt specific data
1487 : * @on: Whether to set or reset the wake-up capability of this irq
1488 : *
1489 : * Conditional, as the underlying parent chip might not implement it.
1490 : */
1491 0 : int irq_chip_set_wake_parent(struct irq_data *data, unsigned int on)
1492 : {
1493 0 : data = data->parent_data;
1494 :
1495 0 : if (data->chip->flags & IRQCHIP_SKIP_SET_WAKE)
1496 : return 0;
1497 :
1498 0 : if (data->chip->irq_set_wake)
1499 0 : return data->chip->irq_set_wake(data, on);
1500 :
1501 : return -ENOSYS;
1502 : }
1503 : EXPORT_SYMBOL_GPL(irq_chip_set_wake_parent);
1504 :
1505 : /**
1506 : * irq_chip_request_resources_parent - Request resources on the parent interrupt
1507 : * @data: Pointer to interrupt specific data
1508 : */
1509 0 : int irq_chip_request_resources_parent(struct irq_data *data)
1510 : {
1511 0 : data = data->parent_data;
1512 :
1513 0 : if (data->chip->irq_request_resources)
1514 0 : return data->chip->irq_request_resources(data);
1515 :
1516 : return -ENOSYS;
1517 : }
1518 : EXPORT_SYMBOL_GPL(irq_chip_request_resources_parent);
1519 :
1520 : /**
1521 : * irq_chip_release_resources_parent - Release resources on the parent interrupt
1522 : * @data: Pointer to interrupt specific data
1523 : */
1524 0 : void irq_chip_release_resources_parent(struct irq_data *data)
1525 : {
1526 0 : data = data->parent_data;
1527 0 : if (data->chip->irq_release_resources)
1528 0 : data->chip->irq_release_resources(data);
1529 0 : }
1530 : EXPORT_SYMBOL_GPL(irq_chip_release_resources_parent);
1531 : #endif
1532 :
1533 : /**
1534 : * irq_chip_compose_msi_msg - Compose msi message for a irq chip
1535 : * @data: Pointer to interrupt specific data
1536 : * @msg: Pointer to the MSI message
1537 : *
1538 : * For hierarchical domains we find the first chip in the hierarchy
1539 : * which implements the irq_compose_msi_msg callback. For non
1540 : * hierarchical we use the top level chip.
1541 : */
1542 0 : int irq_chip_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
1543 : {
1544 : struct irq_data *pos;
1545 :
1546 0 : for (pos = NULL; !pos && data; data = irqd_get_parent_data(data)) {
1547 0 : if (data->chip && data->chip->irq_compose_msi_msg)
1548 0 : pos = data;
1549 : }
1550 :
1551 0 : if (!pos)
1552 : return -ENOSYS;
1553 :
1554 0 : pos->chip->irq_compose_msi_msg(pos, msg);
1555 0 : return 0;
1556 : }
1557 :
1558 : static struct device *irq_get_parent_device(struct irq_data *data)
1559 : {
1560 2 : if (data->domain)
1561 0 : return data->domain->dev;
1562 :
1563 : return NULL;
1564 : }
1565 :
1566 : /**
1567 : * irq_chip_pm_get - Enable power for an IRQ chip
1568 : * @data: Pointer to interrupt specific data
1569 : *
1570 : * Enable the power to the IRQ chip referenced by the interrupt data
1571 : * structure.
1572 : */
1573 2 : int irq_chip_pm_get(struct irq_data *data)
1574 : {
1575 4 : struct device *dev = irq_get_parent_device(data);
1576 : int retval;
1577 :
1578 2 : if (IS_ENABLED(CONFIG_PM) && dev) {
1579 0 : retval = pm_runtime_get_sync(dev);
1580 0 : if (retval < 0) {
1581 : pm_runtime_put_noidle(dev);
1582 : return retval;
1583 : }
1584 : }
1585 :
1586 : return 0;
1587 : }
1588 :
1589 : /**
1590 : * irq_chip_pm_put - Disable power for an IRQ chip
1591 : * @data: Pointer to interrupt specific data
1592 : *
1593 : * Disable the power to the IRQ chip referenced by the interrupt data
1594 : * structure, belongs. Note that power will only be disabled, once this
1595 : * function has been called for all IRQs that have called irq_chip_pm_get().
1596 : */
1597 0 : int irq_chip_pm_put(struct irq_data *data)
1598 : {
1599 0 : struct device *dev = irq_get_parent_device(data);
1600 0 : int retval = 0;
1601 :
1602 0 : if (IS_ENABLED(CONFIG_PM) && dev)
1603 0 : retval = pm_runtime_put(dev);
1604 :
1605 0 : return (retval < 0) ? retval : 0;
1606 : }
|
