Line data    Source code

       1             : // SPDX-License-Identifier: GPL-2.0
       2             : #include <linux/spinlock.h>
       3             : #include <linux/task_work.h>
       4             : #include <linux/resume_user_mode.h>
       5             : 
       6             : static struct callback_head work_exited; /* all we need is ->next == NULL */
       7             : 
       8             : /**
       9             :  * task_work_add - ask the @task to execute @work->func()
      10             :  * @task: the task which should run the callback
      11             :  * @work: the callback to run
      12             :  * @notify: how to notify the targeted task
      13             :  *
      14             :  * Queue @work for task_work_run() below and notify the @task if @notify
      15             :  * is @TWA_RESUME or @TWA_SIGNAL. @TWA_SIGNAL works like signals, in that the
      16             :  * it will interrupt the targeted task and run the task_work. @TWA_RESUME
      17             :  * work is run only when the task exits the kernel and returns to user mode,
      18             :  * or before entering guest mode. Fails if the @task is exiting/exited and thus
      19             :  * it can't process this @work. Otherwise @work->func() will be called when the
      20             :  * @task goes through one of the aforementioned transitions, or exits.
      21             :  *
      22             :  * If the targeted task is exiting, then an error is returned and the work item
      23             :  * is not queued. It's up to the caller to arrange for an alternative mechanism
      24             :  * in that case.
      25             :  *
      26             :  * Note: there is no ordering guarantee on works queued here. The task_work
      27             :  * list is LIFO.
      28             :  *
      29             :  * RETURNS:
      30             :  * 0 if succeeds or -ESRCH.
      31             :  */
      32           0 : int task_work_add(struct task_struct *task, struct callback_head *work,
      33             :                   enum task_work_notify_mode notify)
      34             : {
      35             :         struct callback_head *head;
      36             : 
      37             :         /* record the work call stack in order to print it in KASAN reports */
      38           0 :         kasan_record_aux_stack(work);
      39             : 
      40             :         do {
      41           0 :                 head = READ_ONCE(task->task_works);
      42           0 :                 if (unlikely(head == &work_exited))
      43             :                         return -ESRCH;
      44           0 :                 work->next = head;
      45           0 :         } while (cmpxchg(&task->task_works, head, work) != head);
      46             : 
      47           0 :         switch (notify) {
      48             :         case TWA_NONE:
      49             :                 break;
      50             :         case TWA_RESUME:
      51             :                 set_notify_resume(task);
      52             :                 break;
      53             :         case TWA_SIGNAL:
      54             :                 set_notify_signal(task);
      55             :                 break;
      56             :         default:
      57           0 :                 WARN_ON_ONCE(1);
      58             :                 break;
      59             :         }
      60             : 
      61             :         return 0;
      62             : }
      63             : 
      64             : /**
      65             :  * task_work_cancel_match - cancel a pending work added by task_work_add()
      66             :  * @task: the task which should execute the work
      67             :  * @match: match function to call
      68             :  *
      69             :  * RETURNS:
      70             :  * The found work or NULL if not found.
      71             :  */
      72             : struct callback_head *
      73           0 : task_work_cancel_match(struct task_struct *task,
      74             :                        bool (*match)(struct callback_head *, void *data),
      75             :                        void *data)
      76             : {
      77           0 :         struct callback_head **pprev = &task->task_works;
      78             :         struct callback_head *work;
      79             :         unsigned long flags;
      80             : 
      81           0 :         if (likely(!task_work_pending(task)))
      82             :                 return NULL;
      83             :         /*
      84             :          * If cmpxchg() fails we continue without updating pprev.
      85             :          * Either we raced with task_work_add() which added the
      86             :          * new entry before this work, we will find it again. Or
      87             :          * we raced with task_work_run(), *pprev == NULL/exited.
      88             :          */
      89           0 :         raw_spin_lock_irqsave(&task->pi_lock, flags);
      90           0 :         while ((work = READ_ONCE(*pprev))) {
      91           0 :                 if (!match(work, data))
      92           0 :                         pprev = &work->next;
      93           0 :                 else if (cmpxchg(pprev, work, work->next) == work)
      94             :                         break;
      95             :         }
      96           0 :         raw_spin_unlock_irqrestore(&task->pi_lock, flags);
      97             : 
      98           0 :         return work;
      99             : }
     100             : 
     101           0 : static bool task_work_func_match(struct callback_head *cb, void *data)
     102             : {
     103           0 :         return cb->func == data;
     104             : }
     105             : 
     106             : /**
     107             :  * task_work_cancel - cancel a pending work added by task_work_add()
     108             :  * @task: the task which should execute the work
     109             :  * @func: identifies the work to remove
     110             :  *
     111             :  * Find the last queued pending work with ->func == @func and remove
     112             :  * it from queue.
     113             :  *
     114             :  * RETURNS:
     115             :  * The found work or NULL if not found.
     116             :  */
     117             : struct callback_head *
     118           0 : task_work_cancel(struct task_struct *task, task_work_func_t func)
     119             : {
     120           0 :         return task_work_cancel_match(task, task_work_func_match, func);
     121             : }
     122             : 
     123             : /**
     124             :  * task_work_run - execute the works added by task_work_add()
     125             :  *
     126             :  * Flush the pending works. Should be used by the core kernel code.
     127             :  * Called before the task returns to the user-mode or stops, or when
     128             :  * it exits. In the latter case task_work_add() can no longer add the
     129             :  * new work after task_work_run() returns.
     130             :  */
     131          93 : void task_work_run(void)
     132             : {
     133          93 :         struct task_struct *task = current;
     134             :         struct callback_head *work, *head, *next;
     135             : 
     136             :         for (;;) {
     137             :                 /*
     138             :                  * work->func() can do task_work_add(), do not set
     139             :                  * work_exited unless the list is empty.
     140             :                  */
     141             :                 do {
     142          93 :                         head = NULL;
     143          93 :                         work = READ_ONCE(task->task_works);
     144          93 :                         if (!work) {
     145          93 :                                 if (task->flags & PF_EXITING)
     146             :                                         head = &work_exited;
     147             :                                 else
     148             :                                         break;
     149             :                         }
     150         186 :                 } while (cmpxchg(&task->task_works, work, head) != work);
     151             : 
     152          93 :                 if (!work)
     153             :                         break;
     154             :                 /*
     155             :                  * Synchronize with task_work_cancel(). It can not remove
     156             :                  * the first entry == work, cmpxchg(task_works) must fail.
     157             :                  * But it can remove another entry from the ->next list.
     158             :                  */
     159           0 :                 raw_spin_lock_irq(&task->pi_lock);
     160           0 :                 raw_spin_unlock_irq(&task->pi_lock);
     161             : 
     162             :                 do {
     163           0 :                         next = work->next;
     164           0 :                         work->func(work);
     165           0 :                         work = next;
     166           0 :                         cond_resched();
     167           0 :                 } while (work);
     168             :         }
     169          93 : }