Line data    Source code

       1             : #ifndef INTERNAL_IO_WQ_H
       2             : #define INTERNAL_IO_WQ_H
       3             : 
       4             : #include <linux/refcount.h>
       5             : 
       6             : struct io_wq;
       7             : 
       8             : enum {
       9             :         IO_WQ_WORK_CANCEL       = 1,
      10             :         IO_WQ_WORK_HASHED       = 2,
      11             :         IO_WQ_WORK_UNBOUND      = 4,
      12             :         IO_WQ_WORK_CONCURRENT   = 16,
      13             : 
      14             :         IO_WQ_HASH_SHIFT        = 24,   /* upper 8 bits are used for hash key */
      15             : };
      16             : 
      17             : enum io_wq_cancel {
      18             :         IO_WQ_CANCEL_OK,        /* cancelled before started */
      19             :         IO_WQ_CANCEL_RUNNING,   /* found, running, and attempted cancelled */
      20             :         IO_WQ_CANCEL_NOTFOUND,  /* work not found */
      21             : };
      22             : 
      23             : struct io_wq_work_node {
      24             :         struct io_wq_work_node *next;
      25             : };
      26             : 
      27             : struct io_wq_work_list {
      28             :         struct io_wq_work_node *first;
      29             :         struct io_wq_work_node *last;
      30             : };
      31             : 
      32             : #define wq_list_for_each(pos, prv, head)                        \
      33             :         for (pos = (head)->first, prv = NULL; pos; prv = pos, pos = (pos)->next)
      34             : 
      35             : #define wq_list_for_each_resume(pos, prv)                       \
      36             :         for (; pos; prv = pos, pos = (pos)->next)
      37             : 
      38             : #define wq_list_empty(list)     (READ_ONCE((list)->first) == NULL)
      39             : #define INIT_WQ_LIST(list)      do {                            \
      40             :         (list)->first = NULL;                                        \
      41             : } while (0)
      42             : 
      43             : static inline void wq_list_add_after(struct io_wq_work_node *node,
      44             :                                      struct io_wq_work_node *pos,
      45             :                                      struct io_wq_work_list *list)
      46             : {
      47           0 :         struct io_wq_work_node *next = pos->next;
      48             : 
      49           0 :         pos->next = node;
      50           0 :         node->next = next;
      51           0 :         if (!next)
      52           0 :                 list->last = node;
      53             : }
      54             : 
      55             : /**
      56             :  * wq_list_merge - merge the second list to the first one.
      57             :  * @list0: the first list
      58             :  * @list1: the second list
      59             :  * Return the first node after mergence.
      60             :  */
      61             : static inline struct io_wq_work_node *wq_list_merge(struct io_wq_work_list *list0,
      62             :                                                     struct io_wq_work_list *list1)
      63             : {
      64             :         struct io_wq_work_node *ret;
      65             : 
      66           0 :         if (!list0->first) {
      67           0 :                 ret = list1->first;
      68             :         } else {
      69           0 :                 ret = list0->first;
      70           0 :                 list0->last->next = list1->first;
      71             :         }
      72           0 :         INIT_WQ_LIST(list0);
      73           0 :         INIT_WQ_LIST(list1);
      74             :         return ret;
      75             : }
      76             : 
      77             : static inline void wq_list_add_tail(struct io_wq_work_node *node,
      78             :                                     struct io_wq_work_list *list)
      79             : {
      80           0 :         node->next = NULL;
      81           0 :         if (!list->first) {
      82           0 :                 list->last = node;
      83           0 :                 WRITE_ONCE(list->first, node);
      84             :         } else {
      85           0 :                 list->last->next = node;
      86           0 :                 list->last = node;
      87             :         }
      88             : }
      89             : 
      90             : static inline void wq_list_add_head(struct io_wq_work_node *node,
      91             :                                     struct io_wq_work_list *list)
      92             : {
      93           0 :         node->next = list->first;
      94           0 :         if (!node->next)
      95           0 :                 list->last = node;
      96           0 :         WRITE_ONCE(list->first, node);
      97             : }
      98             : 
      99             : static inline void wq_list_cut(struct io_wq_work_list *list,
     100             :                                struct io_wq_work_node *last,
     101             :                                struct io_wq_work_node *prev)
     102             : {
     103             :         /* first in the list, if prev==NULL */
     104           0 :         if (!prev)
     105           0 :                 WRITE_ONCE(list->first, last->next);
     106             :         else
     107           0 :                 prev->next = last->next;
     108             : 
     109           0 :         if (last == list->last)
     110           0 :                 list->last = prev;
     111           0 :         last->next = NULL;
     112             : }
     113             : 
     114             : static inline void __wq_list_splice(struct io_wq_work_list *list,
     115             :                                     struct io_wq_work_node *to)
     116             : {
     117           0 :         list->last->next = to->next;
     118           0 :         to->next = list->first;
     119           0 :         INIT_WQ_LIST(list);
     120             : }
     121             : 
     122             : static inline bool wq_list_splice(struct io_wq_work_list *list,
     123             :                                   struct io_wq_work_node *to)
     124             : {
     125           0 :         if (!wq_list_empty(list)) {
     126           0 :                 __wq_list_splice(list, to);
     127             :                 return true;
     128             :         }
     129             :         return false;
     130             : }
     131             : 
     132             : static inline void wq_stack_add_head(struct io_wq_work_node *node,
     133             :                                      struct io_wq_work_node *stack)
     134             : {
     135           0 :         node->next = stack->next;
     136           0 :         stack->next = node;
     137             : }
     138             : 
     139             : static inline void wq_list_del(struct io_wq_work_list *list,
     140             :                                struct io_wq_work_node *node,
     141             :                                struct io_wq_work_node *prev)
     142             : {
     143           0 :         wq_list_cut(list, node, prev);
     144             : }
     145             : 
     146             : static inline
     147             : struct io_wq_work_node *wq_stack_extract(struct io_wq_work_node *stack)
     148             : {
     149           0 :         struct io_wq_work_node *node = stack->next;
     150             : 
     151           0 :         stack->next = node->next;
     152             :         return node;
     153             : }
     154             : 
     155             : struct io_wq_work {
     156             :         struct io_wq_work_node list;
     157             :         unsigned flags;
     158             : };
     159             : 
     160             : static inline struct io_wq_work *wq_next_work(struct io_wq_work *work)
     161             : {
     162           0 :         if (!work->list.next)
     163             :                 return NULL;
     164             : 
     165           0 :         return container_of(work->list.next, struct io_wq_work, list);
     166             : }
     167             : 
     168             : typedef struct io_wq_work *(free_work_fn)(struct io_wq_work *);
     169             : typedef void (io_wq_work_fn)(struct io_wq_work *);
     170             : 
     171             : struct io_wq_hash {
     172             :         refcount_t refs;
     173             :         unsigned long map;
     174             :         struct wait_queue_head wait;
     175             : };
     176             : 
     177           0 : static inline void io_wq_put_hash(struct io_wq_hash *hash)
     178             : {
     179           0 :         if (refcount_dec_and_test(&hash->refs))
     180           0 :                 kfree(hash);
     181           0 : }
     182             : 
     183             : struct io_wq_data {
     184             :         struct io_wq_hash *hash;
     185             :         struct task_struct *task;
     186             :         io_wq_work_fn *do_work;
     187             :         free_work_fn *free_work;
     188             : };
     189             : 
     190             : struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data);
     191             : void io_wq_exit_start(struct io_wq *wq);
     192             : void io_wq_put_and_exit(struct io_wq *wq);
     193             : 
     194             : void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work);
     195             : void io_wq_hash_work(struct io_wq_work *work, void *val);
     196             : 
     197             : int io_wq_cpu_affinity(struct io_wq *wq, cpumask_var_t mask);
     198             : int io_wq_max_workers(struct io_wq *wq, int *new_count);
     199             : 
     200             : static inline bool io_wq_is_hashed(struct io_wq_work *work)
     201             : {
     202           0 :         return work->flags & IO_WQ_WORK_HASHED;
     203             : }
     204             : 
     205             : typedef bool (work_cancel_fn)(struct io_wq_work *, void *);
     206             : 
     207             : enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel,
     208             :                                         void *data, bool cancel_all);
     209             : 
     210             : #if defined(CONFIG_IO_WQ)
     211             : extern void io_wq_worker_sleeping(struct task_struct *);
     212             : extern void io_wq_worker_running(struct task_struct *);
     213             : #else
     214             : static inline void io_wq_worker_sleeping(struct task_struct *tsk)
     215             : {
     216             : }
     217             : static inline void io_wq_worker_running(struct task_struct *tsk)
     218             : {
     219             : }
     220             : #endif
     221             : 
     222           0 : static inline bool io_wq_current_is_worker(void)
     223             : {
     224           0 :         return in_task() && (current->flags & PF_IO_WORKER) &&
     225           0 :                 current->worker_private;
     226             : }
     227             : #endif