include/linux/wait.h

   1 #ifndef _LINUX_WAIT_H
   2 #define _LINUX_WAIT_H
   3 /*
   4  * Linux wait queue related types and methods
   5  */
   6 #include <linux/list.h>
   7 #include <linux/stddef.h>
   8 #include <linux/spinlock.h>
   9
  10 #include <asm/current.h>
  11 #include <uapi/linux/wait.h>
  12
  13 typedef struct __wait_queue wait_queue_t;
  14 typedef int (*wait_queue_func_t)(wait_queue_t *wait, unsigned mode, int flags, void *key);
  15 int default_wake_function(wait_queue_t *wait, unsigned mode, int flags, void *key);
  16
  17 /* __wait_queue::flags */
  18 #define WQ_FLAG_EXCLUSIVE       0x01
  19 #define WQ_FLAG_WOKEN           0x02
  20
  21 struct __wait_queue {
  22         unsigned int            flags;
  23         void                    *private;
  24         wait_queue_func_t       func;
  25         struct list_head        task_list;
  26 };
  27
  28 struct wait_bit_key {
  29         void                    *flags;
  30         int                     bit_nr;
  31 #define WAIT_ATOMIC_T_BIT_NR    -1
  32         unsigned long           timeout;
  33 };
  34
  35 struct wait_bit_queue {
  36         struct wait_bit_key     key;
  37         wait_queue_t            wait;
  38 };
  39
  40 struct __wait_queue_head {
  41         spinlock_t              lock;
  42         struct list_head        task_list;
  43 };
  44 typedef struct __wait_queue_head wait_queue_head_t;
  45
  46 struct task_struct;
  47
  48 /*
  49  * Macros for declaration and initialisaton of the datatypes
  50  */
  51
  52 #define __WAITQUEUE_INITIALIZER(name, tsk) {                            \
  53         .private        = tsk,                                          \
  54         .func           = default_wake_function,                        \
  55         .task_list      = { NULL, NULL } }
  56
  57 #define DECLARE_WAITQUEUE(name, tsk)                                    \
  58         wait_queue_t name = __WAITQUEUE_INITIALIZER(name, tsk)
  59
  60 #define __WAIT_QUEUE_HEAD_INITIALIZER(name) {                           \
  61         .lock           = __SPIN_LOCK_UNLOCKED(name.lock),              \
  62         .task_list      = { &(name).task_list, &(name).task_list } }
  63
  64 #define DECLARE_WAIT_QUEUE_HEAD(name) \
  65         wait_queue_head_t name = __WAIT_QUEUE_HEAD_INITIALIZER(name)
  66
  67 #define __WAIT_BIT_KEY_INITIALIZER(word, bit)                           \
  68         { .flags = word, .bit_nr = bit, }
  69
  70 #define __WAIT_ATOMIC_T_KEY_INITIALIZER(p)                              \
  71         { .flags = p, .bit_nr = WAIT_ATOMIC_T_BIT_NR, }
  72
  73 extern void __init_waitqueue_head(wait_queue_head_t *q, const char *name, struct lock_class_key *);
  74
  75 #define init_waitqueue_head(q)                          \
  76         do {                                            \
  77                 static struct lock_class_key __key;     \
  78                                                         \
  79                 __init_waitqueue_head((q), #q, &__key); \
  80         } while (0)
  81
  82 #ifdef CONFIG_LOCKDEP
  83 # define __WAIT_QUEUE_HEAD_INIT_ONSTACK(name) \
  84         ({ init_waitqueue_head(&name); name; })
  85 # define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) \
  86         wait_queue_head_t name = __WAIT_QUEUE_HEAD_INIT_ONSTACK(name)
  87 #else
  88 # define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) DECLARE_WAIT_QUEUE_HEAD(name)
  89 #endif
  90
  91 static inline void init_waitqueue_entry(wait_queue_t *q, struct task_struct *p)
  92 {
  93         q->flags        = 0;
  94         q->private      = p;
  95         q->func         = default_wake_function;
  96 }
  97
  98 static inline void
  99 init_waitqueue_func_entry(wait_queue_t *q, wait_queue_func_t func)
 100 {
 101         q->flags        = 0;
 102         q->private      = NULL;
 103         q->func         = func;
 104 }
 105
 106 /**
 107  * waitqueue_active -- locklessly test for waiters on the queue
 108  * @q: the waitqueue to test for waiters
 109  *
 110  * returns true if the wait list is not empty
 111  *
 112  * NOTE: this function is lockless and requires care, incorrect usage _will_
 113  * lead to sporadic and non-obvious failure.
 114  *
 115  * Use either while holding wait_queue_head_t::lock or when used for wakeups
 116  * with an extra smp_mb() like:
 117  *
 118  *      CPU0 - waker                    CPU1 - waiter
 119  *
 120  *                                      for (;;) {
 121  *      @cond = true;                     prepare_to_wait(&wq, &wait, state);
 122  *      smp_mb();                         // smp_mb() from set_current_state()
 123  *      if (waitqueue_active(wq))         if (@cond)
 124  *        wake_up(wq);                      break;
 125  *                                        schedule();
 126  *                                      }
 127  *                                      finish_wait(&wq, &wait);
 128  *
 129  * Because without the explicit smp_mb() it's possible for the
 130  * waitqueue_active() load to get hoisted over the @cond store such that we'll
 131  * observe an empty wait list while the waiter might not observe @cond.
 132  *
 133  * Also note that this 'optimization' trades a spin_lock() for an smp_mb(),
 134  * which (when the lock is uncontended) are of roughly equal cost.
 135  */
 136 static inline int waitqueue_active(wait_queue_head_t *q)
 137 {
 138         return !list_empty(&q->task_list);
 139 }
 140
 141 /**
 142  * wq_has_sleeper - check if there are any waiting processes
 143  * @wq: wait queue head
 144  *
 145  * Returns true if wq has waiting processes
 146  *
 147  * Please refer to the comment for waitqueue_active.
 148  */
 149 static inline bool wq_has_sleeper(wait_queue_head_t *wq)
 150 {
 151         /*
 152          * We need to be sure we are in sync with the
 153          * add_wait_queue modifications to the wait queue.
 154          *
 155          * This memory barrier should be paired with one on the
 156          * waiting side.
 157          */
 158         smp_mb();
 159         return waitqueue_active(wq);
 160 }
 161
 162 extern void add_wait_queue(wait_queue_head_t *q, wait_queue_t *wait);
 163 extern void add_wait_queue_exclusive(wait_queue_head_t *q, wait_queue_t *wait);
 164 extern void remove_wait_queue(wait_queue_head_t *q, wait_queue_t *wait);
 165
 166 static inline void __add_wait_queue(wait_queue_head_t *head, wait_queue_t *new)
 167 {
 168         list_add(&new->task_list, &head->task_list);
 169 }
 170
 171 /*
 172  * Used for wake-one threads:
 173  */
 174 static inline void
 175 __add_wait_queue_exclusive(wait_queue_head_t *q, wait_queue_t *wait)
 176 {
 177         wait->flags |= WQ_FLAG_EXCLUSIVE;
 178         __add_wait_queue(q, wait);
 179 }
 180
 181 static inline void __add_wait_queue_tail(wait_queue_head_t *head,
 182                                          wait_queue_t *new)
 183 {
 184         list_add_tail(&new->task_list, &head->task_list);
 185 }
 186
 187 static inline void
 188 __add_wait_queue_tail_exclusive(wait_queue_head_t *q, wait_queue_t *wait)
 189 {
 190         wait->flags |= WQ_FLAG_EXCLUSIVE;
 191         __add_wait_queue_tail(q, wait);
 192 }
 193
 194 static inline void
 195 __remove_wait_queue(wait_queue_head_t *head, wait_queue_t *old)
 196 {
 197         list_del(&old->task_list);
 198 }
 199
 200 typedef int wait_bit_action_f(struct wait_bit_key *, int mode);
 201 void __wake_up(wait_queue_head_t *q, unsigned int mode, int nr, void *key);
 202 void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key);
 203 void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode, int nr, void *key);
 204 void __wake_up_locked(wait_queue_head_t *q, unsigned int mode, int nr);
 205 void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr);
 206 void __wake_up_bit(wait_queue_head_t *, void *, int);
 207 int __wait_on_bit(wait_queue_head_t *, struct wait_bit_queue *, wait_bit_action_f *, unsigned);
 208 int __wait_on_bit_lock(wait_queue_head_t *, struct wait_bit_queue *, wait_bit_action_f *, unsigned);
 209 void wake_up_bit(void *, int);
 210 void wake_up_atomic_t(atomic_t *);
 211 int out_of_line_wait_on_bit(void *, int, wait_bit_action_f *, unsigned);
 212 int out_of_line_wait_on_bit_timeout(void *, int, wait_bit_action_f *, unsigned, unsigned long);
 213 int out_of_line_wait_on_bit_lock(void *, int, wait_bit_action_f *, unsigned);
 214 int out_of_line_wait_on_atomic_t(atomic_t *, int (*)(atomic_t *), unsigned);
 215 wait_queue_head_t *bit_waitqueue(void *, int);
 216
 217 #define wake_up(x)                      __wake_up(x, TASK_NORMAL, 1, NULL)
 218 #define wake_up_nr(x, nr)               __wake_up(x, TASK_NORMAL, nr, NULL)
 219 #define wake_up_all(x)                  __wake_up(x, TASK_NORMAL, 0, NULL)
 220 #define wake_up_locked(x)               __wake_up_locked((x), TASK_NORMAL, 1)
 221 #define wake_up_all_locked(x)           __wake_up_locked((x), TASK_NORMAL, 0)
 222
 223 #define wake_up_interruptible(x)        __wake_up(x, TASK_INTERRUPTIBLE, 1, NULL)
 224 #define wake_up_interruptible_nr(x, nr) __wake_up(x, TASK_INTERRUPTIBLE, nr, NULL)
 225 #define wake_up_interruptible_all(x)    __wake_up(x, TASK_INTERRUPTIBLE, 0, NULL)
 226 #define wake_up_interruptible_sync(x)   __wake_up_sync((x), TASK_INTERRUPTIBLE, 1)
 227
 228 /*
 229  * Wakeup macros to be used to report events to the targets.
 230  */
 231 #define wake_up_poll(x, m)                                              \
 232         __wake_up(x, TASK_NORMAL, 1, (void *) (m))
 233 #define wake_up_locked_poll(x, m)                                       \
 234         __wake_up_locked_key((x), TASK_NORMAL, (void *) (m))
 235 #define wake_up_interruptible_poll(x, m)                                \
 236         __wake_up(x, TASK_INTERRUPTIBLE, 1, (void *) (m))
 237 #define wake_up_interruptible_sync_poll(x, m)                           \
 238         __wake_up_sync_key((x), TASK_INTERRUPTIBLE, 1, (void *) (m))
 239
 240 #define ___wait_cond_timeout(condition)                                 \
 241 ({                                                                      \
 242         bool __cond = (condition);                                      \
 243         if (__cond && !__ret)                                           \
 244                 __ret = 1;                                              \
 245         __cond || !__ret;                                               \
 246 })
 247
 248 #define ___wait_is_interruptible(state)                                 \
 249         (!__builtin_constant_p(state) ||                                \
 250                 state == TASK_INTERRUPTIBLE || state == TASK_KILLABLE)  \
 251
 252 extern void init_wait_entry(wait_queue_t *__wait, int flags);
 253
 254 /*
 255  * The below macro ___wait_event() has an explicit shadow of the __ret
 256  * variable when used from the wait_event_*() macros.
 257  *
 258  * This is so that both can use the ___wait_cond_timeout() construct
 259  * to wrap the condition.
 260  *
 261  * The type inconsistency of the wait_event_*() __ret variable is also
 262  * on purpose; we use long where we can return timeout values and int
 263  * otherwise.
 264  */
 265
 266 #define ___wait_event(wq, condition, state, exclusive, ret, cmd)        \
 267 ({                                                                      \
 268         __label__ __out;                                                \
 269         wait_queue_t __wait;                                            \
 270         long __ret = ret;       /* explicit shadow */                   \
 271                                                                         \
 272         init_wait_entry(&__wait, exclusive ? WQ_FLAG_EXCLUSIVE : 0);    \
 273         for (;;) {                                                      \
 274                 long __int = prepare_to_wait_event(&wq, &__wait, state);\
 275                                                                         \
 276                 if (condition)                                          \
 277                         break;                                          \
 278                                                                         \
 279                 if (___wait_is_interruptible(state) && __int) {         \
 280                         __ret = __int;                                  \
 281                         goto __out;                                     \
 282                 }                                                       \
 283                                                                         \
 284                 cmd;                                                    \
 285         }                                                               \
 286         finish_wait(&wq, &__wait);                                      \
 287 __out:  __ret;                                                          \
 288 })
 289
 290 #define __wait_event(wq, condition)                                     \
 291         (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0, 0,  \
 292                             schedule())
 293
 294 /**
 295  * wait_event - sleep until a condition gets true
 296  * @wq: the waitqueue to wait on
 297  * @condition: a C expression for the event to wait for
 298  *
 299  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 300  * @condition evaluates to true. The @condition is checked each time
 301  * the waitqueue @wq is woken up.
 302  *
 303  * wake_up() has to be called after changing any variable that could
 304  * change the result of the wait condition.
 305  */
 306 #define wait_event(wq, condition)                                       \
 307 do {                                                                    \
 308         might_sleep();                                                  \
 309         if (condition)                                                  \
 310                 break;                                                  \
 311         __wait_event(wq, condition);                                    \
 312 } while (0)
 313
 314 #define __io_wait_event(wq, condition)                                  \
 315         (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0, 0,  \
 316                             io_schedule())
 317
 318 /*
 319  * io_wait_event() -- like wait_event() but with io_schedule()
 320  */
 321 #define io_wait_event(wq, condition)                                    \
 322 do {                                                                    \
 323         might_sleep();                                                  \
 324         if (condition)                                                  \
 325                 break;                                                  \
 326         __io_wait_event(wq, condition);                                 \
 327 } while (0)
 328
 329 #define __wait_event_freezable(wq, condition)                           \
 330         ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 0, 0,          \
 331                             schedule(); try_to_freeze())
 332
 333 /**
 334  * wait_event_freezable - sleep (or freeze) until a condition gets true
 335  * @wq: the waitqueue to wait on
 336  * @condition: a C expression for the event to wait for
 337  *
 338  * The process is put to sleep (TASK_INTERRUPTIBLE -- so as not to contribute
 339  * to system load) until the @condition evaluates to true. The
 340  * @condition is checked each time the waitqueue @wq is woken up.
 341  *
 342  * wake_up() has to be called after changing any variable that could
 343  * change the result of the wait condition.
 344  */
 345 #define wait_event_freezable(wq, condition)                             \
 346 ({                                                                      \
 347         int __ret = 0;                                                  \
 348         might_sleep();                                                  \
 349         if (!(condition))                                               \
 350                 __ret = __wait_event_freezable(wq, condition);          \
 351         __ret;                                                          \
 352 })
 353
 354 #define __wait_event_timeout(wq, condition, timeout)                    \
 355         ___wait_event(wq, ___wait_cond_timeout(condition),              \
 356                       TASK_UNINTERRUPTIBLE, 0, timeout,                 \
 357                       __ret = schedule_timeout(__ret))
 358
 359 /**
 360  * wait_event_timeout - sleep until a condition gets true or a timeout elapses
 361  * @wq: the waitqueue to wait on
 362  * @condition: a C expression for the event to wait for
 363  * @timeout: timeout, in jiffies
 364  *
 365  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 366  * @condition evaluates to true. The @condition is checked each time
 367  * the waitqueue @wq is woken up.
 368  *
 369  * wake_up() has to be called after changing any variable that could
 370  * change the result of the wait condition.
 371  *
 372  * Returns:
 373  * 0 if the @condition evaluated to %false after the @timeout elapsed,
 374  * 1 if the @condition evaluated to %true after the @timeout elapsed,
 375  * or the remaining jiffies (at least 1) if the @condition evaluated
 376  * to %true before the @timeout elapsed.
 377  */
 378 #define wait_event_timeout(wq, condition, timeout)                      \
 379 ({                                                                      \
 380         long __ret = timeout;                                           \
 381         might_sleep();                                                  \
 382         if (!___wait_cond_timeout(condition))                           \
 383                 __ret = __wait_event_timeout(wq, condition, timeout);   \
 384         __ret;                                                          \
 385 })
 386
 387 #define __wait_event_freezable_timeout(wq, condition, timeout)          \
 388         ___wait_event(wq, ___wait_cond_timeout(condition),              \
 389                       TASK_INTERRUPTIBLE, 0, timeout,                   \
 390                       __ret = schedule_timeout(__ret); try_to_freeze())
 391
 392 /*
 393  * like wait_event_timeout() -- except it uses TASK_INTERRUPTIBLE to avoid
 394  * increasing load and is freezable.
 395  */
 396 #define wait_event_freezable_timeout(wq, condition, timeout)            \
 397 ({                                                                      \
 398         long __ret = timeout;                                           \
 399         might_sleep();                                                  \
 400         if (!___wait_cond_timeout(condition))                           \
 401                 __ret = __wait_event_freezable_timeout(wq, condition, timeout); \
 402         __ret;                                                          \
 403 })
 404
 405 #define __wait_event_exclusive_cmd(wq, condition, cmd1, cmd2)           \
 406         (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 1, 0,  \
 407                             cmd1; schedule(); cmd2)
 408 /*
 409  * Just like wait_event_cmd(), except it sets exclusive flag
 410  */
 411 #define wait_event_exclusive_cmd(wq, condition, cmd1, cmd2)             \
 412 do {                                                                    \
 413         if (condition)                                                  \
 414                 break;                                                  \
 415         __wait_event_exclusive_cmd(wq, condition, cmd1, cmd2);          \
 416 } while (0)
 417
 418 #define __wait_event_cmd(wq, condition, cmd1, cmd2)                     \
 419         (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0, 0,  \
 420                             cmd1; schedule(); cmd2)
 421
 422 /**
 423  * wait_event_cmd - sleep until a condition gets true
 424  * @wq: the waitqueue to wait on
 425  * @condition: a C expression for the event to wait for
 426  * @cmd1: the command will be executed before sleep
 427  * @cmd2: the command will be executed after sleep
 428  *
 429  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 430  * @condition evaluates to true. The @condition is checked each time
 431  * the waitqueue @wq is woken up.
 432  *
 433  * wake_up() has to be called after changing any variable that could
 434  * change the result of the wait condition.
 435  */
 436 #define wait_event_cmd(wq, condition, cmd1, cmd2)                       \
 437 do {                                                                    \
 438         if (condition)                                                  \
 439                 break;                                                  \
 440         __wait_event_cmd(wq, condition, cmd1, cmd2);                    \
 441 } while (0)
 442
 443 #define __wait_event_interruptible(wq, condition)                       \
 444         ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 0, 0,          \
 445                       schedule())
 446
 447 /**
 448  * wait_event_interruptible - sleep until a condition gets true
 449  * @wq: the waitqueue to wait on
 450  * @condition: a C expression for the event to wait for
 451  *
 452  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 453  * @condition evaluates to true or a signal is received.
 454  * The @condition is checked each time the waitqueue @wq is woken up.
 455  *
 456  * wake_up() has to be called after changing any variable that could
 457  * change the result of the wait condition.
 458  *
 459  * The function will return -ERESTARTSYS if it was interrupted by a
 460  * signal and 0 if @condition evaluated to true.
 461  */
 462 #define wait_event_interruptible(wq, condition)                         \
 463 ({                                                                      \
 464         int __ret = 0;                                                  \
 465         might_sleep();                                                  \
 466         if (!(condition))                                               \
 467                 __ret = __wait_event_interruptible(wq, condition);      \
 468         __ret;                                                          \
 469 })
 470
 471 #define __wait_event_interruptible_timeout(wq, condition, timeout)      \
 472         ___wait_event(wq, ___wait_cond_timeout(condition),              \
 473                       TASK_INTERRUPTIBLE, 0, timeout,                   \
 474                       __ret = schedule_timeout(__ret))
 475
 476 /**
 477  * wait_event_interruptible_timeout - sleep until a condition gets true or a timeout elapses
 478  * @wq: the waitqueue to wait on
 479  * @condition: a C expression for the event to wait for
 480  * @timeout: timeout, in jiffies
 481  *
 482  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 483  * @condition evaluates to true or a signal is received.
 484  * The @condition is checked each time the waitqueue @wq is woken up.
 485  *
 486  * wake_up() has to be called after changing any variable that could
 487  * change the result of the wait condition.
 488  *
 489  * Returns:
 490  * 0 if the @condition evaluated to %false after the @timeout elapsed,
 491  * 1 if the @condition evaluated to %true after the @timeout elapsed,
 492  * the remaining jiffies (at least 1) if the @condition evaluated
 493  * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
 494  * interrupted by a signal.
 495  */
 496 #define wait_event_interruptible_timeout(wq, condition, timeout)        \
 497 ({                                                                      \
 498         long __ret = timeout;                                           \
 499         might_sleep();                                                  \
 500         if (!___wait_cond_timeout(condition))                           \
 501                 __ret = __wait_event_interruptible_timeout(wq,          \
 502                                                 condition, timeout);    \
 503         __ret;                                                          \
 504 })
 505
 506 #define __wait_event_hrtimeout(wq, condition, timeout, state)           \
 507 ({                                                                      \
 508         int __ret = 0;                                                  \
 509         struct hrtimer_sleeper __t;                                     \
 510                                                                         \
 511         hrtimer_init_on_stack(&__t.timer, CLOCK_MONOTONIC,              \
 512                               HRTIMER_MODE_REL);                        \
 513         hrtimer_init_sleeper(&__t, current);                            \
 514         if ((timeout) != KTIME_MAX)                             \
 515                 hrtimer_start_range_ns(&__t.timer, timeout,             \
 516                                        current->timer_slack_ns,         \
 517                                        HRTIMER_MODE_REL);               \
 518                                                                         \
 519         __ret = ___wait_event(wq, condition, state, 0, 0,               \
 520                 if (!__t.task) {                                        \
 521                         __ret = -ETIME;                                 \
 522                         break;                                          \
 523                 }                                                       \
 524                 schedule());                                            \
 525                                                                         \
 526         hrtimer_cancel(&__t.timer);                                     \
 527         destroy_hrtimer_on_stack(&__t.timer);                           \
 528         __ret;                                                          \
 529 })
 530
 531 /**
 532  * wait_event_hrtimeout - sleep until a condition gets true or a timeout elapses
 533  * @wq: the waitqueue to wait on
 534  * @condition: a C expression for the event to wait for
 535  * @timeout: timeout, as a ktime_t
 536  *
 537  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 538  * @condition evaluates to true or a signal is received.
 539  * The @condition is checked each time the waitqueue @wq is woken up.
 540  *
 541  * wake_up() has to be called after changing any variable that could
 542  * change the result of the wait condition.
 543  *
 544  * The function returns 0 if @condition became true, or -ETIME if the timeout
 545  * elapsed.
 546  */
 547 #define wait_event_hrtimeout(wq, condition, timeout)                    \
 548 ({                                                                      \
 549         int __ret = 0;                                                  \
 550         might_sleep();                                                  \
 551         if (!(condition))                                               \
 552                 __ret = __wait_event_hrtimeout(wq, condition, timeout,  \
 553                                                TASK_UNINTERRUPTIBLE);   \
 554         __ret;                                                          \
 555 })
 556
 557 /**
 558  * wait_event_interruptible_hrtimeout - sleep until a condition gets true or a timeout elapses
 559  * @wq: the waitqueue to wait on
 560  * @condition: a C expression for the event to wait for
 561  * @timeout: timeout, as a ktime_t
 562  *
 563  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 564  * @condition evaluates to true or a signal is received.
 565  * The @condition is checked each time the waitqueue @wq is woken up.
 566  *
 567  * wake_up() has to be called after changing any variable that could
 568  * change the result of the wait condition.
 569  *
 570  * The function returns 0 if @condition became true, -ERESTARTSYS if it was
 571  * interrupted by a signal, or -ETIME if the timeout elapsed.
 572  */
 573 #define wait_event_interruptible_hrtimeout(wq, condition, timeout)      \
 574 ({                                                                      \
 575         long __ret = 0;                                                 \
 576         might_sleep();                                                  \
 577         if (!(condition))                                               \
 578                 __ret = __wait_event_hrtimeout(wq, condition, timeout,  \
 579                                                TASK_INTERRUPTIBLE);     \
 580         __ret;                                                          \
 581 })
 582
 583 #define __wait_event_interruptible_exclusive(wq, condition)             \
 584         ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0,          \
 585                       schedule())
 586
 587 #define wait_event_interruptible_exclusive(wq, condition)               \
 588 ({                                                                      \
 589         int __ret = 0;                                                  \
 590         might_sleep();                                                  \
 591         if (!(condition))                                               \
 592                 __ret = __wait_event_interruptible_exclusive(wq, condition);\
 593         __ret;                                                          \
 594 })
 595
 596 #define __wait_event_killable_exclusive(wq, condition)                  \
 597         ___wait_event(wq, condition, TASK_KILLABLE, 1, 0,               \
 598                       schedule())
 599
 600 #define wait_event_killable_exclusive(wq, condition)                    \
 601 ({                                                                      \
 602         int __ret = 0;                                                  \
 603         might_sleep();                                                  \
 604         if (!(condition))                                               \
 605                 __ret = __wait_event_killable_exclusive(wq, condition); \
 606         __ret;                                                          \
 607 })
 608
 609
 610 #define __wait_event_freezable_exclusive(wq, condition)                 \
 611         ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0,          \
 612                         schedule(); try_to_freeze())
 613
 614 #define wait_event_freezable_exclusive(wq, condition)                   \
 615 ({                                                                      \
 616         int __ret = 0;                                                  \
 617         might_sleep();                                                  \
 618         if (!(condition))                                               \
 619                 __ret = __wait_event_freezable_exclusive(wq, condition);\
 620         __ret;                                                          \
 621 })
 622
 623 extern int do_wait_intr(wait_queue_head_t *, wait_queue_t *);
 624 extern int do_wait_intr_irq(wait_queue_head_t *, wait_queue_t *);
 625
 626 #define __wait_event_interruptible_locked(wq, condition, exclusive, fn) \
 627 ({                                                                      \
 628         int __ret;                                                      \
 629         DEFINE_WAIT(__wait);                                            \
 630         if (exclusive)                                                  \
 631                 __wait.flags |= WQ_FLAG_EXCLUSIVE;                      \
 632         do {                                                            \
 633                 __ret = fn(&(wq), &__wait);                             \
 634                 if (__ret)                                              \
 635                         break;                                          \
 636         } while (!(condition));                                         \
 637         __remove_wait_queue(&(wq), &__wait);                            \
 638         __set_current_state(TASK_RUNNING);                              \
 639         __ret;                                                          \
 640 })
 641
 642
 643 /**
 644  * wait_event_interruptible_locked - sleep until a condition gets true
 645  * @wq: the waitqueue to wait on
 646  * @condition: a C expression for the event to wait for
 647  *
 648  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 649  * @condition evaluates to true or a signal is received.
 650  * The @condition is checked each time the waitqueue @wq is woken up.
 651  *
 652  * It must be called with wq.lock being held.  This spinlock is
 653  * unlocked while sleeping but @condition testing is done while lock
 654  * is held and when this macro exits the lock is held.
 655  *
 656  * The lock is locked/unlocked using spin_lock()/spin_unlock()
 657  * functions which must match the way they are locked/unlocked outside
 658  * of this macro.
 659  *
 660  * wake_up_locked() has to be called after changing any variable that could
 661  * change the result of the wait condition.
 662  *
 663  * The function will return -ERESTARTSYS if it was interrupted by a
 664  * signal and 0 if @condition evaluated to true.
 665  */
 666 #define wait_event_interruptible_locked(wq, condition)                  \
 667         ((condition)                                                    \
 668          ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr))
 669
 670 /**
 671  * wait_event_interruptible_locked_irq - sleep until a condition gets true
 672  * @wq: the waitqueue to wait on
 673  * @condition: a C expression for the event to wait for
 674  *
 675  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 676  * @condition evaluates to true or a signal is received.
 677  * The @condition is checked each time the waitqueue @wq is woken up.
 678  *
 679  * It must be called with wq.lock being held.  This spinlock is
 680  * unlocked while sleeping but @condition testing is done while lock
 681  * is held and when this macro exits the lock is held.
 682  *
 683  * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
 684  * functions which must match the way they are locked/unlocked outside
 685  * of this macro.
 686  *
 687  * wake_up_locked() has to be called after changing any variable that could
 688  * change the result of the wait condition.
 689  *
 690  * The function will return -ERESTARTSYS if it was interrupted by a
 691  * signal and 0 if @condition evaluated to true.
 692  */
 693 #define wait_event_interruptible_locked_irq(wq, condition)              \
 694         ((condition)                                                    \
 695          ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr_irq))
 696
 697 /**
 698  * wait_event_interruptible_exclusive_locked - sleep exclusively until a condition gets true
 699  * @wq: the waitqueue to wait on
 700  * @condition: a C expression for the event to wait for
 701  *
 702  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 703  * @condition evaluates to true or a signal is received.
 704  * The @condition is checked each time the waitqueue @wq is woken up.
 705  *
 706  * It must be called with wq.lock being held.  This spinlock is
 707  * unlocked while sleeping but @condition testing is done while lock
 708  * is held and when this macro exits the lock is held.
 709  *
 710  * The lock is locked/unlocked using spin_lock()/spin_unlock()
 711  * functions which must match the way they are locked/unlocked outside
 712  * of this macro.
 713  *
 714  * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
 715  * set thus when other process waits process on the list if this
 716  * process is awaken further processes are not considered.
 717  *
 718  * wake_up_locked() has to be called after changing any variable that could
 719  * change the result of the wait condition.
 720  *
 721  * The function will return -ERESTARTSYS if it was interrupted by a
 722  * signal and 0 if @condition evaluated to true.
 723  */
 724 #define wait_event_interruptible_exclusive_locked(wq, condition)        \
 725         ((condition)                                                    \
 726          ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr))
 727
 728 /**
 729  * wait_event_interruptible_exclusive_locked_irq - sleep until a condition gets true
 730  * @wq: the waitqueue to wait on
 731  * @condition: a C expression for the event to wait for
 732  *
 733  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 734  * @condition evaluates to true or a signal is received.
 735  * The @condition is checked each time the waitqueue @wq is woken up.
 736  *
 737  * It must be called with wq.lock being held.  This spinlock is
 738  * unlocked while sleeping but @condition testing is done while lock
 739  * is held and when this macro exits the lock is held.
 740  *
 741  * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
 742  * functions which must match the way they are locked/unlocked outside
 743  * of this macro.
 744  *
 745  * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
 746  * set thus when other process waits process on the list if this
 747  * process is awaken further processes are not considered.
 748  *
 749  * wake_up_locked() has to be called after changing any variable that could
 750  * change the result of the wait condition.
 751  *
 752  * The function will return -ERESTARTSYS if it was interrupted by a
 753  * signal and 0 if @condition evaluated to true.
 754  */
 755 #define wait_event_interruptible_exclusive_locked_irq(wq, condition)    \
 756         ((condition)                                                    \
 757          ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr_irq))
 758
 759
 760 #define __wait_event_killable(wq, condition)                            \
 761         ___wait_event(wq, condition, TASK_KILLABLE, 0, 0, schedule())
 762
 763 /**
 764  * wait_event_killable - sleep until a condition gets true
 765  * @wq: the waitqueue to wait on
 766  * @condition: a C expression for the event to wait for
 767  *
 768  * The process is put to sleep (TASK_KILLABLE) until the
 769  * @condition evaluates to true or a signal is received.
 770  * The @condition is checked each time the waitqueue @wq is woken up.
 771  *
 772  * wake_up() has to be called after changing any variable that could
 773  * change the result of the wait condition.
 774  *
 775  * The function will return -ERESTARTSYS if it was interrupted by a
 776  * signal and 0 if @condition evaluated to true.
 777  */
 778 #define wait_event_killable(wq, condition)                              \
 779 ({                                                                      \
 780         int __ret = 0;                                                  \
 781         might_sleep();                                                  \
 782         if (!(condition))                                               \
 783                 __ret = __wait_event_killable(wq, condition);           \
 784         __ret;                                                          \
 785 })
 786
 787
 788 #define __wait_event_lock_irq(wq, condition, lock, cmd)                 \
 789         (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0, 0,  \
 790                             spin_unlock_irq(&lock);                     \
 791                             cmd;                                        \
 792                             schedule();                                 \
 793                             spin_lock_irq(&lock))
 794
 795 /**
 796  * wait_event_lock_irq_cmd - sleep until a condition gets true. The
 797  *                           condition is checked under the lock. This
 798  *                           is expected to be called with the lock
 799  *                           taken.
 800  * @wq: the waitqueue to wait on
 801  * @condition: a C expression for the event to wait for
 802  * @lock: a locked spinlock_t, which will be released before cmd
 803  *        and schedule() and reacquired afterwards.
 804  * @cmd: a command which is invoked outside the critical section before
 805  *       sleep
 806  *
 807  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 808  * @condition evaluates to true. The @condition is checked each time
 809  * the waitqueue @wq is woken up.
 810  *
 811  * wake_up() has to be called after changing any variable that could
 812  * change the result of the wait condition.
 813  *
 814  * This is supposed to be called while holding the lock. The lock is
 815  * dropped before invoking the cmd and going to sleep and is reacquired
 816  * afterwards.
 817  */
 818 #define wait_event_lock_irq_cmd(wq, condition, lock, cmd)               \
 819 do {                                                                    \
 820         if (condition)                                                  \
 821                 break;                                                  \
 822         __wait_event_lock_irq(wq, condition, lock, cmd);                \
 823 } while (0)
 824
 825 /**
 826  * wait_event_lock_irq - sleep until a condition gets true. The
 827  *                       condition is checked under the lock. This
 828  *                       is expected to be called with the lock
 829  *                       taken.
 830  * @wq: the waitqueue to wait on
 831  * @condition: a C expression for the event to wait for
 832  * @lock: a locked spinlock_t, which will be released before schedule()
 833  *        and reacquired afterwards.
 834  *
 835  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 836  * @condition evaluates to true. The @condition is checked each time
 837  * the waitqueue @wq is woken up.
 838  *
 839  * wake_up() has to be called after changing any variable that could
 840  * change the result of the wait condition.
 841  *
 842  * This is supposed to be called while holding the lock. The lock is
 843  * dropped before going to sleep and is reacquired afterwards.
 844  */
 845 #define wait_event_lock_irq(wq, condition, lock)                        \
 846 do {                                                                    \
 847         if (condition)                                                  \
 848                 break;                                                  \
 849         __wait_event_lock_irq(wq, condition, lock, );                   \
 850 } while (0)
 851
 852
 853 #define __wait_event_interruptible_lock_irq(wq, condition, lock, cmd)   \
 854         ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 0, 0,          \
 855                       spin_unlock_irq(&lock);                           \
 856                       cmd;                                              \
 857                       schedule();                                       \
 858                       spin_lock_irq(&lock))
 859
 860 /**
 861  * wait_event_interruptible_lock_irq_cmd - sleep until a condition gets true.
 862  *              The condition is checked under the lock. This is expected to
 863  *              be called with the lock taken.
 864  * @wq: the waitqueue to wait on
 865  * @condition: a C expression for the event to wait for
 866  * @lock: a locked spinlock_t, which will be released before cmd and
 867  *        schedule() and reacquired afterwards.
 868  * @cmd: a command which is invoked outside the critical section before
 869  *       sleep
 870  *
 871  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 872  * @condition evaluates to true or a signal is received. The @condition is
 873  * checked each time the waitqueue @wq is woken up.
 874  *
 875  * wake_up() has to be called after changing any variable that could
 876  * change the result of the wait condition.
 877  *
 878  * This is supposed to be called while holding the lock. The lock is
 879  * dropped before invoking the cmd and going to sleep and is reacquired
 880  * afterwards.
 881  *
 882  * The macro will return -ERESTARTSYS if it was interrupted by a signal
 883  * and 0 if @condition evaluated to true.
 884  */
 885 #define wait_event_interruptible_lock_irq_cmd(wq, condition, lock, cmd) \
 886 ({                                                                      \
 887         int __ret = 0;                                                  \
 888         if (!(condition))                                               \
 889                 __ret = __wait_event_interruptible_lock_irq(wq,         \
 890                                                 condition, lock, cmd);  \
 891         __ret;                                                          \
 892 })
 893
 894 /**
 895  * wait_event_interruptible_lock_irq - sleep until a condition gets true.
 896  *              The condition is checked under the lock. This is expected
 897  *              to be called with the lock taken.
 898  * @wq: the waitqueue to wait on
 899  * @condition: a C expression for the event to wait for
 900  * @lock: a locked spinlock_t, which will be released before schedule()
 901  *        and reacquired afterwards.
 902  *
 903  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 904  * @condition evaluates to true or signal is received. The @condition is
 905  * checked each time the waitqueue @wq is woken up.
 906  *
 907  * wake_up() has to be called after changing any variable that could
 908  * change the result of the wait condition.
 909  *
 910  * This is supposed to be called while holding the lock. The lock is
 911  * dropped before going to sleep and is reacquired afterwards.
 912  *
 913  * The macro will return -ERESTARTSYS if it was interrupted by a signal
 914  * and 0 if @condition evaluated to true.
 915  */
 916 #define wait_event_interruptible_lock_irq(wq, condition, lock)          \
 917 ({                                                                      \
 918         int __ret = 0;                                                  \
 919         if (!(condition))                                               \
 920                 __ret = __wait_event_interruptible_lock_irq(wq,         \
 921                                                 condition, lock,);      \
 922         __ret;                                                          \
 923 })
 924
 925 #define __wait_event_interruptible_lock_irq_timeout(wq, condition,      \
 926                                                     lock, timeout)      \
 927         ___wait_event(wq, ___wait_cond_timeout(condition),              \
 928                       TASK_INTERRUPTIBLE, 0, timeout,                   \
 929                       spin_unlock_irq(&lock);                           \
 930                       __ret = schedule_timeout(__ret);                  \
 931                       spin_lock_irq(&lock));
 932
 933 /**
 934  * wait_event_interruptible_lock_irq_timeout - sleep until a condition gets
 935  *              true or a timeout elapses. The condition is checked under
 936  *              the lock. This is expected to be called with the lock taken.
 937  * @wq: the waitqueue to wait on
 938  * @condition: a C expression for the event to wait for
 939  * @lock: a locked spinlock_t, which will be released before schedule()
 940  *        and reacquired afterwards.
 941  * @timeout: timeout, in jiffies
 942  *
 943  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 944  * @condition evaluates to true or signal is received. The @condition is
 945  * checked each time the waitqueue @wq is woken up.
 946  *
 947  * wake_up() has to be called after changing any variable that could
 948  * change the result of the wait condition.
 949  *
 950  * This is supposed to be called while holding the lock. The lock is
 951  * dropped before going to sleep and is reacquired afterwards.
 952  *
 953  * The function returns 0 if the @timeout elapsed, -ERESTARTSYS if it
 954  * was interrupted by a signal, and the remaining jiffies otherwise
 955  * if the condition evaluated to true before the timeout elapsed.
 956  */
 957 #define wait_event_interruptible_lock_irq_timeout(wq, condition, lock,  \
 958                                                   timeout)              \
 959 ({                                                                      \
 960         long __ret = timeout;                                           \
 961         if (!___wait_cond_timeout(condition))                           \
 962                 __ret = __wait_event_interruptible_lock_irq_timeout(    \
 963                                         wq, condition, lock, timeout);  \
 964         __ret;                                                          \
 965 })
 966
 967 /*
 968  * Waitqueues which are removed from the waitqueue_head at wakeup time
 969  */
 970 void prepare_to_wait(wait_queue_head_t *q, wait_queue_t *wait, int state);
 971 void prepare_to_wait_exclusive(wait_queue_head_t *q, wait_queue_t *wait, int state);
 972 long prepare_to_wait_event(wait_queue_head_t *q, wait_queue_t *wait, int state);
 973 void finish_wait(wait_queue_head_t *q, wait_queue_t *wait);
 974 long wait_woken(wait_queue_t *wait, unsigned mode, long timeout);
 975 int woken_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key);
 976 int autoremove_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key);
 977 int wake_bit_function(wait_queue_t *wait, unsigned mode, int sync, void *key);
 978
 979 #define DEFINE_WAIT_FUNC(name, function)                                \
 980         wait_queue_t name = {                                           \
 981                 .private        = current,                              \
 982                 .func           = function,                             \
 983                 .task_list      = LIST_HEAD_INIT((name).task_list),     \
 984         }
 985
 986 #define DEFINE_WAIT(name) DEFINE_WAIT_FUNC(name, autoremove_wake_function)
 987
 988 #define DEFINE_WAIT_BIT(name, word, bit)                                \
 989         struct wait_bit_queue name = {                                  \
 990                 .key = __WAIT_BIT_KEY_INITIALIZER(word, bit),           \
 991                 .wait   = {                                             \
 992                         .private        = current,                      \
 993                         .func           = wake_bit_function,            \
 994                         .task_list      =                               \
 995                                 LIST_HEAD_INIT((name).wait.task_list),  \
 996                 },                                                      \
 997         }
 998
 999 #define init_wait(wait)                                                 \
1000         do {                                                            \
1001                 (wait)->private = current;                              \
1002                 (wait)->func = autoremove_wake_function;                \
1003                 INIT_LIST_HEAD(&(wait)->task_list);                     \
1004                 (wait)->flags = 0;                                      \
1005         } while (0)
1006
1007
1008 extern int bit_wait(struct wait_bit_key *, int);
1009 extern int bit_wait_io(struct wait_bit_key *, int);
1010 extern int bit_wait_timeout(struct wait_bit_key *, int);
1011 extern int bit_wait_io_timeout(struct wait_bit_key *, int);
1012
1013 /**
1014  * wait_on_bit - wait for a bit to be cleared
1015  * @word: the word being waited on, a kernel virtual address
1016  * @bit: the bit of the word being waited on
1017  * @mode: the task state to sleep in
1018  *
1019  * There is a standard hashed waitqueue table for generic use. This
1020  * is the part of the hashtable's accessor API that waits on a bit.
1021  * For instance, if one were to have waiters on a bitflag, one would
1022  * call wait_on_bit() in threads waiting for the bit to clear.
1023  * One uses wait_on_bit() where one is waiting for the bit to clear,
1024  * but has no intention of setting it.
1025  * Returned value will be zero if the bit was cleared, or non-zero
1026  * if the process received a signal and the mode permitted wakeup
1027  * on that signal.
1028  */
1029 static inline int
1030 wait_on_bit(unsigned long *word, int bit, unsigned mode)
1031 {
1032         might_sleep();
1033         if (!test_bit(bit, word))
1034                 return 0;
1035         return out_of_line_wait_on_bit(word, bit,
1036                                        bit_wait,
1037                                        mode);
1038 }
1039
1040 /**
1041  * wait_on_bit_io - wait for a bit to be cleared
1042  * @word: the word being waited on, a kernel virtual address
1043  * @bit: the bit of the word being waited on
1044  * @mode: the task state to sleep in
1045  *
1046  * Use the standard hashed waitqueue table to wait for a bit
1047  * to be cleared.  This is similar to wait_on_bit(), but calls
1048  * io_schedule() instead of schedule() for the actual waiting.
1049  *
1050  * Returned value will be zero if the bit was cleared, or non-zero
1051  * if the process received a signal and the mode permitted wakeup
1052  * on that signal.
1053  */
1054 static inline int
1055 wait_on_bit_io(unsigned long *word, int bit, unsigned mode)
1056 {
1057         might_sleep();
1058         if (!test_bit(bit, word))
1059                 return 0;
1060         return out_of_line_wait_on_bit(word, bit,
1061                                        bit_wait_io,
1062                                        mode);
1063 }
1064
1065 /**
1066  * wait_on_bit_timeout - wait for a bit to be cleared or a timeout elapses
1067  * @word: the word being waited on, a kernel virtual address
1068  * @bit: the bit of the word being waited on
1069  * @mode: the task state to sleep in
1070  * @timeout: timeout, in jiffies
1071  *
1072  * Use the standard hashed waitqueue table to wait for a bit
1073  * to be cleared. This is similar to wait_on_bit(), except also takes a
1074  * timeout parameter.
1075  *
1076  * Returned value will be zero if the bit was cleared before the
1077  * @timeout elapsed, or non-zero if the @timeout elapsed or process
1078  * received a signal and the mode permitted wakeup on that signal.
1079  */
1080 static inline int
1081 wait_on_bit_timeout(unsigned long *word, int bit, unsigned mode,
1082                     unsigned long timeout)
1083 {
1084         might_sleep();
1085         if (!test_bit(bit, word))
1086                 return 0;
1087         return out_of_line_wait_on_bit_timeout(word, bit,
1088                                                bit_wait_timeout,
1089                                                mode, timeout);
1090 }
1091
1092 /**
1093  * wait_on_bit_action - wait for a bit to be cleared
1094  * @word: the word being waited on, a kernel virtual address
1095  * @bit: the bit of the word being waited on
1096  * @action: the function used to sleep, which may take special actions
1097  * @mode: the task state to sleep in
1098  *
1099  * Use the standard hashed waitqueue table to wait for a bit
1100  * to be cleared, and allow the waiting action to be specified.
1101  * This is like wait_on_bit() but allows fine control of how the waiting
1102  * is done.
1103  *
1104  * Returned value will be zero if the bit was cleared, or non-zero
1105  * if the process received a signal and the mode permitted wakeup
1106  * on that signal.
1107  */
1108 static inline int
1109 wait_on_bit_action(unsigned long *word, int bit, wait_bit_action_f *action,
1110                    unsigned mode)
1111 {
1112         might_sleep();
1113         if (!test_bit(bit, word))
1114                 return 0;
1115         return out_of_line_wait_on_bit(word, bit, action, mode);
1116 }
1117
1118 /**
1119  * wait_on_bit_lock - wait for a bit to be cleared, when wanting to set it
1120  * @word: the word being waited on, a kernel virtual address
1121  * @bit: the bit of the word being waited on
1122  * @mode: the task state to sleep in
1123  *
1124  * There is a standard hashed waitqueue table for generic use. This
1125  * is the part of the hashtable's accessor API that waits on a bit
1126  * when one intends to set it, for instance, trying to lock bitflags.
1127  * For instance, if one were to have waiters trying to set bitflag
1128  * and waiting for it to clear before setting it, one would call
1129  * wait_on_bit() in threads waiting to be able to set the bit.
1130  * One uses wait_on_bit_lock() where one is waiting for the bit to
1131  * clear with the intention of setting it, and when done, clearing it.
1132  *
1133  * Returns zero if the bit was (eventually) found to be clear and was
1134  * set.  Returns non-zero if a signal was delivered to the process and
1135  * the @mode allows that signal to wake the process.
1136  */
1137 static inline int
1138 wait_on_bit_lock(unsigned long *word, int bit, unsigned mode)
1139 {
1140         might_sleep();
1141         if (!test_and_set_bit(bit, word))
1142                 return 0;
1143         return out_of_line_wait_on_bit_lock(word, bit, bit_wait, mode);
1144 }
1145
1146 /**
1147  * wait_on_bit_lock_io - wait for a bit to be cleared, when wanting to set it
1148  * @word: the word being waited on, a kernel virtual address
1149  * @bit: the bit of the word being waited on
1150  * @mode: the task state to sleep in
1151  *
1152  * Use the standard hashed waitqueue table to wait for a bit
1153  * to be cleared and then to atomically set it.  This is similar
1154  * to wait_on_bit(), but calls io_schedule() instead of schedule()
1155  * for the actual waiting.
1156  *
1157  * Returns zero if the bit was (eventually) found to be clear and was
1158  * set.  Returns non-zero if a signal was delivered to the process and
1159  * the @mode allows that signal to wake the process.
1160  */
1161 static inline int
1162 wait_on_bit_lock_io(unsigned long *word, int bit, unsigned mode)
1163 {
1164         might_sleep();
1165         if (!test_and_set_bit(bit, word))
1166                 return 0;
1167         return out_of_line_wait_on_bit_lock(word, bit, bit_wait_io, mode);
1168 }
1169
1170 /**
1171  * wait_on_bit_lock_action - wait for a bit to be cleared, when wanting to set it
1172  * @word: the word being waited on, a kernel virtual address
1173  * @bit: the bit of the word being waited on
1174  * @action: the function used to sleep, which may take special actions
1175  * @mode: the task state to sleep in
1176  *
1177  * Use the standard hashed waitqueue table to wait for a bit
1178  * to be cleared and then to set it, and allow the waiting action
1179  * to be specified.
1180  * This is like wait_on_bit() but allows fine control of how the waiting
1181  * is done.
1182  *
1183  * Returns zero if the bit was (eventually) found to be clear and was
1184  * set.  Returns non-zero if a signal was delivered to the process and
1185  * the @mode allows that signal to wake the process.
1186  */
1187 static inline int
1188 wait_on_bit_lock_action(unsigned long *word, int bit, wait_bit_action_f *action,
1189                         unsigned mode)
1190 {
1191         might_sleep();
1192         if (!test_and_set_bit(bit, word))
1193                 return 0;
1194         return out_of_line_wait_on_bit_lock(word, bit, action, mode);
1195 }
1196
1197 /**
1198  * wait_on_atomic_t - Wait for an atomic_t to become 0
1199  * @val: The atomic value being waited on, a kernel virtual address
1200  * @action: the function used to sleep, which may take special actions
1201  * @mode: the task state to sleep in
1202  *
1203  * Wait for an atomic_t to become 0.  We abuse the bit-wait waitqueue table for
1204  * the purpose of getting a waitqueue, but we set the key to a bit number
1205  * outside of the target 'word'.
1206  */
1207 static inline
1208 int wait_on_atomic_t(atomic_t *val, int (*action)(atomic_t *), unsigned mode)
1209 {
1210         might_sleep();
1211         if (atomic_read(val) == 0)
1212                 return 0;
1213         return out_of_line_wait_on_atomic_t(val, action, mode);
1214 }
1215
1216 #endif /* _LINUX_WAIT_H */