include/linux/wait.h

   1 #ifndef _LINUX_WAIT_H
   2 #define _LINUX_WAIT_H
   3 /*
   4  * Linux wait queue related types and methods
   5  */
   6 #include <linux/list.h>
   7 #include <linux/stddef.h>
   8 #include <linux/spinlock.h>
   9
  10 #include <asm/current.h>
  11 #include <uapi/linux/wait.h>
  12
  13 typedef struct __wait_queue wait_queue_t;
  14 typedef int (*wait_queue_func_t)(wait_queue_t *wait, unsigned mode, int flags, void *key);
  15 int default_wake_function(wait_queue_t *wait, unsigned mode, int flags, void *key);
  16
  17 /* __wait_queue::flags */
  18 #define WQ_FLAG_EXCLUSIVE       0x01
  19 #define WQ_FLAG_WOKEN           0x02
  20
  21 struct __wait_queue {
  22         unsigned int            flags;
  23         void                    *private;
  24         wait_queue_func_t       func;
  25         struct list_head        task_list;
  26 };
  27
  28 struct wait_bit_key {
  29         void                    *flags;
  30         int                     bit_nr;
  31 #define WAIT_ATOMIC_T_BIT_NR    -1
  32         unsigned long           timeout;
  33 };
  34
  35 struct wait_bit_queue {
  36         struct wait_bit_key     key;
  37         wait_queue_t            wait;
  38 };
  39
  40 struct __wait_queue_head {
  41         spinlock_t              lock;
  42         struct list_head        task_list;
  43 };
  44 typedef struct __wait_queue_head wait_queue_head_t;
  45
  46 struct task_struct;
  47
  48 /*
  49  * Macros for declaration and initialisaton of the datatypes
  50  */
  51
  52 #define __WAITQUEUE_INITIALIZER(name, tsk) {                            \
  53         .private        = tsk,                                          \
  54         .func           = default_wake_function,                        \
  55         .task_list      = { NULL, NULL } }
  56
  57 #define DECLARE_WAITQUEUE(name, tsk)                                    \
  58         wait_queue_t name = __WAITQUEUE_INITIALIZER(name, tsk)
  59
  60 #define __WAIT_QUEUE_HEAD_INITIALIZER(name) {                           \
  61         .lock           = __SPIN_LOCK_UNLOCKED(name.lock),              \
  62         .task_list      = { &(name).task_list, &(name).task_list } }
  63
  64 #define DECLARE_WAIT_QUEUE_HEAD(name) \
  65         wait_queue_head_t name = __WAIT_QUEUE_HEAD_INITIALIZER(name)
  66
  67 #define __WAIT_BIT_KEY_INITIALIZER(word, bit)                           \
  68         { .flags = word, .bit_nr = bit, }
  69
  70 #define __WAIT_ATOMIC_T_KEY_INITIALIZER(p)                              \
  71         { .flags = p, .bit_nr = WAIT_ATOMIC_T_BIT_NR, }
  72
  73 extern void __init_waitqueue_head(wait_queue_head_t *q, const char *name, struct lock_class_key *);
  74
  75 #define init_waitqueue_head(q)                          \
  76         do {                                            \
  77                 static struct lock_class_key __key;     \
  78                                                         \
  79                 __init_waitqueue_head((q), #q, &__key); \
  80         } while (0)
  81
  82 #ifdef CONFIG_LOCKDEP
  83 # define __WAIT_QUEUE_HEAD_INIT_ONSTACK(name) \
  84         ({ init_waitqueue_head(&name); name; })
  85 # define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) \
  86         wait_queue_head_t name = __WAIT_QUEUE_HEAD_INIT_ONSTACK(name)
  87 #else
  88 # define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) DECLARE_WAIT_QUEUE_HEAD(name)
  89 #endif
  90
  91 static inline void init_waitqueue_entry(wait_queue_t *q, struct task_struct *p)
  92 {
  93         q->flags        = 0;
  94         q->private      = p;
  95         q->func         = default_wake_function;
  96 }
  97
  98 static inline void
  99 init_waitqueue_func_entry(wait_queue_t *q, wait_queue_func_t func)
 100 {
 101         q->flags        = 0;
 102         q->private      = NULL;
 103         q->func         = func;
 104 }
 105
 106 /**
 107  * waitqueue_active -- locklessly test for waiters on the queue
 108  * @q: the waitqueue to test for waiters
 109  *
 110  * returns true if the wait list is not empty
 111  *
 112  * NOTE: this function is lockless and requires care, incorrect usage _will_
 113  * lead to sporadic and non-obvious failure.
 114  *
 115  * Use either while holding wait_queue_head_t::lock or when used for wakeups
 116  * with an extra smp_mb() like:
 117  *
 118  *      CPU0 - waker                    CPU1 - waiter
 119  *
 120  *                                      for (;;) {
 121  *      @cond = true;                     prepare_to_wait(&wq, &wait, state);
 122  *      smp_mb();                         // smp_mb() from set_current_state()
 123  *      if (waitqueue_active(wq))         if (@cond)
 124  *        wake_up(wq);                      break;
 125  *                                        schedule();
 126  *                                      }
 127  *                                      finish_wait(&wq, &wait);
 128  *
 129  * Because without the explicit smp_mb() it's possible for the
 130  * waitqueue_active() load to get hoisted over the @cond store such that we'll
 131  * observe an empty wait list while the waiter might not observe @cond.
 132  *
 133  * Also note that this 'optimization' trades a spin_lock() for an smp_mb(),
 134  * which (when the lock is uncontended) are of roughly equal cost.
 135  */
 136 static inline int waitqueue_active(wait_queue_head_t *q)
 137 {
 138         return !list_empty(&q->task_list);
 139 }
 140
 141 /**
 142  * wq_has_sleeper - check if there are any waiting processes
 143  * @wq: wait queue head
 144  *
 145  * Returns true if wq has waiting processes
 146  *
 147  * Please refer to the comment for waitqueue_active.
 148  */
 149 static inline bool wq_has_sleeper(wait_queue_head_t *wq)
 150 {
 151         /*
 152          * We need to be sure we are in sync with the
 153          * add_wait_queue modifications to the wait queue.
 154          *
 155          * This memory barrier should be paired with one on the
 156          * waiting side.
 157          */
 158         smp_mb();
 159         return waitqueue_active(wq);
 160 }
 161
 162 extern void add_wait_queue(wait_queue_head_t *q, wait_queue_t *wait);
 163 extern void add_wait_queue_exclusive(wait_queue_head_t *q, wait_queue_t *wait);
 164 extern void remove_wait_queue(wait_queue_head_t *q, wait_queue_t *wait);
 165
 166 static inline void __add_wait_queue(wait_queue_head_t *head, wait_queue_t *new)
 167 {
 168         list_add(&new->task_list, &head->task_list);
 169 }
 170
 171 /*
 172  * Used for wake-one threads:
 173  */
 174 static inline void
 175 __add_wait_queue_exclusive(wait_queue_head_t *q, wait_queue_t *wait)
 176 {
 177         wait->flags |= WQ_FLAG_EXCLUSIVE;
 178         __add_wait_queue(q, wait);
 179 }
 180
 181 static inline void __add_wait_queue_tail(wait_queue_head_t *head,
 182                                          wait_queue_t *new)
 183 {
 184         list_add_tail(&new->task_list, &head->task_list);
 185 }
 186
 187 static inline void
 188 __add_wait_queue_tail_exclusive(wait_queue_head_t *q, wait_queue_t *wait)
 189 {
 190         wait->flags |= WQ_FLAG_EXCLUSIVE;
 191         __add_wait_queue_tail(q, wait);
 192 }
 193
 194 static inline void
 195 __remove_wait_queue(wait_queue_head_t *head, wait_queue_t *old)
 196 {
 197         list_del(&old->task_list);
 198 }
 199
 200 typedef int wait_bit_action_f(struct wait_bit_key *, int mode);
 201 void __wake_up(wait_queue_head_t *q, unsigned int mode, int nr, void *key);
 202 void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key);
 203 void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode, int nr, void *key);
 204 void __wake_up_locked(wait_queue_head_t *q, unsigned int mode, int nr);
 205 void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr);
 206 void __wake_up_bit(wait_queue_head_t *, void *, int);
 207 int __wait_on_bit(wait_queue_head_t *, struct wait_bit_queue *, wait_bit_action_f *, unsigned);
 208 int __wait_on_bit_lock(wait_queue_head_t *, struct wait_bit_queue *, wait_bit_action_f *, unsigned);
 209 void wake_up_bit(void *, int);
 210 void wake_up_atomic_t(atomic_t *);
 211 int out_of_line_wait_on_bit(void *, int, wait_bit_action_f *, unsigned);
 212 int out_of_line_wait_on_bit_timeout(void *, int, wait_bit_action_f *, unsigned, unsigned long);
 213 int out_of_line_wait_on_bit_lock(void *, int, wait_bit_action_f *, unsigned);
 214 int out_of_line_wait_on_atomic_t(atomic_t *, int (*)(atomic_t *), unsigned);
 215 wait_queue_head_t *bit_waitqueue(void *, int);
 216
 217 #define wake_up(x)                      __wake_up(x, TASK_NORMAL, 1, NULL)
 218 #define wake_up_nr(x, nr)               __wake_up(x, TASK_NORMAL, nr, NULL)
 219 #define wake_up_all(x)                  __wake_up(x, TASK_NORMAL, 0, NULL)
 220 #define wake_up_locked(x)               __wake_up_locked((x), TASK_NORMAL, 1)
 221 #define wake_up_all_locked(x)           __wake_up_locked((x), TASK_NORMAL, 0)
 222
 223 #define wake_up_interruptible(x)        __wake_up(x, TASK_INTERRUPTIBLE, 1, NULL)
 224 #define wake_up_interruptible_nr(x, nr) __wake_up(x, TASK_INTERRUPTIBLE, nr, NULL)
 225 #define wake_up_interruptible_all(x)    __wake_up(x, TASK_INTERRUPTIBLE, 0, NULL)
 226 #define wake_up_interruptible_sync(x)   __wake_up_sync((x), TASK_INTERRUPTIBLE, 1)
 227
 228 /*
 229  * Wakeup macros to be used to report events to the targets.
 230  */
 231 #define wake_up_poll(x, m)                                              \
 232         __wake_up(x, TASK_NORMAL, 1, (void *) (m))
 233 #define wake_up_locked_poll(x, m)                                       \
 234         __wake_up_locked_key((x), TASK_NORMAL, (void *) (m))
 235 #define wake_up_interruptible_poll(x, m)                                \
 236         __wake_up(x, TASK_INTERRUPTIBLE, 1, (void *) (m))
 237 #define wake_up_interruptible_sync_poll(x, m)                           \
 238         __wake_up_sync_key((x), TASK_INTERRUPTIBLE, 1, (void *) (m))
 239
 240 #define ___wait_cond_timeout(condition)                                 \
 241 ({                                                                      \
 242         bool __cond = (condition);                                      \
 243         if (__cond && !__ret)                                           \
 244                 __ret = 1;                                              \
 245         __cond || !__ret;                                               \
 246 })
 247
 248 #define ___wait_is_interruptible(state)                                 \
 249         (!__builtin_constant_p(state) ||                                \
 250                 state == TASK_INTERRUPTIBLE || state == TASK_KILLABLE)  \
 251
 252 extern void init_wait_entry(wait_queue_t *__wait, int flags);
 253
 254 /*
 255  * The below macro ___wait_event() has an explicit shadow of the __ret
 256  * variable when used from the wait_event_*() macros.
 257  *
 258  * This is so that both can use the ___wait_cond_timeout() construct
 259  * to wrap the condition.
 260  *
 261  * The type inconsistency of the wait_event_*() __ret variable is also
 262  * on purpose; we use long where we can return timeout values and int
 263  * otherwise.
 264  */
 265
 266 #define ___wait_event(wq, condition, state, exclusive, ret, cmd)        \
 267 ({                                                                      \
 268         __label__ __out;                                                \
 269         wait_queue_t __wait;                                            \
 270         long __ret = ret;       /* explicit shadow */                   \
 271                                                                         \
 272         init_wait_entry(&__wait, exclusive ? WQ_FLAG_EXCLUSIVE : 0);    \
 273         for (;;) {                                                      \
 274                 long __int = prepare_to_wait_event(&wq, &__wait, state);\
 275                                                                         \
 276                 if (condition)                                          \
 277                         break;                                          \
 278                                                                         \
 279                 if (___wait_is_interruptible(state) && __int) {         \
 280                         __ret = __int;                                  \
 281                         goto __out;                                     \
 282                 }                                                       \
 283                                                                         \
 284                 cmd;                                                    \
 285         }                                                               \
 286         finish_wait(&wq, &__wait);                                      \
 287 __out:  __ret;                                                          \
 288 })
 289
 290 #define __wait_event(wq, condition)                                     \
 291         (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0, 0,  \
 292                             schedule())
 293
 294 /**
 295  * wait_event - sleep until a condition gets true
 296  * @wq: the waitqueue to wait on
 297  * @condition: a C expression for the event to wait for
 298  *
 299  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 300  * @condition evaluates to true. The @condition is checked each time
 301  * the waitqueue @wq is woken up.
 302  *
 303  * wake_up() has to be called after changing any variable that could
 304  * change the result of the wait condition.
 305  */
 306 #define wait_event(wq, condition)                                       \
 307 do {                                                                    \
 308         might_sleep();                                                  \
 309         if (condition)                                                  \
 310                 break;                                                  \
 311         __wait_event(wq, condition);                                    \
 312 } while (0)
 313
 314 #define __io_wait_event(wq, condition)                                  \
 315         (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0, 0,  \
 316                             io_schedule())
 317
 318 /*
 319  * io_wait_event() -- like wait_event() but with io_schedule()
 320  */
 321 #define io_wait_event(wq, condition)                                    \
 322 do {                                                                    \
 323         might_sleep();                                                  \
 324         if (condition)                                                  \
 325                 break;                                                  \
 326         __io_wait_event(wq, condition);                                 \
 327 } while (0)
 328
 329 #define __wait_event_freezable(wq, condition)                           \
 330         ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 0, 0,          \
 331                             schedule(); try_to_freeze())
 332
 333 /**
 334  * wait_event_freezable - sleep (or freeze) until a condition gets true
 335  * @wq: the waitqueue to wait on
 336  * @condition: a C expression for the event to wait for
 337  *
 338  * The process is put to sleep (TASK_INTERRUPTIBLE -- so as not to contribute
 339  * to system load) until the @condition evaluates to true. The
 340  * @condition is checked each time the waitqueue @wq is woken up.
 341  *
 342  * wake_up() has to be called after changing any variable that could
 343  * change the result of the wait condition.
 344  */
 345 #define wait_event_freezable(wq, condition)                             \
 346 ({                                                                      \
 347         int __ret = 0;                                                  \
 348         might_sleep();                                                  \
 349         if (!(condition))                                               \
 350                 __ret = __wait_event_freezable(wq, condition);          \
 351         __ret;                                                          \
 352 })
 353
 354 #define __wait_event_timeout(wq, condition, timeout)                    \
 355         ___wait_event(wq, ___wait_cond_timeout(condition),              \
 356                       TASK_UNINTERRUPTIBLE, 0, timeout,                 \
 357                       __ret = schedule_timeout(__ret))
 358
 359 /**
 360  * wait_event_timeout - sleep until a condition gets true or a timeout elapses
 361  * @wq: the waitqueue to wait on
 362  * @condition: a C expression for the event to wait for
 363  * @timeout: timeout, in jiffies
 364  *
 365  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 366  * @condition evaluates to true. The @condition is checked each time
 367  * the waitqueue @wq is woken up.
 368  *
 369  * wake_up() has to be called after changing any variable that could
 370  * change the result of the wait condition.
 371  *
 372  * Returns:
 373  * 0 if the @condition evaluated to %false after the @timeout elapsed,
 374  * 1 if the @condition evaluated to %true after the @timeout elapsed,
 375  * or the remaining jiffies (at least 1) if the @condition evaluated
 376  * to %true before the @timeout elapsed.
 377  */
 378 #define wait_event_timeout(wq, condition, timeout)                      \
 379 ({                                                                      \
 380         long __ret = timeout;                                           \
 381         might_sleep();                                                  \
 382         if (!___wait_cond_timeout(condition))                           \
 383                 __ret = __wait_event_timeout(wq, condition, timeout);   \
 384         __ret;                                                          \
 385 })
 386
 387 #define __wait_event_freezable_timeout(wq, condition, timeout)          \
 388         ___wait_event(wq, ___wait_cond_timeout(condition),              \
 389                       TASK_INTERRUPTIBLE, 0, timeout,                   \
 390                       __ret = schedule_timeout(__ret); try_to_freeze())
 391
 392 /*
 393  * like wait_event_timeout() -- except it uses TASK_INTERRUPTIBLE to avoid
 394  * increasing load and is freezable.
 395  */
 396 #define wait_event_freezable_timeout(wq, condition, timeout)            \
 397 ({                                                                      \
 398         long __ret = timeout;                                           \
 399         might_sleep();                                                  \
 400         if (!___wait_cond_timeout(condition))                           \
 401                 __ret = __wait_event_freezable_timeout(wq, condition, timeout); \
 402         __ret;                                                          \
 403 })
 404
 405 #define __wait_event_exclusive_cmd(wq, condition, cmd1, cmd2)           \
 406         (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 1, 0,  \
 407                             cmd1; schedule(); cmd2)
 408 /*
 409  * Just like wait_event_cmd(), except it sets exclusive flag
 410  */
 411 #define wait_event_exclusive_cmd(wq, condition, cmd1, cmd2)             \
 412 do {                                                                    \
 413         if (condition)                                                  \
 414                 break;                                                  \
 415         __wait_event_exclusive_cmd(wq, condition, cmd1, cmd2);          \
 416 } while (0)
 417
 418 #define __wait_event_cmd(wq, condition, cmd1, cmd2)                     \
 419         (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0, 0,  \
 420                             cmd1; schedule(); cmd2)
 421
 422 /**
 423  * wait_event_cmd - sleep until a condition gets true
 424  * @wq: the waitqueue to wait on
 425  * @condition: a C expression for the event to wait for
 426  * @cmd1: the command will be executed before sleep
 427  * @cmd2: the command will be executed after sleep
 428  *
 429  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 430  * @condition evaluates to true. The @condition is checked each time
 431  * the waitqueue @wq is woken up.
 432  *
 433  * wake_up() has to be called after changing any variable that could
 434  * change the result of the wait condition.
 435  */
 436 #define wait_event_cmd(wq, condition, cmd1, cmd2)                       \
 437 do {                                                                    \
 438         if (condition)                                                  \
 439                 break;                                                  \
 440         __wait_event_cmd(wq, condition, cmd1, cmd2);                    \
 441 } while (0)
 442
 443 #define __wait_event_interruptible(wq, condition)                       \
 444         ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 0, 0,          \
 445                       schedule())
 446
 447 /**
 448  * wait_event_interruptible - sleep until a condition gets true
 449  * @wq: the waitqueue to wait on
 450  * @condition: a C expression for the event to wait for
 451  *
 452  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 453  * @condition evaluates to true or a signal is received.
 454  * The @condition is checked each time the waitqueue @wq is woken up.
 455  *
 456  * wake_up() has to be called after changing any variable that could
 457  * change the result of the wait condition.
 458  *
 459  * The function will return -ERESTARTSYS if it was interrupted by a
 460  * signal and 0 if @condition evaluated to true.
 461  */
 462 #define wait_event_interruptible(wq, condition)                         \
 463 ({                                                                      \
 464         int __ret = 0;                                                  \
 465         might_sleep();                                                  \
 466         if (!(condition))                                               \
 467                 __ret = __wait_event_interruptible(wq, condition);      \
 468         __ret;                                                          \
 469 })
 470
 471 #define __wait_event_interruptible_timeout(wq, condition, timeout)      \
 472         ___wait_event(wq, ___wait_cond_timeout(condition),              \
 473                       TASK_INTERRUPTIBLE, 0, timeout,                   \
 474                       __ret = schedule_timeout(__ret))
 475
 476 /**
 477  * wait_event_interruptible_timeout - sleep until a condition gets true or a timeout elapses
 478  * @wq: the waitqueue to wait on
 479  * @condition: a C expression for the event to wait for
 480  * @timeout: timeout, in jiffies
 481  *
 482  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 483  * @condition evaluates to true or a signal is received.
 484  * The @condition is checked each time the waitqueue @wq is woken up.
 485  *
 486  * wake_up() has to be called after changing any variable that could
 487  * change the result of the wait condition.
 488  *
 489  * Returns:
 490  * 0 if the @condition evaluated to %false after the @timeout elapsed,
 491  * 1 if the @condition evaluated to %true after the @timeout elapsed,
 492  * the remaining jiffies (at least 1) if the @condition evaluated
 493  * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
 494  * interrupted by a signal.
 495  */
 496 #define wait_event_interruptible_timeout(wq, condition, timeout)        \
 497 ({                                                                      \
 498         long __ret = timeout;                                           \
 499         might_sleep();                                                  \
 500         if (!___wait_cond_timeout(condition))                           \
 501                 __ret = __wait_event_interruptible_timeout(wq,          \
 502                                                 condition, timeout);    \
 503         __ret;                                                          \
 504 })
 505
 506 #define __wait_event_hrtimeout(wq, condition, timeout, state)           \
 507 ({                                                                      \
 508         int __ret = 0;                                                  \
 509         struct hrtimer_sleeper __t;                                     \
 510                                                                         \
 511         hrtimer_init_on_stack(&__t.timer, CLOCK_MONOTONIC,              \
 512                               HRTIMER_MODE_REL);                        \
 513         hrtimer_init_sleeper(&__t, current);                            \
 514         if ((timeout) != KTIME_MAX)                             \
 515                 hrtimer_start_range_ns(&__t.timer, timeout,             \
 516                                        current->timer_slack_ns,         \
 517                                        HRTIMER_MODE_REL);               \
 518                                                                         \
 519         __ret = ___wait_event(wq, condition, state, 0, 0,               \
 520                 if (!__t.task) {                                        \
 521                         __ret = -ETIME;                                 \
 522                         break;                                          \
 523                 }                                                       \
 524                 schedule());                                            \
 525                                                                         \
 526         hrtimer_cancel(&__t.timer);                                     \
 527         destroy_hrtimer_on_stack(&__t.timer);                           \
 528         __ret;                                                          \
 529 })
 530
 531 /**
 532  * wait_event_hrtimeout - sleep until a condition gets true or a timeout elapses
 533  * @wq: the waitqueue to wait on
 534  * @condition: a C expression for the event to wait for
 535  * @timeout: timeout, as a ktime_t
 536  *
 537  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 538  * @condition evaluates to true or a signal is received.
 539  * The @condition is checked each time the waitqueue @wq is woken up.
 540  *
 541  * wake_up() has to be called after changing any variable that could
 542  * change the result of the wait condition.
 543  *
 544  * The function returns 0 if @condition became true, or -ETIME if the timeout
 545  * elapsed.
 546  */
 547 #define wait_event_hrtimeout(wq, condition, timeout)                    \
 548 ({                                                                      \
 549         int __ret = 0;                                                  \
 550         might_sleep();                                                  \
 551         if (!(condition))                                               \
 552                 __ret = __wait_event_hrtimeout(wq, condition, timeout,  \
 553                                                TASK_UNINTERRUPTIBLE);   \
 554         __ret;                                                          \
 555 })
 556
 557 /**
 558  * wait_event_interruptible_hrtimeout - sleep until a condition gets true or a timeout elapses
 559  * @wq: the waitqueue to wait on
 560  * @condition: a C expression for the event to wait for
 561  * @timeout: timeout, as a ktime_t
 562  *
 563  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 564  * @condition evaluates to true or a signal is received.
 565  * The @condition is checked each time the waitqueue @wq is woken up.
 566  *
 567  * wake_up() has to be called after changing any variable that could
 568  * change the result of the wait condition.
 569  *
 570  * The function returns 0 if @condition became true, -ERESTARTSYS if it was
 571  * interrupted by a signal, or -ETIME if the timeout elapsed.
 572  */
 573 #define wait_event_interruptible_hrtimeout(wq, condition, timeout)      \
 574 ({                                                                      \
 575         long __ret = 0;                                                 \
 576         might_sleep();                                                  \
 577         if (!(condition))                                               \
 578                 __ret = __wait_event_hrtimeout(wq, condition, timeout,  \
 579                                                TASK_INTERRUPTIBLE);     \
 580         __ret;                                                          \
 581 })
 582
 583 #define __wait_event_interruptible_exclusive(wq, condition)             \
 584         ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0,          \
 585                       schedule())
 586
 587 #define wait_event_interruptible_exclusive(wq, condition)               \
 588 ({                                                                      \
 589         int __ret = 0;                                                  \
 590         might_sleep();                                                  \
 591         if (!(condition))                                               \
 592                 __ret = __wait_event_interruptible_exclusive(wq, condition);\
 593         __ret;                                                          \
 594 })
 595
 596 #define __wait_event_killable_exclusive(wq, condition)                  \
 597         ___wait_event(wq, condition, TASK_KILLABLE, 1, 0,               \
 598                       schedule())
 599
 600 #define wait_event_killable_exclusive(wq, condition)                    \
 601 ({                                                                      \
 602         int __ret = 0;                                                  \
 603         might_sleep();                                                  \
 604         if (!(condition))                                               \
 605                 __ret = __wait_event_killable_exclusive(wq, condition); \
 606         __ret;                                                          \
 607 })
 608
 609
 610 #define __wait_event_freezable_exclusive(wq, condition)                 \
 611         ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0,          \
 612                         schedule(); try_to_freeze())
 613
 614 #define wait_event_freezable_exclusive(wq, condition)                   \
 615 ({                                                                      \
 616         int __ret = 0;                                                  \
 617         might_sleep();                                                  \
 618         if (!(condition))                                               \
 619                 __ret = __wait_event_freezable_exclusive(wq, condition);\
 620         __ret;                                                          \
 621 })
 622
 623
 624 #define __wait_event_interruptible_locked(wq, condition, exclusive, irq) \
 625 ({                                                                      \
 626         int __ret = 0;                                                  \
 627         DEFINE_WAIT(__wait);                                            \
 628         if (exclusive)                                                  \
 629                 __wait.flags |= WQ_FLAG_EXCLUSIVE;                      \
 630         do {                                                            \
 631                 if (likely(list_empty(&__wait.task_list)))              \
 632                         __add_wait_queue_tail(&(wq), &__wait);          \
 633                 set_current_state(TASK_INTERRUPTIBLE);                  \
 634                 if (signal_pending(current)) {                          \
 635                         __ret = -ERESTARTSYS;                           \
 636                         break;                                          \
 637                 }                                                       \
 638                 if (irq)                                                \
 639                         spin_unlock_irq(&(wq).lock);                    \
 640                 else                                                    \
 641                         spin_unlock(&(wq).lock);                        \
 642                 schedule();                                             \
 643                 if (irq)                                                \
 644                         spin_lock_irq(&(wq).lock);                      \
 645                 else                                                    \
 646                         spin_lock(&(wq).lock);                          \
 647         } while (!(condition));                                         \
 648         __remove_wait_queue(&(wq), &__wait);                            \
 649         __set_current_state(TASK_RUNNING);                              \
 650         __ret;                                                          \
 651 })
 652
 653
 654 /**
 655  * wait_event_interruptible_locked - sleep until a condition gets true
 656  * @wq: the waitqueue to wait on
 657  * @condition: a C expression for the event to wait for
 658  *
 659  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 660  * @condition evaluates to true or a signal is received.
 661  * The @condition is checked each time the waitqueue @wq is woken up.
 662  *
 663  * It must be called with wq.lock being held.  This spinlock is
 664  * unlocked while sleeping but @condition testing is done while lock
 665  * is held and when this macro exits the lock is held.
 666  *
 667  * The lock is locked/unlocked using spin_lock()/spin_unlock()
 668  * functions which must match the way they are locked/unlocked outside
 669  * of this macro.
 670  *
 671  * wake_up_locked() has to be called after changing any variable that could
 672  * change the result of the wait condition.
 673  *
 674  * The function will return -ERESTARTSYS if it was interrupted by a
 675  * signal and 0 if @condition evaluated to true.
 676  */
 677 #define wait_event_interruptible_locked(wq, condition)                  \
 678         ((condition)                                                    \
 679          ? 0 : __wait_event_interruptible_locked(wq, condition, 0, 0))
 680
 681 /**
 682  * wait_event_interruptible_locked_irq - sleep until a condition gets true
 683  * @wq: the waitqueue to wait on
 684  * @condition: a C expression for the event to wait for
 685  *
 686  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 687  * @condition evaluates to true or a signal is received.
 688  * The @condition is checked each time the waitqueue @wq is woken up.
 689  *
 690  * It must be called with wq.lock being held.  This spinlock is
 691  * unlocked while sleeping but @condition testing is done while lock
 692  * is held and when this macro exits the lock is held.
 693  *
 694  * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
 695  * functions which must match the way they are locked/unlocked outside
 696  * of this macro.
 697  *
 698  * wake_up_locked() has to be called after changing any variable that could
 699  * change the result of the wait condition.
 700  *
 701  * The function will return -ERESTARTSYS if it was interrupted by a
 702  * signal and 0 if @condition evaluated to true.
 703  */
 704 #define wait_event_interruptible_locked_irq(wq, condition)              \
 705         ((condition)                                                    \
 706          ? 0 : __wait_event_interruptible_locked(wq, condition, 0, 1))
 707
 708 /**
 709  * wait_event_interruptible_exclusive_locked - sleep exclusively until a condition gets true
 710  * @wq: the waitqueue to wait on
 711  * @condition: a C expression for the event to wait for
 712  *
 713  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 714  * @condition evaluates to true or a signal is received.
 715  * The @condition is checked each time the waitqueue @wq is woken up.
 716  *
 717  * It must be called with wq.lock being held.  This spinlock is
 718  * unlocked while sleeping but @condition testing is done while lock
 719  * is held and when this macro exits the lock is held.
 720  *
 721  * The lock is locked/unlocked using spin_lock()/spin_unlock()
 722  * functions which must match the way they are locked/unlocked outside
 723  * of this macro.
 724  *
 725  * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
 726  * set thus when other process waits process on the list if this
 727  * process is awaken further processes are not considered.
 728  *
 729  * wake_up_locked() has to be called after changing any variable that could
 730  * change the result of the wait condition.
 731  *
 732  * The function will return -ERESTARTSYS if it was interrupted by a
 733  * signal and 0 if @condition evaluated to true.
 734  */
 735 #define wait_event_interruptible_exclusive_locked(wq, condition)        \
 736         ((condition)                                                    \
 737          ? 0 : __wait_event_interruptible_locked(wq, condition, 1, 0))
 738
 739 /**
 740  * wait_event_interruptible_exclusive_locked_irq - sleep until a condition gets true
 741  * @wq: the waitqueue to wait on
 742  * @condition: a C expression for the event to wait for
 743  *
 744  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 745  * @condition evaluates to true or a signal is received.
 746  * The @condition is checked each time the waitqueue @wq is woken up.
 747  *
 748  * It must be called with wq.lock being held.  This spinlock is
 749  * unlocked while sleeping but @condition testing is done while lock
 750  * is held and when this macro exits the lock is held.
 751  *
 752  * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
 753  * functions which must match the way they are locked/unlocked outside
 754  * of this macro.
 755  *
 756  * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
 757  * set thus when other process waits process on the list if this
 758  * process is awaken further processes are not considered.
 759  *
 760  * wake_up_locked() has to be called after changing any variable that could
 761  * change the result of the wait condition.
 762  *
 763  * The function will return -ERESTARTSYS if it was interrupted by a
 764  * signal and 0 if @condition evaluated to true.
 765  */
 766 #define wait_event_interruptible_exclusive_locked_irq(wq, condition)    \
 767         ((condition)                                                    \
 768          ? 0 : __wait_event_interruptible_locked(wq, condition, 1, 1))
 769
 770
 771 #define __wait_event_killable(wq, condition)                            \
 772         ___wait_event(wq, condition, TASK_KILLABLE, 0, 0, schedule())
 773
 774 /**
 775  * wait_event_killable - sleep until a condition gets true
 776  * @wq: the waitqueue to wait on
 777  * @condition: a C expression for the event to wait for
 778  *
 779  * The process is put to sleep (TASK_KILLABLE) until the
 780  * @condition evaluates to true or a signal is received.
 781  * The @condition is checked each time the waitqueue @wq is woken up.
 782  *
 783  * wake_up() has to be called after changing any variable that could
 784  * change the result of the wait condition.
 785  *
 786  * The function will return -ERESTARTSYS if it was interrupted by a
 787  * signal and 0 if @condition evaluated to true.
 788  */
 789 #define wait_event_killable(wq, condition)                              \
 790 ({                                                                      \
 791         int __ret = 0;                                                  \
 792         might_sleep();                                                  \
 793         if (!(condition))                                               \
 794                 __ret = __wait_event_killable(wq, condition);           \
 795         __ret;                                                          \
 796 })
 797
 798
 799 #define __wait_event_lock_irq(wq, condition, lock, cmd)                 \
 800         (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0, 0,  \
 801                             spin_unlock_irq(&lock);                     \
 802                             cmd;                                        \
 803                             schedule();                                 \
 804                             spin_lock_irq(&lock))
 805
 806 /**
 807  * wait_event_lock_irq_cmd - sleep until a condition gets true. The
 808  *                           condition is checked under the lock. This
 809  *                           is expected to be called with the lock
 810  *                           taken.
 811  * @wq: the waitqueue to wait on
 812  * @condition: a C expression for the event to wait for
 813  * @lock: a locked spinlock_t, which will be released before cmd
 814  *        and schedule() and reacquired afterwards.
 815  * @cmd: a command which is invoked outside the critical section before
 816  *       sleep
 817  *
 818  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 819  * @condition evaluates to true. The @condition is checked each time
 820  * the waitqueue @wq is woken up.
 821  *
 822  * wake_up() has to be called after changing any variable that could
 823  * change the result of the wait condition.
 824  *
 825  * This is supposed to be called while holding the lock. The lock is
 826  * dropped before invoking the cmd and going to sleep and is reacquired
 827  * afterwards.
 828  */
 829 #define wait_event_lock_irq_cmd(wq, condition, lock, cmd)               \
 830 do {                                                                    \
 831         if (condition)                                                  \
 832                 break;                                                  \
 833         __wait_event_lock_irq(wq, condition, lock, cmd);                \
 834 } while (0)
 835
 836 /**
 837  * wait_event_lock_irq - sleep until a condition gets true. The
 838  *                       condition is checked under the lock. This
 839  *                       is expected to be called with the lock
 840  *                       taken.
 841  * @wq: the waitqueue to wait on
 842  * @condition: a C expression for the event to wait for
 843  * @lock: a locked spinlock_t, which will be released before schedule()
 844  *        and reacquired afterwards.
 845  *
 846  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 847  * @condition evaluates to true. The @condition is checked each time
 848  * the waitqueue @wq is woken up.
 849  *
 850  * wake_up() has to be called after changing any variable that could
 851  * change the result of the wait condition.
 852  *
 853  * This is supposed to be called while holding the lock. The lock is
 854  * dropped before going to sleep and is reacquired afterwards.
 855  */
 856 #define wait_event_lock_irq(wq, condition, lock)                        \
 857 do {                                                                    \
 858         if (condition)                                                  \
 859                 break;                                                  \
 860         __wait_event_lock_irq(wq, condition, lock, );                   \
 861 } while (0)
 862
 863
 864 #define __wait_event_interruptible_lock_irq(wq, condition, lock, cmd)   \
 865         ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 0, 0,          \
 866                       spin_unlock_irq(&lock);                           \
 867                       cmd;                                              \
 868                       schedule();                                       \
 869                       spin_lock_irq(&lock))
 870
 871 /**
 872  * wait_event_interruptible_lock_irq_cmd - sleep until a condition gets true.
 873  *              The condition is checked under the lock. This is expected to
 874  *              be called with the lock taken.
 875  * @wq: the waitqueue to wait on
 876  * @condition: a C expression for the event to wait for
 877  * @lock: a locked spinlock_t, which will be released before cmd and
 878  *        schedule() and reacquired afterwards.
 879  * @cmd: a command which is invoked outside the critical section before
 880  *       sleep
 881  *
 882  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 883  * @condition evaluates to true or a signal is received. The @condition is
 884  * checked each time the waitqueue @wq is woken up.
 885  *
 886  * wake_up() has to be called after changing any variable that could
 887  * change the result of the wait condition.
 888  *
 889  * This is supposed to be called while holding the lock. The lock is
 890  * dropped before invoking the cmd and going to sleep and is reacquired
 891  * afterwards.
 892  *
 893  * The macro will return -ERESTARTSYS if it was interrupted by a signal
 894  * and 0 if @condition evaluated to true.
 895  */
 896 #define wait_event_interruptible_lock_irq_cmd(wq, condition, lock, cmd) \
 897 ({                                                                      \
 898         int __ret = 0;                                                  \
 899         if (!(condition))                                               \
 900                 __ret = __wait_event_interruptible_lock_irq(wq,         \
 901                                                 condition, lock, cmd);  \
 902         __ret;                                                          \
 903 })
 904
 905 /**
 906  * wait_event_interruptible_lock_irq - sleep until a condition gets true.
 907  *              The condition is checked under the lock. This is expected
 908  *              to be called with the lock taken.
 909  * @wq: the waitqueue to wait on
 910  * @condition: a C expression for the event to wait for
 911  * @lock: a locked spinlock_t, which will be released before schedule()
 912  *        and reacquired afterwards.
 913  *
 914  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 915  * @condition evaluates to true or signal is received. The @condition is
 916  * checked each time the waitqueue @wq is woken up.
 917  *
 918  * wake_up() has to be called after changing any variable that could
 919  * change the result of the wait condition.
 920  *
 921  * This is supposed to be called while holding the lock. The lock is
 922  * dropped before going to sleep and is reacquired afterwards.
 923  *
 924  * The macro will return -ERESTARTSYS if it was interrupted by a signal
 925  * and 0 if @condition evaluated to true.
 926  */
 927 #define wait_event_interruptible_lock_irq(wq, condition, lock)          \
 928 ({                                                                      \
 929         int __ret = 0;                                                  \
 930         if (!(condition))                                               \
 931                 __ret = __wait_event_interruptible_lock_irq(wq,         \
 932                                                 condition, lock,);      \
 933         __ret;                                                          \
 934 })
 935
 936 #define __wait_event_interruptible_lock_irq_timeout(wq, condition,      \
 937                                                     lock, timeout)      \
 938         ___wait_event(wq, ___wait_cond_timeout(condition),              \
 939                       TASK_INTERRUPTIBLE, 0, timeout,                   \
 940                       spin_unlock_irq(&lock);                           \
 941                       __ret = schedule_timeout(__ret);                  \
 942                       spin_lock_irq(&lock));
 943
 944 /**
 945  * wait_event_interruptible_lock_irq_timeout - sleep until a condition gets
 946  *              true or a timeout elapses. The condition is checked under
 947  *              the lock. This is expected to be called with the lock taken.
 948  * @wq: the waitqueue to wait on
 949  * @condition: a C expression for the event to wait for
 950  * @lock: a locked spinlock_t, which will be released before schedule()
 951  *        and reacquired afterwards.
 952  * @timeout: timeout, in jiffies
 953  *
 954  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 955  * @condition evaluates to true or signal is received. The @condition is
 956  * checked each time the waitqueue @wq is woken up.
 957  *
 958  * wake_up() has to be called after changing any variable that could
 959  * change the result of the wait condition.
 960  *
 961  * This is supposed to be called while holding the lock. The lock is
 962  * dropped before going to sleep and is reacquired afterwards.
 963  *
 964  * The function returns 0 if the @timeout elapsed, -ERESTARTSYS if it
 965  * was interrupted by a signal, and the remaining jiffies otherwise
 966  * if the condition evaluated to true before the timeout elapsed.
 967  */
 968 #define wait_event_interruptible_lock_irq_timeout(wq, condition, lock,  \
 969                                                   timeout)              \
 970 ({                                                                      \
 971         long __ret = timeout;                                           \
 972         if (!___wait_cond_timeout(condition))                           \
 973                 __ret = __wait_event_interruptible_lock_irq_timeout(    \
 974                                         wq, condition, lock, timeout);  \
 975         __ret;                                                          \
 976 })
 977
 978 /*
 979  * Waitqueues which are removed from the waitqueue_head at wakeup time
 980  */
 981 void prepare_to_wait(wait_queue_head_t *q, wait_queue_t *wait, int state);
 982 void prepare_to_wait_exclusive(wait_queue_head_t *q, wait_queue_t *wait, int state);
 983 long prepare_to_wait_event(wait_queue_head_t *q, wait_queue_t *wait, int state);
 984 void finish_wait(wait_queue_head_t *q, wait_queue_t *wait);
 985 long wait_woken(wait_queue_t *wait, unsigned mode, long timeout);
 986 int woken_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key);
 987 int autoremove_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key);
 988 int wake_bit_function(wait_queue_t *wait, unsigned mode, int sync, void *key);
 989
 990 #define DEFINE_WAIT_FUNC(name, function)                                \
 991         wait_queue_t name = {                                           \
 992                 .private        = current,                              \
 993                 .func           = function,                             \
 994                 .task_list      = LIST_HEAD_INIT((name).task_list),     \
 995         }
 996
 997 #define DEFINE_WAIT(name) DEFINE_WAIT_FUNC(name, autoremove_wake_function)
 998
 999 #define DEFINE_WAIT_BIT(name, word, bit)                                \
1000         struct wait_bit_queue name = {                                  \
1001                 .key = __WAIT_BIT_KEY_INITIALIZER(word, bit),           \
1002                 .wait   = {                                             \
1003                         .private        = current,                      \
1004                         .func           = wake_bit_function,            \
1005                         .task_list      =                               \
1006                                 LIST_HEAD_INIT((name).wait.task_list),  \
1007                 },                                                      \
1008         }
1009
1010 #define init_wait(wait)                                                 \
1011         do {                                                            \
1012                 (wait)->private = current;                              \
1013                 (wait)->func = autoremove_wake_function;                \
1014                 INIT_LIST_HEAD(&(wait)->task_list);                     \
1015                 (wait)->flags = 0;                                      \
1016         } while (0)
1017
1018
1019 extern int bit_wait(struct wait_bit_key *, int);
1020 extern int bit_wait_io(struct wait_bit_key *, int);
1021 extern int bit_wait_timeout(struct wait_bit_key *, int);
1022 extern int bit_wait_io_timeout(struct wait_bit_key *, int);
1023
1024 /**
1025  * wait_on_bit - wait for a bit to be cleared
1026  * @word: the word being waited on, a kernel virtual address
1027  * @bit: the bit of the word being waited on
1028  * @mode: the task state to sleep in
1029  *
1030  * There is a standard hashed waitqueue table for generic use. This
1031  * is the part of the hashtable's accessor API that waits on a bit.
1032  * For instance, if one were to have waiters on a bitflag, one would
1033  * call wait_on_bit() in threads waiting for the bit to clear.
1034  * One uses wait_on_bit() where one is waiting for the bit to clear,
1035  * but has no intention of setting it.
1036  * Returned value will be zero if the bit was cleared, or non-zero
1037  * if the process received a signal and the mode permitted wakeup
1038  * on that signal.
1039  */
1040 static inline int
1041 wait_on_bit(unsigned long *word, int bit, unsigned mode)
1042 {
1043         might_sleep();
1044         if (!test_bit(bit, word))
1045                 return 0;
1046         return out_of_line_wait_on_bit(word, bit,
1047                                        bit_wait,
1048                                        mode);
1049 }
1050
1051 /**
1052  * wait_on_bit_io - wait for a bit to be cleared
1053  * @word: the word being waited on, a kernel virtual address
1054  * @bit: the bit of the word being waited on
1055  * @mode: the task state to sleep in
1056  *
1057  * Use the standard hashed waitqueue table to wait for a bit
1058  * to be cleared.  This is similar to wait_on_bit(), but calls
1059  * io_schedule() instead of schedule() for the actual waiting.
1060  *
1061  * Returned value will be zero if the bit was cleared, or non-zero
1062  * if the process received a signal and the mode permitted wakeup
1063  * on that signal.
1064  */
1065 static inline int
1066 wait_on_bit_io(unsigned long *word, int bit, unsigned mode)
1067 {
1068         might_sleep();
1069         if (!test_bit(bit, word))
1070                 return 0;
1071         return out_of_line_wait_on_bit(word, bit,
1072                                        bit_wait_io,
1073                                        mode);
1074 }
1075
1076 /**
1077  * wait_on_bit_timeout - wait for a bit to be cleared or a timeout elapses
1078  * @word: the word being waited on, a kernel virtual address
1079  * @bit: the bit of the word being waited on
1080  * @mode: the task state to sleep in
1081  * @timeout: timeout, in jiffies
1082  *
1083  * Use the standard hashed waitqueue table to wait for a bit
1084  * to be cleared. This is similar to wait_on_bit(), except also takes a
1085  * timeout parameter.
1086  *
1087  * Returned value will be zero if the bit was cleared before the
1088  * @timeout elapsed, or non-zero if the @timeout elapsed or process
1089  * received a signal and the mode permitted wakeup on that signal.
1090  */
1091 static inline int
1092 wait_on_bit_timeout(unsigned long *word, int bit, unsigned mode,
1093                     unsigned long timeout)
1094 {
1095         might_sleep();
1096         if (!test_bit(bit, word))
1097                 return 0;
1098         return out_of_line_wait_on_bit_timeout(word, bit,
1099                                                bit_wait_timeout,
1100                                                mode, timeout);
1101 }
1102
1103 /**
1104  * wait_on_bit_action - wait for a bit to be cleared
1105  * @word: the word being waited on, a kernel virtual address
1106  * @bit: the bit of the word being waited on
1107  * @action: the function used to sleep, which may take special actions
1108  * @mode: the task state to sleep in
1109  *
1110  * Use the standard hashed waitqueue table to wait for a bit
1111  * to be cleared, and allow the waiting action to be specified.
1112  * This is like wait_on_bit() but allows fine control of how the waiting
1113  * is done.
1114  *
1115  * Returned value will be zero if the bit was cleared, or non-zero
1116  * if the process received a signal and the mode permitted wakeup
1117  * on that signal.
1118  */
1119 static inline int
1120 wait_on_bit_action(unsigned long *word, int bit, wait_bit_action_f *action,
1121                    unsigned mode)
1122 {
1123         might_sleep();
1124         if (!test_bit(bit, word))
1125                 return 0;
1126         return out_of_line_wait_on_bit(word, bit, action, mode);
1127 }
1128
1129 /**
1130  * wait_on_bit_lock - wait for a bit to be cleared, when wanting to set it
1131  * @word: the word being waited on, a kernel virtual address
1132  * @bit: the bit of the word being waited on
1133  * @mode: the task state to sleep in
1134  *
1135  * There is a standard hashed waitqueue table for generic use. This
1136  * is the part of the hashtable's accessor API that waits on a bit
1137  * when one intends to set it, for instance, trying to lock bitflags.
1138  * For instance, if one were to have waiters trying to set bitflag
1139  * and waiting for it to clear before setting it, one would call
1140  * wait_on_bit() in threads waiting to be able to set the bit.
1141  * One uses wait_on_bit_lock() where one is waiting for the bit to
1142  * clear with the intention of setting it, and when done, clearing it.
1143  *
1144  * Returns zero if the bit was (eventually) found to be clear and was
1145  * set.  Returns non-zero if a signal was delivered to the process and
1146  * the @mode allows that signal to wake the process.
1147  */
1148 static inline int
1149 wait_on_bit_lock(unsigned long *word, int bit, unsigned mode)
1150 {
1151         might_sleep();
1152         if (!test_and_set_bit(bit, word))
1153                 return 0;
1154         return out_of_line_wait_on_bit_lock(word, bit, bit_wait, mode);
1155 }
1156
1157 /**
1158  * wait_on_bit_lock_io - wait for a bit to be cleared, when wanting to set it
1159  * @word: the word being waited on, a kernel virtual address
1160  * @bit: the bit of the word being waited on
1161  * @mode: the task state to sleep in
1162  *
1163  * Use the standard hashed waitqueue table to wait for a bit
1164  * to be cleared and then to atomically set it.  This is similar
1165  * to wait_on_bit(), but calls io_schedule() instead of schedule()
1166  * for the actual waiting.
1167  *
1168  * Returns zero if the bit was (eventually) found to be clear and was
1169  * set.  Returns non-zero if a signal was delivered to the process and
1170  * the @mode allows that signal to wake the process.
1171  */
1172 static inline int
1173 wait_on_bit_lock_io(unsigned long *word, int bit, unsigned mode)
1174 {
1175         might_sleep();
1176         if (!test_and_set_bit(bit, word))
1177                 return 0;
1178         return out_of_line_wait_on_bit_lock(word, bit, bit_wait_io, mode);
1179 }
1180
1181 /**
1182  * wait_on_bit_lock_action - wait for a bit to be cleared, when wanting to set it
1183  * @word: the word being waited on, a kernel virtual address
1184  * @bit: the bit of the word being waited on
1185  * @action: the function used to sleep, which may take special actions
1186  * @mode: the task state to sleep in
1187  *
1188  * Use the standard hashed waitqueue table to wait for a bit
1189  * to be cleared and then to set it, and allow the waiting action
1190  * to be specified.
1191  * This is like wait_on_bit() but allows fine control of how the waiting
1192  * is done.
1193  *
1194  * Returns zero if the bit was (eventually) found to be clear and was
1195  * set.  Returns non-zero if a signal was delivered to the process and
1196  * the @mode allows that signal to wake the process.
1197  */
1198 static inline int
1199 wait_on_bit_lock_action(unsigned long *word, int bit, wait_bit_action_f *action,
1200                         unsigned mode)
1201 {
1202         might_sleep();
1203         if (!test_and_set_bit(bit, word))
1204                 return 0;
1205         return out_of_line_wait_on_bit_lock(word, bit, action, mode);
1206 }
1207
1208 /**
1209  * wait_on_atomic_t - Wait for an atomic_t to become 0
1210  * @val: The atomic value being waited on, a kernel virtual address
1211  * @action: the function used to sleep, which may take special actions
1212  * @mode: the task state to sleep in
1213  *
1214  * Wait for an atomic_t to become 0.  We abuse the bit-wait waitqueue table for
1215  * the purpose of getting a waitqueue, but we set the key to a bit number
1216  * outside of the target 'word'.
1217  */
1218 static inline
1219 int wait_on_atomic_t(atomic_t *val, int (*action)(atomic_t *), unsigned mode)
1220 {
1221         might_sleep();
1222         if (atomic_read(val) == 0)
1223                 return 0;
1224         return out_of_line_wait_on_atomic_t(val, action, mode);
1225 }
1226
1227 #endif /* _LINUX_WAIT_H */