Merge branch 'for-3.19' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu

Pull percpu updates from Tejun Heo:
 "Nothing interesting.  A patch to convert the remaining __get_cpu_var()
  users, another to fix non-critical off-by-one in an assertion and a
  cosmetic conversion to lockless_dereference() in percpu-ref.

  The back-merge from mainline is to receive lockless_dereference()"

* 'for-3.19' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu:
  percpu: Replace smp_read_barrier_depends() with lockless_dereference()
  percpu: Convert remaining __get_cpu_var uses in 3.18-rcX
  percpu: off by one in BUG_ON()

diff --combined include/linux/percpu-refcount.h
index 530b249f7ea49faf2b9a252c1f114f6a0a87e365,6b0c818721427d8d6c8f1c6406937c99fe0052b4..b4337646388be2cfa76c8a3b1e09c48e29bc1851
--- 1/include/linux/percpu-refcount.h
--- 2/include/linux/percpu-refcount.h
+++ b/include/linux/percpu-refcount.h
@@@ -128,18 -128,10 +128,16 @@@ static inline void percpu_ref_kill(stru
  static inline bool __ref_is_percpu(struct percpu_ref *ref,
                                          unsigned long __percpu **percpu_countp)
  {
-       unsigned long percpu_ptr = ACCESS_ONCE(ref->percpu_count_ptr);
- 
        /* paired with smp_store_release() in percpu_ref_reinit() */
-       smp_read_barrier_depends();
+       unsigned long percpu_ptr = lockless_dereference(ref->percpu_count_ptr);
  
 -      if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC))
 +      /*
 +       * Theoretically, the following could test just ATOMIC; however,
 +       * then we'd have to mask off DEAD separately as DEAD may be
 +       * visible without ATOMIC if we race with percpu_ref_kill().  DEAD
 +       * implies ATOMIC anyway.  Test them together.
 +       */
 +      if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD))
                return false;
  
        *percpu_countp = (unsigned long __percpu *)percpu_ptr;
@@@ -147,41 -139,27 +145,41 @@@
  }
  
  /**
 - * percpu_ref_get - increment a percpu refcount
 + * percpu_ref_get_many - increment a percpu refcount
   * @ref: percpu_ref to get
 + * @nr: number of references to get
   *
 - * Analagous to atomic_long_inc().
 + * Analogous to atomic_long_add().
   *
   * This function is safe to call as long as @ref is between init and exit.
   */
 -static inline void percpu_ref_get(struct percpu_ref *ref)
 +static inline void percpu_ref_get_many(struct percpu_ref *ref, unsigned long nr)
  {
        unsigned long __percpu *percpu_count;
  
        rcu_read_lock_sched();
  
        if (__ref_is_percpu(ref, &percpu_count))
 -              this_cpu_inc(*percpu_count);
 +              this_cpu_add(*percpu_count, nr);
        else
 -              atomic_long_inc(&ref->count);
 +              atomic_long_add(nr, &ref->count);
  
        rcu_read_unlock_sched();
  }
  
 +/**
 + * percpu_ref_get - increment a percpu refcount
 + * @ref: percpu_ref to get
 + *
 + * Analagous to atomic_long_inc().
 + *
 + * This function is safe to call as long as @ref is between init and exit.
 + */
 +static inline void percpu_ref_get(struct percpu_ref *ref)
 +{
 +      percpu_ref_get_many(ref, 1);
 +}
 +
  /**
   * percpu_ref_tryget - try to increment a percpu refcount
   * @ref: percpu_ref to try-get
@@@ -245,43 -223,28 +243,43 @@@ static inline bool percpu_ref_tryget_li
  }
  
  /**
 - * percpu_ref_put - decrement a percpu refcount
 + * percpu_ref_put_many - decrement a percpu refcount
   * @ref: percpu_ref to put
 + * @nr: number of references to put
   *
   * Decrement the refcount, and if 0, call the release function (which was passed
   * to percpu_ref_init())
   *
   * This function is safe to call as long as @ref is between init and exit.
   */
 -static inline void percpu_ref_put(struct percpu_ref *ref)
 +static inline void percpu_ref_put_many(struct percpu_ref *ref, unsigned long nr)
  {
        unsigned long __percpu *percpu_count;
  
        rcu_read_lock_sched();
  
        if (__ref_is_percpu(ref, &percpu_count))
 -              this_cpu_dec(*percpu_count);
 -      else if (unlikely(atomic_long_dec_and_test(&ref->count)))
 +              this_cpu_sub(*percpu_count, nr);
 +      else if (unlikely(atomic_long_sub_and_test(nr, &ref->count)))
                ref->release(ref);
  
        rcu_read_unlock_sched();
  }
  
 +/**
 + * percpu_ref_put - decrement a percpu refcount
 + * @ref: percpu_ref to put
 + *
 + * Decrement the refcount, and if 0, call the release function (which was passed
 + * to percpu_ref_init())
 + *
 + * This function is safe to call as long as @ref is between init and exit.
 + */
 +static inline void percpu_ref_put(struct percpu_ref *ref)
 +{
 +      percpu_ref_put_many(ref, 1);
 +}
 +
  /**
   * percpu_ref_is_zero - test whether a percpu refcount reached zero
   * @ref: percpu_ref to test

diff --combined kernel/time/tick-sched.c
index 1f4356037a7d5e1068c148640757870c1f3854d3,b1c6a512cdd05ce022db0a75cf948b4f65df2520..4d54b7540585a83c41a2dc0da34b1f9e7e9077d7
--- 1/kernel/time/tick-sched.c
--- 2/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@@ -235,7 -235,7 +235,7 @@@ void tick_nohz_full_kick(void
        if (!tick_nohz_full_cpu(smp_processor_id()))
                return;
  
-       irq_work_queue(&__get_cpu_var(nohz_full_kick_work));
+       irq_work_queue(this_cpu_ptr(&nohz_full_kick_work));
  }
  
  /*
@@@ -585,7 -585,7 +585,7 @@@ static ktime_t tick_nohz_stop_sched_tic
                last_jiffies = jiffies;
        } while (read_seqretry(&jiffies_lock, seq));
  
 -      if (rcu_needs_cpu(cpu, &rcu_delta_jiffies) ||
 +      if (rcu_needs_cpu(&rcu_delta_jiffies) ||
            arch_needs_cpu() || irq_work_needs_cpu()) {
                next_jiffies = last_jiffies + 1;
                delta_jiffies = 1;