Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu

* 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu:
  sched: move rq_weight data array out of .percpu
  percpu: allow pcpu_alloc() to be called with IRQs off

diff --combined kernel/sched.c
index e88689522e66efa87942141e7ef24e28a97c8d73,526d237b8ce50c66c909d07a380f8da58b91bac3..a455dca884a6231ddf34e28bc9f0bb1e47ddb34d
--- 1/kernel/sched.c
--- 2/kernel/sched.c
+++ b/kernel/sched.c
@@@ -676,7 -676,6 +676,7 @@@ inline void update_rq_clock(struct rq *
  
  /**
   * runqueue_is_locked
 + * @cpu: the processor in question.
   *
   * Returns true if the current cpu runqueue is locked.
   * This interface allows printk to be called with the runqueue lock
@@@ -781,7 -780,7 +781,7 @@@ static int sched_feat_open(struct inod
        return single_open(filp, sched_feat_show, NULL);
  }
  
 -static struct file_operations sched_feat_fops = {
 +static const struct file_operations sched_feat_fops = {
        .open           = sched_feat_open,
        .write          = sched_feat_write,
        .read           = seq_read,
@@@ -1564,11 -1563,7 +1564,7 @@@ static unsigned long cpu_avg_load_per_t
  
  #ifdef CONFIG_FAIR_GROUP_SCHED
  
- struct update_shares_data {
-       unsigned long rq_weight[NR_CPUS];
- };
- 
- static DEFINE_PER_CPU(struct update_shares_data, update_shares_data);
+ static __read_mostly unsigned long *update_shares_data;
  
  static void __set_se_shares(struct sched_entity *se, unsigned long shares);
  
@@@ -1578,12 -1573,12 +1574,12 @@@
  static void update_group_shares_cpu(struct task_group *tg, int cpu,
                                    unsigned long sd_shares,
                                    unsigned long sd_rq_weight,
-                                   struct update_shares_data *usd)
+                                   unsigned long *usd_rq_weight)
  {
        unsigned long shares, rq_weight;
        int boost = 0;
  
-       rq_weight = usd->rq_weight[cpu];
+       rq_weight = usd_rq_weight[cpu];
        if (!rq_weight) {
                boost = 1;
                rq_weight = NICE_0_LOAD;
@@@ -1618,7 -1613,7 +1614,7 @@@
  static int tg_shares_up(struct task_group *tg, void *data)
  {
        unsigned long weight, rq_weight = 0, shares = 0;
-       struct update_shares_data *usd;
+       unsigned long *usd_rq_weight;
        struct sched_domain *sd = data;
        unsigned long flags;
        int i;
@@@ -1627,11 -1622,11 +1623,11 @@@
                return 0;
  
        local_irq_save(flags);
-       usd = &__get_cpu_var(update_shares_data);
+       usd_rq_weight = per_cpu_ptr(update_shares_data, smp_processor_id());
  
        for_each_cpu(i, sched_domain_span(sd)) {
                weight = tg->cfs_rq[i]->load.weight;
-               usd->rq_weight[i] = weight;
+               usd_rq_weight[i] = weight;
  
                /*
                 * If there are currently no tasks on the cpu pretend there
@@@ -1652,7 -1647,7 +1648,7 @@@
                shares = tg->shares;
  
        for_each_cpu(i, sched_domain_span(sd))
-               update_group_shares_cpu(tg, i, shares, rq_weight, usd);
+               update_group_shares_cpu(tg, i, shares, rq_weight, usd_rq_weight);
  
        local_irq_restore(flags);
  
@@@ -2312,7 -2307,7 +2308,7 @@@ static int try_to_wake_up(struct task_s
  {
        int cpu, orig_cpu, this_cpu, success = 0;
        unsigned long flags;
 -      struct rq *rq;
 +      struct rq *rq, *orig_rq;
  
        if (!sched_feat(SYNC_WAKEUPS))
                wake_flags &= ~WF_SYNC;
@@@ -2320,7 -2315,7 +2316,7 @@@
        this_cpu = get_cpu();
  
        smp_wmb();
 -      rq = task_rq_lock(p, &flags);
 +      rq = orig_rq = task_rq_lock(p, &flags);
        update_rq_clock(rq);
        if (!(p->state & state))
                goto out;
@@@ -2351,10 -2346,6 +2347,10 @@@
                set_task_cpu(p, cpu);
  
        rq = task_rq_lock(p, &flags);
 +
 +      if (rq != orig_rq)
 +              update_rq_clock(rq);
 +
        WARN_ON(p->state != TASK_WAKING);
        cpu = task_cpu(p);
  
@@@ -2519,18 -2510,23 +2515,18 @@@ void sched_fork(struct task_struct *p, 
  
        __sched_fork(p);
  
 -      /*
 -       * Make sure we do not leak PI boosting priority to the child.
 -       */
 -      p->prio = current->normal_prio;
 -
        /*
         * Revert to default priority/policy on fork if requested.
         */
        if (unlikely(p->sched_reset_on_fork)) {
 -              if (p->policy == SCHED_FIFO || p->policy == SCHED_RR)
 +              if (p->policy == SCHED_FIFO || p->policy == SCHED_RR) {
                        p->policy = SCHED_NORMAL;
 -
 -              if (p->normal_prio < DEFAULT_PRIO)
 -                      p->prio = DEFAULT_PRIO;
 +                      p->normal_prio = p->static_prio;
 +              }
  
                if (PRIO_TO_NICE(p->static_prio) < 0) {
                        p->static_prio = NICE_TO_PRIO(0);
 +                      p->normal_prio = p->static_prio;
                        set_load_weight(p);
                }
  
@@@ -2541,11 -2537,6 +2537,11 @@@
                p->sched_reset_on_fork = 0;
        }
  
 +      /*
 +       * Make sure we do not leak PI boosting priority to the child.
 +       */
 +      p->prio = current->normal_prio;
 +
        if (!rt_prio(p->prio))
                p->sched_class = &fair_sched_class;
  
@@@ -2586,6 -2577,8 +2582,6 @@@ void wake_up_new_task(struct task_struc
        BUG_ON(p->state != TASK_RUNNING);
        update_rq_clock(rq);
  
 -      p->prio = effective_prio(p);
 -
        if (!p->sched_class->task_new || !current->se.on_rq) {
                activate_task(rq, p, 0);
        } else {
@@@ -3661,7 -3654,6 +3657,7 @@@ static void update_group_power(struct s
  
  /**
   * update_sg_lb_stats - Update sched_group's statistics for load balancing.
 + * @sd: The sched_domain whose statistics are to be updated.
   * @group: sched_group whose statistics are to be updated.
   * @this_cpu: Cpu for which load balance is currently performed.
   * @idle: Idle status of this_cpu
@@@ -6724,6 -6716,9 +6720,6 @@@ EXPORT_SYMBOL(yield)
  /*
   * This task is about to go to sleep on IO. Increment rq->nr_iowait so
   * that process accounting knows that this is a task in IO wait state.
 - *
 - * But don't do that if it is a deliberate, throttling IO wait (this task
 - * has set its backing_dev_info: the queue against which it should throttle)
   */
  void __sched io_schedule(void)
  {
@@@ -9407,6 -9402,10 +9403,10 @@@ void __init sched_init(void
  #endif /* CONFIG_USER_SCHED */
  #endif /* CONFIG_GROUP_SCHED */
  
+ #if defined CONFIG_FAIR_GROUP_SCHED && defined CONFIG_SMP
+       update_shares_data = __alloc_percpu(nr_cpu_ids * sizeof(unsigned long),
+                                           __alignof__(unsigned long));
+ #endif
        for_each_possible_cpu(i) {
                struct rq *rq;