Merge branch 'upstream' of git://git.linux-mips.org/pub/scm/ralf/upstream-linus

[karo-tx-linux.git] / kernel / sched / rt.c

diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index a2740b775b456ed27c56ad088007c83e1873458b..bd2267ad404fa78de092b8bc8f228cfea2dca87b 100644 (file)
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -229,6 +229,14 @@ int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
 
 #ifdef CONFIG_SMP
 
+static int pull_rt_task(struct rq *this_rq);
+
+static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev)
+{
+       /* Try to pull RT tasks here if we lower this rq's prio */
+       return rq->rt.highest_prio.curr > prev->prio;
+}
+
 static inline int rt_overloaded(struct rq *rq)
 {
        return atomic_read(&rq->rd->rto_count);
@@ -315,6 +323,15 @@ static inline int has_pushable_tasks(struct rq *rq)
        return !plist_head_empty(&rq->rt.pushable_tasks);
 }
 
+static inline void set_post_schedule(struct rq *rq)
+{
+       /*
+        * We detect this state here so that we can avoid taking the RQ
+        * lock again later if there is no need to push
+        */
+       rq->post_schedule = has_pushable_tasks(rq);
+}
+
 static void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
 {
        plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
@@ -359,6 +376,19 @@ void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
 {
 }
 
+static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev)
+{
+       return false;
+}
+
+static inline int pull_rt_task(struct rq *this_rq)
+{
+       return 0;
+}
+
+static inline void set_post_schedule(struct rq *rq)
+{
+}
 #endif /* CONFIG_SMP */
 
 static inline int on_rt_rq(struct sched_rt_entity *rt_se)
@@ -440,11 +470,6 @@ static void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
                dequeue_rt_entity(rt_se);
 }
 
-static inline int rt_rq_throttled(struct rt_rq *rt_rq)
-{
-       return rt_rq->rt_throttled && !rt_rq->rt_nr_boosted;
-}
-
 static int rt_se_boosted(struct sched_rt_entity *rt_se)
 {
        struct rt_rq *rt_rq = group_rt_rq(rt_se);
@@ -515,11 +540,6 @@ static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
 {
 }
 
-static inline int rt_rq_throttled(struct rt_rq *rt_rq)
-{
-       return rt_rq->rt_throttled;
-}
-
 static inline const struct cpumask *sched_rt_period_mask(void)
 {
        return cpu_online_mask;
@@ -538,6 +558,14 @@ static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)
 
 #endif /* CONFIG_RT_GROUP_SCHED */
 
+bool sched_rt_bandwidth_account(struct rt_rq *rt_rq)
+{
+       struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
+
+       return (hrtimer_active(&rt_b->rt_period_timer) ||
+               rt_rq->rt_time < rt_b->rt_runtime);
+}
+
 #ifdef CONFIG_SMP
 /*
  * We ran out of runtime, see if we can borrow some from our neighbours.
@@ -1310,15 +1338,7 @@ static struct task_struct *_pick_next_task_rt(struct rq *rq)
 {
        struct sched_rt_entity *rt_se;
        struct task_struct *p;
-       struct rt_rq *rt_rq;
-
-       rt_rq = &rq->rt;
-
-       if (!rt_rq->rt_nr_running)
-               return NULL;
-
-       if (rt_rq_throttled(rt_rq))
-               return NULL;
+       struct rt_rq *rt_rq  = &rq->rt;
 
        do {
                rt_se = pick_next_rt_entity(rq, rt_rq);
@@ -1332,21 +1352,46 @@ static struct task_struct *_pick_next_task_rt(struct rq *rq)
        return p;
 }
 
-static struct task_struct *pick_next_task_rt(struct rq *rq)
+static struct task_struct *
+pick_next_task_rt(struct rq *rq, struct task_struct *prev)
 {
-       struct task_struct *p = _pick_next_task_rt(rq);
+       struct task_struct *p;
+       struct rt_rq *rt_rq = &rq->rt;
+
+       if (need_pull_rt_task(rq, prev)) {
+               pull_rt_task(rq);
+               /*
+                * pull_rt_task() can drop (and re-acquire) rq->lock; this
+                * means a dl or stop task can slip in, in which case we need
+                * to re-start task selection.
+                */
+               if (unlikely((rq->stop && rq->stop->on_rq) ||
+                            rq->dl.dl_nr_running))
+                       return RETRY_TASK;
+       }
+
+       /*
+        * We may dequeue prev's rt_rq in put_prev_task().
+        * So, we update time before rt_nr_running check.
+        */
+       if (prev->sched_class == &rt_sched_class)
+               update_curr_rt(rq);
+
+       if (!rt_rq->rt_nr_running)
+               return NULL;
+
+       if (rt_rq_throttled(rt_rq))
+               return NULL;
+
+       put_prev_task(rq, prev);
+
+       p = _pick_next_task_rt(rq);
 
        /* The running task is never eligible for pushing */
        if (p)
                dequeue_pushable_task(rq, p);
 
-#ifdef CONFIG_SMP
-       /*
-        * We detect this state here so that we can avoid taking the RQ
-        * lock again later if there is no need to push
-        */
-       rq->post_schedule = has_pushable_tasks(rq);
-#endif
+       set_post_schedule(rq);
 
        return p;
 }
@@ -1716,13 +1761,6 @@ skip:
        return ret;
 }
 
-static void pre_schedule_rt(struct rq *rq, struct task_struct *prev)
-{
-       /* Try to pull RT tasks here if we lower this rq's prio */
-       if (rq->rt.highest_prio.curr > prev->prio)
-               pull_rt_task(rq);
-}
-
 static void post_schedule_rt(struct rq *rq)
 {
        push_rt_tasks(rq);
@@ -1825,7 +1863,7 @@ static void switched_from_rt(struct rq *rq, struct task_struct *p)
                resched_task(rq->curr);
 }
 
-void init_sched_rt_class(void)
+void __init init_sched_rt_class(void)
 {
        unsigned int i;
 
@@ -1999,7 +2037,6 @@ const struct sched_class rt_sched_class = {
        .set_cpus_allowed       = set_cpus_allowed_rt,
        .rq_online              = rq_online_rt,
        .rq_offline             = rq_offline_rt,
-       .pre_schedule           = pre_schedule_rt,
        .post_schedule          = post_schedule_rt,
        .task_woken             = task_woken_rt,
        .switched_from          = switched_from_rt,