sched_clock: record TSC after gtod

[mv-sheeva.git] / kernel / sched_clock.c

diff --git a/kernel/sched_clock.c b/kernel/sched_clock.c
index 9c597e37f7de66b598db43fda9beebcb1b752ef0..28ff6bf5e02b9697657bed07941a98ddae88fa0e 100644 (file)
--- a/kernel/sched_clock.c
+++ b/kernel/sched_clock.c
@@ -40,11 +40,14 @@ struct sched_clock_data {
         */
        raw_spinlock_t          lock;
 
-       unsigned long           prev_jiffies;
+       unsigned long           tick_jiffies;
        u64                     prev_raw;
        u64                     tick_raw;
        u64                     tick_gtod;
        u64                     clock;
+#ifdef CONFIG_NO_HZ
+       int                     check_max;
+#endif
 };
 
 static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data);
@@ -59,23 +62,61 @@ static inline struct sched_clock_data *cpu_sdc(int cpu)
        return &per_cpu(sched_clock_data, cpu);
 }
 
+static __read_mostly int sched_clock_running;
+
 void sched_clock_init(void)
 {
        u64 ktime_now = ktime_to_ns(ktime_get());
-       u64 now = 0;
+       unsigned long now_jiffies = jiffies;
        int cpu;
 
        for_each_possible_cpu(cpu) {
                struct sched_clock_data *scd = cpu_sdc(cpu);
 
                scd->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
-               scd->prev_jiffies = jiffies;
-               scd->prev_raw = now;
-               scd->tick_raw = now;
+               scd->tick_jiffies = now_jiffies;
+               scd->prev_raw = 0;
+               scd->tick_raw = 0;
                scd->tick_gtod = ktime_now;
                scd->clock = ktime_now;
+#ifdef CONFIG_NO_HZ
+               scd->check_max = 1;
+#endif
        }
+
+       sched_clock_running = 1;
+}
+
+#ifdef CONFIG_NO_HZ
+/*
+ * The dynamic ticks makes the delta jiffies inaccurate. This
+ * prevents us from checking the maximum time update.
+ * Disable the maximum check during stopped ticks.
+ */
+void sched_clock_tick_stop(int cpu)
+{
+       struct sched_clock_data *scd = cpu_sdc(cpu);
+
+       scd->check_max = 0;
+}
+
+void sched_clock_tick_start(int cpu)
+{
+       struct sched_clock_data *scd = cpu_sdc(cpu);
+
+       scd->check_max = 1;
+}
+
+static int check_max(struct sched_clock_data *scd)
+{
+       return scd->check_max;
+}
+#else
+static int check_max(struct sched_clock_data *scd)
+{
+       return 1;
 }
+#endif /* CONFIG_NO_HZ */
 
 /*
  * update the percpu scd from the raw @now value
@@ -83,25 +124,32 @@ void sched_clock_init(void)
  *  - filter out backward motion
  *  - use jiffies to generate a min,max window to clip the raw values
  */
-static void __update_sched_clock(struct sched_clock_data *scd, u64 now)
+static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *time)
 {
        unsigned long now_jiffies = jiffies;
-       long delta_jiffies = now_jiffies - scd->prev_jiffies;
+       long delta_jiffies = now_jiffies - scd->tick_jiffies;
        u64 clock = scd->clock;
        u64 min_clock, max_clock;
        s64 delta = now - scd->prev_raw;
 
        WARN_ON_ONCE(!irqs_disabled());
-       min_clock = scd->tick_gtod + delta_jiffies * TICK_NSEC;
+
+       min_clock = scd->tick_gtod +
+               (delta_jiffies ? delta_jiffies - 1 : 0) * TICK_NSEC;
 
        if (unlikely(delta < 0)) {
                clock++;
                goto out;
        }
 
-       max_clock = min_clock + TICK_NSEC;
+       /*
+        * The clock must stay within a jiffie of the gtod.
+        * But since we may be at the start of a jiffy or the end of one
+        * we add another jiffy buffer.
+        */
+       max_clock = scd->tick_gtod + (2 + delta_jiffies) * TICK_NSEC;
 
-       if (unlikely(clock + delta > max_clock)) {
+       if (unlikely(clock + delta > max_clock) && check_max(scd)) {
                if (clock < max_clock)
                        clock = max_clock;
                else
@@ -114,9 +162,12 @@ static void __update_sched_clock(struct sched_clock_data *scd, u64 now)
        if (unlikely(clock < min_clock))
                clock = min_clock;
 
-       scd->prev_raw = now;
-       scd->prev_jiffies = now_jiffies;
-       scd->clock = clock;
+       if (time)
+               *time = clock;
+       else {
+               scd->prev_raw = now;
+               scd->clock = clock;
+       }
 }
 
 static void lock_double_clock(struct sched_clock_data *data1,
@@ -136,6 +187,9 @@ u64 sched_clock_cpu(int cpu)
        struct sched_clock_data *scd = cpu_sdc(cpu);
        u64 now, clock;
 
+       if (unlikely(!sched_clock_running))
+               return 0ull;
+
        WARN_ON_ONCE(!irqs_disabled());
        now = sched_clock();
 
@@ -153,39 +207,47 @@ u64 sched_clock_cpu(int cpu)
                now -= my_scd->tick_raw;
                now += scd->tick_raw;
 
-               now -= my_scd->tick_gtod;
-               now += scd->tick_gtod;
+               now += my_scd->tick_gtod;
+               now -= scd->tick_gtod;
 
                __raw_spin_unlock(&my_scd->lock);
+
+               __update_sched_clock(scd, now, &clock);
+
+               __raw_spin_unlock(&scd->lock);
+
        } else {
                __raw_spin_lock(&scd->lock);
+               __update_sched_clock(scd, now, NULL);
+               clock = scd->clock;
+               __raw_spin_unlock(&scd->lock);
        }
 
-       __update_sched_clock(scd, now);
-       clock = scd->clock;
-
-       __raw_spin_unlock(&scd->lock);
-
        return clock;
 }
 
 void sched_clock_tick(void)
 {
        struct sched_clock_data *scd = this_scd();
+       unsigned long now_jiffies = jiffies;
        u64 now, now_gtod;
 
+       if (unlikely(!sched_clock_running))
+               return;
+
        WARN_ON_ONCE(!irqs_disabled());
 
-       now = sched_clock();
        now_gtod = ktime_to_ns(ktime_get());
+       now = sched_clock();
 
        __raw_spin_lock(&scd->lock);
-       __update_sched_clock(scd, now);
+       __update_sched_clock(scd, now, NULL);
        /*
         * update tick_gtod after __update_sched_clock() because that will
         * already observe 1 new jiffy; adding a new tick_gtod to that would
         * increase the clock 2 jiffies.
         */
+       scd->tick_jiffies = now_jiffies;
        scd->tick_raw = now;
        scd->tick_gtod = now_gtod;
        __raw_spin_unlock(&scd->lock);