2 * Copyright 2006 Andi Kleen, SUSE Labs.
3 * Subject to the GNU Public License, v.2
5 * Fast user context implementation of clock_gettime, gettimeofday, and time.
7 * 32 Bit compat layer by Stefani Seibold <stefani@seibold.net>
8 * sponsored by Rohde & Schwarz GmbH & Co. KG Munich/Germany
10 * The code should have no internal unresolved relocations.
11 * Check with readelf after changing.
14 /* Disable profiling for userspace code: */
15 #define DISABLE_BRANCH_PROFILING
17 #include <uapi/linux/time.h>
18 #include <asm/vgtod.h>
21 #include <asm/unistd.h>
23 #include <linux/math64.h>
24 #include <linux/time.h>
26 #define gtod (&VVAR(vsyscall_gtod_data))
28 extern int __vdso_clock_gettime(clockid_t clock, struct timespec *ts);
29 extern int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz);
30 extern time_t __vdso_time(time_t *t);
32 #ifdef CONFIG_HPET_TIMER
34 __attribute__((visibility("hidden")));
36 static notrace cycle_t vread_hpet(void)
38 return *(const volatile u32 *)(&hpet_page + HPET_COUNTER);
44 #include <linux/kernel.h>
45 #include <asm/vsyscall.h>
46 #include <asm/fixmap.h>
47 #include <asm/pvclock.h>
49 notrace static long vdso_fallback_gettime(long clock, struct timespec *ts)
52 asm("syscall" : "=a" (ret) :
53 "0" (__NR_clock_gettime), "D" (clock), "S" (ts) : "memory");
57 notrace static long vdso_fallback_gtod(struct timeval *tv, struct timezone *tz)
61 asm("syscall" : "=a" (ret) :
62 "0" (__NR_gettimeofday), "D" (tv), "S" (tz) : "memory");
66 #ifdef CONFIG_PARAVIRT_CLOCK
68 static notrace const struct pvclock_vsyscall_time_info *get_pvti(int cpu)
70 const struct pvclock_vsyscall_time_info *pvti_base;
71 int idx = cpu / (PAGE_SIZE/PVTI_SIZE);
72 int offset = cpu % (PAGE_SIZE/PVTI_SIZE);
74 BUG_ON(PVCLOCK_FIXMAP_BEGIN + idx > PVCLOCK_FIXMAP_END);
76 pvti_base = (struct pvclock_vsyscall_time_info *)
77 __fix_to_virt(PVCLOCK_FIXMAP_BEGIN+idx);
79 return &pvti_base[offset];
82 static notrace cycle_t vread_pvclock(int *mode)
84 const struct pvclock_vsyscall_time_info *pvti;
93 * Note: hypervisor must guarantee that:
94 * 1. cpu ID number maps 1:1 to per-CPU pvclock time info.
95 * 2. that per-CPU pvclock time info is updated if the
96 * underlying CPU changes.
97 * 3. that version is increased whenever underlying CPU
102 cpu = __getcpu() & VGETCPU_CPU_MASK;
103 /* TODO: We can put vcpu id into higher bits of pvti.version.
104 * This will save a couple of cycles by getting rid of
105 * __getcpu() calls (Gleb).
108 pvti = get_pvti(cpu);
110 version = __pvclock_read_cycles(&pvti->pvti, &ret, &flags);
113 * Test we're still on the cpu as well as the version.
114 * We could have been migrated just after the first
115 * vgetcpu but before fetching the version, so we
116 * wouldn't notice a version change.
118 cpu1 = __getcpu() & VGETCPU_CPU_MASK;
119 } while (unlikely(cpu != cpu1 ||
120 (pvti->pvti.version & 1) ||
121 pvti->pvti.version != version));
123 if (unlikely(!(flags & PVCLOCK_TSC_STABLE_BIT)))
126 /* refer to tsc.c read_tsc() comment for rationale */
127 last = gtod->cycle_last;
129 if (likely(ret >= last))
138 notrace static long vdso_fallback_gettime(long clock, struct timespec *ts)
143 "mov %%ebx, %%edx \n"
145 "call __kernel_vsyscall \n"
146 "mov %%edx, %%ebx \n"
148 : "0" (__NR_clock_gettime), "g" (clock), "c" (ts)
153 notrace static long vdso_fallback_gtod(struct timeval *tv, struct timezone *tz)
158 "mov %%ebx, %%edx \n"
160 "call __kernel_vsyscall \n"
161 "mov %%edx, %%ebx \n"
163 : "0" (__NR_gettimeofday), "g" (tv), "c" (tz)
168 #ifdef CONFIG_PARAVIRT_CLOCK
170 static notrace cycle_t vread_pvclock(int *mode)
179 notrace static cycle_t vread_tsc(void)
185 * Empirically, a fence (of type that depends on the CPU)
186 * before rdtsc is enough to ensure that rdtsc is ordered
187 * with respect to loads. The various CPU manuals are unclear
188 * as to whether rdtsc can be reordered with later loads,
189 * but no one has ever seen it happen.
192 ret = (cycle_t)__native_read_tsc();
194 last = gtod->cycle_last;
196 if (likely(ret >= last))
200 * GCC likes to generate cmov here, but this branch is extremely
201 * predictable (it's just a funciton of time and the likely is
202 * very likely) and there's a data dependence, so force GCC
203 * to generate a branch instead. I don't barrier() because
204 * we don't actually need a barrier, and if this function
205 * ever gets inlined it will generate worse code.
211 notrace static inline u64 vgetsns(int *mode)
216 if (gtod->vclock_mode == VCLOCK_TSC)
217 cycles = vread_tsc();
218 #ifdef CONFIG_HPET_TIMER
219 else if (gtod->vclock_mode == VCLOCK_HPET)
220 cycles = vread_hpet();
222 #ifdef CONFIG_PARAVIRT_CLOCK
223 else if (gtod->vclock_mode == VCLOCK_PVCLOCK)
224 cycles = vread_pvclock(mode);
228 v = (cycles - gtod->cycle_last) & gtod->mask;
229 return v * gtod->mult;
232 /* Code size doesn't matter (vdso is 4k anyway) and this is faster. */
233 notrace static int __always_inline do_realtime(struct timespec *ts)
240 seq = gtod_read_begin(gtod);
241 mode = gtod->vclock_mode;
242 ts->tv_sec = gtod->wall_time_sec;
243 ns = gtod->wall_time_snsec;
244 ns += vgetsns(&mode);
246 } while (unlikely(gtod_read_retry(gtod, seq)));
248 ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
254 notrace static int __always_inline do_monotonic(struct timespec *ts)
261 seq = gtod_read_begin(gtod);
262 mode = gtod->vclock_mode;
263 ts->tv_sec = gtod->monotonic_time_sec;
264 ns = gtod->monotonic_time_snsec;
265 ns += vgetsns(&mode);
267 } while (unlikely(gtod_read_retry(gtod, seq)));
269 ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
275 notrace static void do_realtime_coarse(struct timespec *ts)
279 seq = gtod_read_begin(gtod);
280 ts->tv_sec = gtod->wall_time_coarse_sec;
281 ts->tv_nsec = gtod->wall_time_coarse_nsec;
282 } while (unlikely(gtod_read_retry(gtod, seq)));
285 notrace static void do_monotonic_coarse(struct timespec *ts)
289 seq = gtod_read_begin(gtod);
290 ts->tv_sec = gtod->monotonic_time_coarse_sec;
291 ts->tv_nsec = gtod->monotonic_time_coarse_nsec;
292 } while (unlikely(gtod_read_retry(gtod, seq)));
295 notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
299 if (do_realtime(ts) == VCLOCK_NONE)
302 case CLOCK_MONOTONIC:
303 if (do_monotonic(ts) == VCLOCK_NONE)
306 case CLOCK_REALTIME_COARSE:
307 do_realtime_coarse(ts);
309 case CLOCK_MONOTONIC_COARSE:
310 do_monotonic_coarse(ts);
318 return vdso_fallback_gettime(clock, ts);
320 int clock_gettime(clockid_t, struct timespec *)
321 __attribute__((weak, alias("__vdso_clock_gettime")));
323 notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
325 if (likely(tv != NULL)) {
326 if (unlikely(do_realtime((struct timespec *)tv) == VCLOCK_NONE))
327 return vdso_fallback_gtod(tv, tz);
330 if (unlikely(tz != NULL)) {
331 tz->tz_minuteswest = gtod->tz_minuteswest;
332 tz->tz_dsttime = gtod->tz_dsttime;
337 int gettimeofday(struct timeval *, struct timezone *)
338 __attribute__((weak, alias("__vdso_gettimeofday")));
341 * This will break when the xtime seconds get inaccurate, but that is
344 notrace time_t __vdso_time(time_t *t)
346 /* This is atomic on x86 so we don't need any locks. */
347 time_t result = ACCESS_ONCE(gtod->wall_time_sec);
354 __attribute__((weak, alias("__vdso_time")));