drivers/clocksource/arm_arch_timer.c

   1 /*
   2  *  linux/drivers/clocksource/arm_arch_timer.c
   3  *
   4  *  Copyright (C) 2011 ARM Ltd.
   5  *  All Rights Reserved
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11 #include <linux/init.h>
  12 #include <linux/kernel.h>
  13 #include <linux/device.h>
  14 #include <linux/smp.h>
  15 #include <linux/cpu.h>
  16 #include <linux/cpu_pm.h>
  17 #include <linux/clockchips.h>
  18 #include <linux/clocksource.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/of_irq.h>
  21 #include <linux/of_address.h>
  22 #include <linux/io.h>
  23 #include <linux/slab.h>
  24 #include <linux/sched_clock.h>
  25 #include <linux/acpi.h>
  26
  27 #include <asm/arch_timer.h>
  28 #include <asm/virt.h>
  29
  30 #include <clocksource/arm_arch_timer.h>
  31
  32 #define CNTTIDR         0x08
  33 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4))
  34
  35 #define CNTVCT_LO       0x08
  36 #define CNTVCT_HI       0x0c
  37 #define CNTFRQ          0x10
  38 #define CNTP_TVAL       0x28
  39 #define CNTP_CTL        0x2c
  40 #define CNTV_TVAL       0x38
  41 #define CNTV_CTL        0x3c
  42
  43 #define ARCH_CP15_TIMER BIT(0)
  44 #define ARCH_MEM_TIMER  BIT(1)
  45 static unsigned arch_timers_present __initdata;
  46
  47 static void __iomem *arch_counter_base;
  48
  49 struct arch_timer {
  50         void __iomem *base;
  51         struct clock_event_device evt;
  52 };
  53
  54 #define to_arch_timer(e) container_of(e, struct arch_timer, evt)
  55
  56 static u32 arch_timer_rate;
  57
  58 enum ppi_nr {
  59         PHYS_SECURE_PPI,
  60         PHYS_NONSECURE_PPI,
  61         VIRT_PPI,
  62         HYP_PPI,
  63         MAX_TIMER_PPI
  64 };
  65
  66 static int arch_timer_ppi[MAX_TIMER_PPI];
  67
  68 static struct clock_event_device __percpu *arch_timer_evt;
  69
  70 static bool arch_timer_use_virtual = true;
  71 static bool arch_timer_c3stop;
  72 static bool arch_timer_mem_use_virtual;
  73
  74 /*
  75  * Architected system timer support.
  76  */
  77
  78 static __always_inline
  79 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
  80                           struct clock_event_device *clk)
  81 {
  82         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
  83                 struct arch_timer *timer = to_arch_timer(clk);
  84                 switch (reg) {
  85                 case ARCH_TIMER_REG_CTRL:
  86                         writel_relaxed(val, timer->base + CNTP_CTL);
  87                         break;
  88                 case ARCH_TIMER_REG_TVAL:
  89                         writel_relaxed(val, timer->base + CNTP_TVAL);
  90                         break;
  91                 }
  92         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
  93                 struct arch_timer *timer = to_arch_timer(clk);
  94                 switch (reg) {
  95                 case ARCH_TIMER_REG_CTRL:
  96                         writel_relaxed(val, timer->base + CNTV_CTL);
  97                         break;
  98                 case ARCH_TIMER_REG_TVAL:
  99                         writel_relaxed(val, timer->base + CNTV_TVAL);
 100                         break;
 101                 }
 102         } else {
 103                 arch_timer_reg_write_cp15(access, reg, val);
 104         }
 105 }
 106
 107 static __always_inline
 108 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
 109                         struct clock_event_device *clk)
 110 {
 111         u32 val;
 112
 113         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 114                 struct arch_timer *timer = to_arch_timer(clk);
 115                 switch (reg) {
 116                 case ARCH_TIMER_REG_CTRL:
 117                         val = readl_relaxed(timer->base + CNTP_CTL);
 118                         break;
 119                 case ARCH_TIMER_REG_TVAL:
 120                         val = readl_relaxed(timer->base + CNTP_TVAL);
 121                         break;
 122                 }
 123         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 124                 struct arch_timer *timer = to_arch_timer(clk);
 125                 switch (reg) {
 126                 case ARCH_TIMER_REG_CTRL:
 127                         val = readl_relaxed(timer->base + CNTV_CTL);
 128                         break;
 129                 case ARCH_TIMER_REG_TVAL:
 130                         val = readl_relaxed(timer->base + CNTV_TVAL);
 131                         break;
 132                 }
 133         } else {
 134                 val = arch_timer_reg_read_cp15(access, reg);
 135         }
 136
 137         return val;
 138 }
 139
 140 static __always_inline irqreturn_t timer_handler(const int access,
 141                                         struct clock_event_device *evt)
 142 {
 143         unsigned long ctrl;
 144
 145         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
 146         if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
 147                 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
 148                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
 149                 evt->event_handler(evt);
 150                 return IRQ_HANDLED;
 151         }
 152
 153         return IRQ_NONE;
 154 }
 155
 156 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
 157 {
 158         struct clock_event_device *evt = dev_id;
 159
 160         return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
 161 }
 162
 163 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
 164 {
 165         struct clock_event_device *evt = dev_id;
 166
 167         return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
 168 }
 169
 170 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
 171 {
 172         struct clock_event_device *evt = dev_id;
 173
 174         return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
 175 }
 176
 177 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
 178 {
 179         struct clock_event_device *evt = dev_id;
 180
 181         return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
 182 }
 183
 184 static __always_inline void timer_set_mode(const int access, int mode,
 185                                   struct clock_event_device *clk)
 186 {
 187         unsigned long ctrl;
 188         switch (mode) {
 189         case CLOCK_EVT_MODE_UNUSED:
 190         case CLOCK_EVT_MODE_SHUTDOWN:
 191                 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 192                 ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
 193                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 194                 break;
 195         default:
 196                 break;
 197         }
 198 }
 199
 200 static void arch_timer_set_mode_virt(enum clock_event_mode mode,
 201                                      struct clock_event_device *clk)
 202 {
 203         timer_set_mode(ARCH_TIMER_VIRT_ACCESS, mode, clk);
 204 }
 205
 206 static void arch_timer_set_mode_phys(enum clock_event_mode mode,
 207                                      struct clock_event_device *clk)
 208 {
 209         timer_set_mode(ARCH_TIMER_PHYS_ACCESS, mode, clk);
 210 }
 211
 212 static void arch_timer_set_mode_virt_mem(enum clock_event_mode mode,
 213                                          struct clock_event_device *clk)
 214 {
 215         timer_set_mode(ARCH_TIMER_MEM_VIRT_ACCESS, mode, clk);
 216 }
 217
 218 static void arch_timer_set_mode_phys_mem(enum clock_event_mode mode,
 219                                          struct clock_event_device *clk)
 220 {
 221         timer_set_mode(ARCH_TIMER_MEM_PHYS_ACCESS, mode, clk);
 222 }
 223
 224 static __always_inline void set_next_event(const int access, unsigned long evt,
 225                                            struct clock_event_device *clk)
 226 {
 227         unsigned long ctrl;
 228         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 229         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 230         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 231         arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
 232         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 233 }
 234
 235 static int arch_timer_set_next_event_virt(unsigned long evt,
 236                                           struct clock_event_device *clk)
 237 {
 238         set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 239         return 0;
 240 }
 241
 242 static int arch_timer_set_next_event_phys(unsigned long evt,
 243                                           struct clock_event_device *clk)
 244 {
 245         set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 246         return 0;
 247 }
 248
 249 static int arch_timer_set_next_event_virt_mem(unsigned long evt,
 250                                               struct clock_event_device *clk)
 251 {
 252         set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
 253         return 0;
 254 }
 255
 256 static int arch_timer_set_next_event_phys_mem(unsigned long evt,
 257                                               struct clock_event_device *clk)
 258 {
 259         set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
 260         return 0;
 261 }
 262
 263 static void __arch_timer_setup(unsigned type,
 264                                struct clock_event_device *clk)
 265 {
 266         clk->features = CLOCK_EVT_FEAT_ONESHOT;
 267
 268         if (type == ARCH_CP15_TIMER) {
 269                 if (arch_timer_c3stop)
 270                         clk->features |= CLOCK_EVT_FEAT_C3STOP;
 271                 clk->name = "arch_sys_timer";
 272                 clk->rating = 450;
 273                 clk->cpumask = cpumask_of(smp_processor_id());
 274                 if (arch_timer_use_virtual) {
 275                         clk->irq = arch_timer_ppi[VIRT_PPI];
 276                         clk->set_mode = arch_timer_set_mode_virt;
 277                         clk->set_next_event = arch_timer_set_next_event_virt;
 278                 } else {
 279                         clk->irq = arch_timer_ppi[PHYS_SECURE_PPI];
 280                         clk->set_mode = arch_timer_set_mode_phys;
 281                         clk->set_next_event = arch_timer_set_next_event_phys;
 282                 }
 283         } else {
 284                 clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
 285                 clk->name = "arch_mem_timer";
 286                 clk->rating = 400;
 287                 clk->cpumask = cpu_all_mask;
 288                 if (arch_timer_mem_use_virtual) {
 289                         clk->set_mode = arch_timer_set_mode_virt_mem;
 290                         clk->set_next_event =
 291                                 arch_timer_set_next_event_virt_mem;
 292                 } else {
 293                         clk->set_mode = arch_timer_set_mode_phys_mem;
 294                         clk->set_next_event =
 295                                 arch_timer_set_next_event_phys_mem;
 296                 }
 297         }
 298
 299         clk->set_mode(CLOCK_EVT_MODE_SHUTDOWN, clk);
 300
 301         clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
 302 }
 303
 304 static void arch_timer_evtstrm_enable(int divider)
 305 {
 306         u32 cntkctl = arch_timer_get_cntkctl();
 307
 308         cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
 309         /* Set the divider and enable virtual event stream */
 310         cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
 311                         | ARCH_TIMER_VIRT_EVT_EN;
 312         arch_timer_set_cntkctl(cntkctl);
 313         elf_hwcap |= HWCAP_EVTSTRM;
 314 #ifdef CONFIG_COMPAT
 315         compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;
 316 #endif
 317 }
 318
 319 static void arch_timer_configure_evtstream(void)
 320 {
 321         int evt_stream_div, pos;
 322
 323         /* Find the closest power of two to the divisor */
 324         evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ;
 325         pos = fls(evt_stream_div);
 326         if (pos > 1 && !(evt_stream_div & (1 << (pos - 2))))
 327                 pos--;
 328         /* enable event stream */
 329         arch_timer_evtstrm_enable(min(pos, 15));
 330 }
 331
 332 static void arch_counter_set_user_access(void)
 333 {
 334         u32 cntkctl = arch_timer_get_cntkctl();
 335
 336         /* Disable user access to the timers and the physical counter */
 337         /* Also disable virtual event stream */
 338         cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
 339                         | ARCH_TIMER_USR_VT_ACCESS_EN
 340                         | ARCH_TIMER_VIRT_EVT_EN
 341                         | ARCH_TIMER_USR_PCT_ACCESS_EN);
 342
 343         /* Enable user access to the virtual counter */
 344         cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
 345
 346         arch_timer_set_cntkctl(cntkctl);
 347 }
 348
 349 static int arch_timer_setup(struct clock_event_device *clk)
 350 {
 351         __arch_timer_setup(ARCH_CP15_TIMER, clk);
 352
 353         if (arch_timer_use_virtual)
 354                 enable_percpu_irq(arch_timer_ppi[VIRT_PPI], 0);
 355         else {
 356                 enable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI], 0);
 357                 if (arch_timer_ppi[PHYS_NONSECURE_PPI])
 358                         enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], 0);
 359         }
 360
 361         arch_counter_set_user_access();
 362         if (IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM))
 363                 arch_timer_configure_evtstream();
 364
 365         return 0;
 366 }
 367
 368 static void
 369 arch_timer_detect_rate(void __iomem *cntbase, struct device_node *np)
 370 {
 371         /* Who has more than one independent system counter? */
 372         if (arch_timer_rate)
 373                 return;
 374
 375         /*
 376          * Try to determine the frequency from the device tree or CNTFRQ,
 377          * if ACPI is enabled, get the frequency from CNTFRQ ONLY.
 378          */
 379         if (!acpi_disabled ||
 380             of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) {
 381                 if (cntbase)
 382                         arch_timer_rate = readl_relaxed(cntbase + CNTFRQ);
 383                 else
 384                         arch_timer_rate = arch_timer_get_cntfrq();
 385         }
 386
 387         /* Check the timer frequency. */
 388         if (arch_timer_rate == 0)
 389                 pr_warn("Architected timer frequency not available\n");
 390 }
 391
 392 static void arch_timer_banner(unsigned type)
 393 {
 394         pr_info("Architected %s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
 395                      type & ARCH_CP15_TIMER ? "cp15" : "",
 396                      type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  " and " : "",
 397                      type & ARCH_MEM_TIMER ? "mmio" : "",
 398                      (unsigned long)arch_timer_rate / 1000000,
 399                      (unsigned long)(arch_timer_rate / 10000) % 100,
 400                      type & ARCH_CP15_TIMER ?
 401                         arch_timer_use_virtual ? "virt" : "phys" :
 402                         "",
 403                      type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  "/" : "",
 404                      type & ARCH_MEM_TIMER ?
 405                         arch_timer_mem_use_virtual ? "virt" : "phys" :
 406                         "");
 407 }
 408
 409 u32 arch_timer_get_rate(void)
 410 {
 411         return arch_timer_rate;
 412 }
 413
 414 static u64 arch_counter_get_cntvct_mem(void)
 415 {
 416         u32 vct_lo, vct_hi, tmp_hi;
 417
 418         do {
 419                 vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
 420                 vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
 421                 tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
 422         } while (vct_hi != tmp_hi);
 423
 424         return ((u64) vct_hi << 32) | vct_lo;
 425 }
 426
 427 /*
 428  * Default to cp15 based access because arm64 uses this function for
 429  * sched_clock() before DT is probed and the cp15 method is guaranteed
 430  * to exist on arm64. arm doesn't use this before DT is probed so even
 431  * if we don't have the cp15 accessors we won't have a problem.
 432  */
 433 u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;
 434
 435 static cycle_t arch_counter_read(struct clocksource *cs)
 436 {
 437         return arch_timer_read_counter();
 438 }
 439
 440 static cycle_t arch_counter_read_cc(const struct cyclecounter *cc)
 441 {
 442         return arch_timer_read_counter();
 443 }
 444
 445 static struct clocksource clocksource_counter = {
 446         .name   = "arch_sys_counter",
 447         .rating = 400,
 448         .read   = arch_counter_read,
 449         .mask   = CLOCKSOURCE_MASK(56),
 450         .flags  = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_SUSPEND_NONSTOP,
 451 };
 452
 453 static struct cyclecounter cyclecounter = {
 454         .read   = arch_counter_read_cc,
 455         .mask   = CLOCKSOURCE_MASK(56),
 456 };
 457
 458 static struct timecounter timecounter;
 459
 460 struct timecounter *arch_timer_get_timecounter(void)
 461 {
 462         return &timecounter;
 463 }
 464
 465 static void __init arch_counter_register(unsigned type)
 466 {
 467         u64 start_count;
 468
 469         /* Register the CP15 based counter if we have one */
 470         if (type & ARCH_CP15_TIMER) {
 471                 if (IS_ENABLED(CONFIG_ARM64) || arch_timer_use_virtual)
 472                         arch_timer_read_counter = arch_counter_get_cntvct;
 473                 else
 474                         arch_timer_read_counter = arch_counter_get_cntpct;
 475         } else {
 476                 arch_timer_read_counter = arch_counter_get_cntvct_mem;
 477
 478                 /* If the clocksource name is "arch_sys_counter" the
 479                  * VDSO will attempt to read the CP15-based counter.
 480                  * Ensure this does not happen when CP15-based
 481                  * counter is not available.
 482                  */
 483                 clocksource_counter.name = "arch_mem_counter";
 484         }
 485
 486         start_count = arch_timer_read_counter();
 487         clocksource_register_hz(&clocksource_counter, arch_timer_rate);
 488         cyclecounter.mult = clocksource_counter.mult;
 489         cyclecounter.shift = clocksource_counter.shift;
 490         timecounter_init(&timecounter, &cyclecounter, start_count);
 491
 492         /* 56 bits minimum, so we assume worst case rollover */
 493         sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);
 494 }
 495
 496 static void arch_timer_stop(struct clock_event_device *clk)
 497 {
 498         pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n",
 499                  clk->irq, smp_processor_id());
 500
 501         if (arch_timer_use_virtual)
 502                 disable_percpu_irq(arch_timer_ppi[VIRT_PPI]);
 503         else {
 504                 disable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI]);
 505                 if (arch_timer_ppi[PHYS_NONSECURE_PPI])
 506                         disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]);
 507         }
 508
 509         clk->set_mode(CLOCK_EVT_MODE_UNUSED, clk);
 510 }
 511
 512 static int arch_timer_cpu_notify(struct notifier_block *self,
 513                                            unsigned long action, void *hcpu)
 514 {
 515         /*
 516          * Grab cpu pointer in each case to avoid spurious
 517          * preemptible warnings
 518          */
 519         switch (action & ~CPU_TASKS_FROZEN) {
 520         case CPU_STARTING:
 521                 arch_timer_setup(this_cpu_ptr(arch_timer_evt));
 522                 break;
 523         case CPU_DYING:
 524                 arch_timer_stop(this_cpu_ptr(arch_timer_evt));
 525                 break;
 526         }
 527
 528         return NOTIFY_OK;
 529 }
 530
 531 static struct notifier_block arch_timer_cpu_nb = {
 532         .notifier_call = arch_timer_cpu_notify,
 533 };
 534
 535 #ifdef CONFIG_CPU_PM
 536 static unsigned int saved_cntkctl;
 537 static int arch_timer_cpu_pm_notify(struct notifier_block *self,
 538                                     unsigned long action, void *hcpu)
 539 {
 540         if (action == CPU_PM_ENTER)
 541                 saved_cntkctl = arch_timer_get_cntkctl();
 542         else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT)
 543                 arch_timer_set_cntkctl(saved_cntkctl);
 544         return NOTIFY_OK;
 545 }
 546
 547 static struct notifier_block arch_timer_cpu_pm_notifier = {
 548         .notifier_call = arch_timer_cpu_pm_notify,
 549 };
 550
 551 static int __init arch_timer_cpu_pm_init(void)
 552 {
 553         return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
 554 }
 555 #else
 556 static int __init arch_timer_cpu_pm_init(void)
 557 {
 558         return 0;
 559 }
 560 #endif
 561
 562 static int __init arch_timer_register(void)
 563 {
 564         int err;
 565         int ppi;
 566
 567         arch_timer_evt = alloc_percpu(struct clock_event_device);
 568         if (!arch_timer_evt) {
 569                 err = -ENOMEM;
 570                 goto out;
 571         }
 572
 573         if (arch_timer_use_virtual) {
 574                 ppi = arch_timer_ppi[VIRT_PPI];
 575                 err = request_percpu_irq(ppi, arch_timer_handler_virt,
 576                                          "arch_timer", arch_timer_evt);
 577         } else {
 578                 ppi = arch_timer_ppi[PHYS_SECURE_PPI];
 579                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
 580                                          "arch_timer", arch_timer_evt);
 581                 if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) {
 582                         ppi = arch_timer_ppi[PHYS_NONSECURE_PPI];
 583                         err = request_percpu_irq(ppi, arch_timer_handler_phys,
 584                                                  "arch_timer", arch_timer_evt);
 585                         if (err)
 586                                 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
 587                                                 arch_timer_evt);
 588                 }
 589         }
 590
 591         if (err) {
 592                 pr_err("arch_timer: can't register interrupt %d (%d)\n",
 593                        ppi, err);
 594                 goto out_free;
 595         }
 596
 597         err = register_cpu_notifier(&arch_timer_cpu_nb);
 598         if (err)
 599                 goto out_free_irq;
 600
 601         err = arch_timer_cpu_pm_init();
 602         if (err)
 603                 goto out_unreg_notify;
 604
 605         /* Immediately configure the timer on the boot CPU */
 606         arch_timer_setup(this_cpu_ptr(arch_timer_evt));
 607
 608         return 0;
 609
 610 out_unreg_notify:
 611         unregister_cpu_notifier(&arch_timer_cpu_nb);
 612 out_free_irq:
 613         if (arch_timer_use_virtual)
 614                 free_percpu_irq(arch_timer_ppi[VIRT_PPI], arch_timer_evt);
 615         else {
 616                 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
 617                                 arch_timer_evt);
 618                 if (arch_timer_ppi[PHYS_NONSECURE_PPI])
 619                         free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI],
 620                                         arch_timer_evt);
 621         }
 622
 623 out_free:
 624         free_percpu(arch_timer_evt);
 625 out:
 626         return err;
 627 }
 628
 629 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
 630 {
 631         int ret;
 632         irq_handler_t func;
 633         struct arch_timer *t;
 634
 635         t = kzalloc(sizeof(*t), GFP_KERNEL);
 636         if (!t)
 637                 return -ENOMEM;
 638
 639         t->base = base;
 640         t->evt.irq = irq;
 641         __arch_timer_setup(ARCH_MEM_TIMER, &t->evt);
 642
 643         if (arch_timer_mem_use_virtual)
 644                 func = arch_timer_handler_virt_mem;
 645         else
 646                 func = arch_timer_handler_phys_mem;
 647
 648         ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
 649         if (ret) {
 650                 pr_err("arch_timer: Failed to request mem timer irq\n");
 651                 kfree(t);
 652         }
 653
 654         return ret;
 655 }
 656
 657 static const struct of_device_id arch_timer_of_match[] __initconst = {
 658         { .compatible   = "arm,armv7-timer",    },
 659         { .compatible   = "arm,armv8-timer",    },
 660         {},
 661 };
 662
 663 static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
 664         { .compatible   = "arm,armv7-timer-mem", },
 665         {},
 666 };
 667
 668 static bool __init
 669 arch_timer_probed(int type, const struct of_device_id *matches)
 670 {
 671         struct device_node *dn;
 672         bool probed = true;
 673
 674         dn = of_find_matching_node(NULL, matches);
 675         if (dn && of_device_is_available(dn) && !(arch_timers_present & type))
 676                 probed = false;
 677         of_node_put(dn);
 678
 679         return probed;
 680 }
 681
 682 static void __init arch_timer_common_init(void)
 683 {
 684         unsigned mask = ARCH_CP15_TIMER | ARCH_MEM_TIMER;
 685
 686         /* Wait until both nodes are probed if we have two timers */
 687         if ((arch_timers_present & mask) != mask) {
 688                 if (!arch_timer_probed(ARCH_MEM_TIMER, arch_timer_mem_of_match))
 689                         return;
 690                 if (!arch_timer_probed(ARCH_CP15_TIMER, arch_timer_of_match))
 691                         return;
 692         }
 693
 694         arch_timer_banner(arch_timers_present);
 695         arch_counter_register(arch_timers_present);
 696         arch_timer_arch_init();
 697 }
 698
 699 static void __init arch_timer_init(void)
 700 {
 701         /*
 702          * If HYP mode is available, we know that the physical timer
 703          * has been configured to be accessible from PL1. Use it, so
 704          * that a guest can use the virtual timer instead.
 705          *
 706          * If no interrupt provided for virtual timer, we'll have to
 707          * stick to the physical timer. It'd better be accessible...
 708          */
 709         if (is_hyp_mode_available() || !arch_timer_ppi[VIRT_PPI]) {
 710                 arch_timer_use_virtual = false;
 711
 712                 if (!arch_timer_ppi[PHYS_SECURE_PPI] ||
 713                     !arch_timer_ppi[PHYS_NONSECURE_PPI]) {
 714                         pr_warn("arch_timer: No interrupt available, giving up\n");
 715                         return;
 716                 }
 717         }
 718
 719         arch_timer_register();
 720         arch_timer_common_init();
 721 }
 722
 723 static void __init arch_timer_of_init(struct device_node *np)
 724 {
 725         int i;
 726
 727         if (arch_timers_present & ARCH_CP15_TIMER) {
 728                 pr_warn("arch_timer: multiple nodes in dt, skipping\n");
 729                 return;
 730         }
 731
 732         arch_timers_present |= ARCH_CP15_TIMER;
 733         for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++)
 734                 arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
 735
 736         arch_timer_detect_rate(NULL, np);
 737
 738         arch_timer_c3stop = !of_property_read_bool(np, "always-on");
 739
 740         /*
 741          * If we cannot rely on firmware initializing the timer registers then
 742          * we should use the physical timers instead.
 743          */
 744         if (IS_ENABLED(CONFIG_ARM) &&
 745             of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
 746                         arch_timer_use_virtual = false;
 747
 748         arch_timer_init();
 749 }
 750 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
 751 CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
 752
 753 static void __init arch_timer_mem_init(struct device_node *np)
 754 {
 755         struct device_node *frame, *best_frame = NULL;
 756         void __iomem *cntctlbase, *base;
 757         unsigned int irq;
 758         u32 cnttidr;
 759
 760         arch_timers_present |= ARCH_MEM_TIMER;
 761         cntctlbase = of_iomap(np, 0);
 762         if (!cntctlbase) {
 763                 pr_err("arch_timer: Can't find CNTCTLBase\n");
 764                 return;
 765         }
 766
 767         cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
 768         iounmap(cntctlbase);
 769
 770         /*
 771          * Try to find a virtual capable frame. Otherwise fall back to a
 772          * physical capable frame.
 773          */
 774         for_each_available_child_of_node(np, frame) {
 775                 int n;
 776
 777                 if (of_property_read_u32(frame, "frame-number", &n)) {
 778                         pr_err("arch_timer: Missing frame-number\n");
 779                         of_node_put(best_frame);
 780                         of_node_put(frame);
 781                         return;
 782                 }
 783
 784                 if (cnttidr & CNTTIDR_VIRT(n)) {
 785                         of_node_put(best_frame);
 786                         best_frame = frame;
 787                         arch_timer_mem_use_virtual = true;
 788                         break;
 789                 }
 790                 of_node_put(best_frame);
 791                 best_frame = of_node_get(frame);
 792         }
 793
 794         base = arch_counter_base = of_iomap(best_frame, 0);
 795         if (!base) {
 796                 pr_err("arch_timer: Can't map frame's registers\n");
 797                 of_node_put(best_frame);
 798                 return;
 799         }
 800
 801         if (arch_timer_mem_use_virtual)
 802                 irq = irq_of_parse_and_map(best_frame, 1);
 803         else
 804                 irq = irq_of_parse_and_map(best_frame, 0);
 805         of_node_put(best_frame);
 806         if (!irq) {
 807                 pr_err("arch_timer: Frame missing %s irq",
 808                        arch_timer_mem_use_virtual ? "virt" : "phys");
 809                 return;
 810         }
 811
 812         arch_timer_detect_rate(base, np);
 813         arch_timer_mem_register(base, irq);
 814         arch_timer_common_init();
 815 }
 816 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
 817                        arch_timer_mem_init);
 818
 819 #ifdef CONFIG_ACPI
 820 static int __init map_generic_timer_interrupt(u32 interrupt, u32 flags)
 821 {
 822         int trigger, polarity;
 823
 824         if (!interrupt)
 825                 return 0;
 826
 827         trigger = (flags & ACPI_GTDT_INTERRUPT_MODE) ? ACPI_EDGE_SENSITIVE
 828                         : ACPI_LEVEL_SENSITIVE;
 829
 830         polarity = (flags & ACPI_GTDT_INTERRUPT_POLARITY) ? ACPI_ACTIVE_LOW
 831                         : ACPI_ACTIVE_HIGH;
 832
 833         return acpi_register_gsi(NULL, interrupt, trigger, polarity);
 834 }
 835
 836 /* Initialize per-processor generic timer */
 837 static int __init arch_timer_acpi_init(struct acpi_table_header *table)
 838 {
 839         struct acpi_table_gtdt *gtdt;
 840
 841         if (arch_timers_present & ARCH_CP15_TIMER) {
 842                 pr_warn("arch_timer: already initialized, skipping\n");
 843                 return -EINVAL;
 844         }
 845
 846         gtdt = container_of(table, struct acpi_table_gtdt, header);
 847
 848         arch_timers_present |= ARCH_CP15_TIMER;
 849
 850         arch_timer_ppi[PHYS_SECURE_PPI] =
 851                 map_generic_timer_interrupt(gtdt->secure_el1_interrupt,
 852                 gtdt->secure_el1_flags);
 853
 854         arch_timer_ppi[PHYS_NONSECURE_PPI] =
 855                 map_generic_timer_interrupt(gtdt->non_secure_el1_interrupt,
 856                 gtdt->non_secure_el1_flags);
 857
 858         arch_timer_ppi[VIRT_PPI] =
 859                 map_generic_timer_interrupt(gtdt->virtual_timer_interrupt,
 860                 gtdt->virtual_timer_flags);
 861
 862         arch_timer_ppi[HYP_PPI] =
 863                 map_generic_timer_interrupt(gtdt->non_secure_el2_interrupt,
 864                 gtdt->non_secure_el2_flags);
 865
 866         /* Get the frequency from CNTFRQ */
 867         arch_timer_detect_rate(NULL, NULL);
 868
 869         /* Always-on capability */
 870         arch_timer_c3stop = !(gtdt->non_secure_el1_flags & ACPI_GTDT_ALWAYS_ON);
 871
 872         arch_timer_init();
 873         return 0;
 874 }
 875
 876 /* Initialize all the generic timers presented in GTDT */
 877 void __init acpi_generic_timer_init(void)
 878 {
 879         if (acpi_disabled)
 880                 return;
 881
 882         acpi_table_parse(ACPI_SIG_GTDT, arch_timer_acpi_init);
 883 }
 884 #endif