drivers/irqchip/irq-mips-gic.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 2008 Ralf Baechle (ralf@linux-mips.org)
   7  * Copyright (C) 2012 MIPS Technologies, Inc.  All rights reserved.
   8  */
   9 #include <linux/bitmap.h>
  10 #include <linux/clocksource.h>
  11 #include <linux/init.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/irq.h>
  14 #include <linux/irqchip.h>
  15 #include <linux/irqchip/mips-gic.h>
  16 #include <linux/of_address.h>
  17 #include <linux/sched.h>
  18 #include <linux/smp.h>
  19
  20 #include <asm/mips-cm.h>
  21 #include <asm/setup.h>
  22 #include <asm/traps.h>
  23
  24 #include <dt-bindings/interrupt-controller/mips-gic.h>
  25
  26 unsigned int gic_present;
  27
  28 struct gic_pcpu_mask {
  29         DECLARE_BITMAP(pcpu_mask, GIC_MAX_INTRS);
  30 };
  31
  32 struct gic_irq_spec {
  33         enum {
  34                 GIC_DEVICE,
  35                 GIC_IPI
  36         } type;
  37
  38         union {
  39                 struct cpumask *ipimask;
  40                 unsigned int hwirq;
  41         };
  42 };
  43
  44 static unsigned long __gic_base_addr;
  45
  46 static void __iomem *gic_base;
  47 static struct gic_pcpu_mask pcpu_masks[NR_CPUS];
  48 static DEFINE_SPINLOCK(gic_lock);
  49 static struct irq_domain *gic_irq_domain;
  50 static struct irq_domain *gic_dev_domain;
  51 static struct irq_domain *gic_ipi_domain;
  52 static int gic_shared_intrs;
  53 static int gic_vpes;
  54 static unsigned int gic_cpu_pin;
  55 static unsigned int timer_cpu_pin;
  56 static struct irq_chip gic_level_irq_controller, gic_edge_irq_controller;
  57 DECLARE_BITMAP(ipi_resrv, GIC_MAX_INTRS);
  58
  59 static void __gic_irq_dispatch(void);
  60
  61 static inline u32 gic_read32(unsigned int reg)
  62 {
  63         return __raw_readl(gic_base + reg);
  64 }
  65
  66 static inline u64 gic_read64(unsigned int reg)
  67 {
  68         return __raw_readq(gic_base + reg);
  69 }
  70
  71 static inline unsigned long gic_read(unsigned int reg)
  72 {
  73         if (!mips_cm_is64)
  74                 return gic_read32(reg);
  75         else
  76                 return gic_read64(reg);
  77 }
  78
  79 static inline void gic_write32(unsigned int reg, u32 val)
  80 {
  81         return __raw_writel(val, gic_base + reg);
  82 }
  83
  84 static inline void gic_write64(unsigned int reg, u64 val)
  85 {
  86         return __raw_writeq(val, gic_base + reg);
  87 }
  88
  89 static inline void gic_write(unsigned int reg, unsigned long val)
  90 {
  91         if (!mips_cm_is64)
  92                 return gic_write32(reg, (u32)val);
  93         else
  94                 return gic_write64(reg, (u64)val);
  95 }
  96
  97 static inline void gic_update_bits(unsigned int reg, unsigned long mask,
  98                                    unsigned long val)
  99 {
 100         unsigned long regval;
 101
 102         regval = gic_read(reg);
 103         regval &= ~mask;
 104         regval |= val;
 105         gic_write(reg, regval);
 106 }
 107
 108 static inline void gic_reset_mask(unsigned int intr)
 109 {
 110         gic_write(GIC_REG(SHARED, GIC_SH_RMASK) + GIC_INTR_OFS(intr),
 111                   1ul << GIC_INTR_BIT(intr));
 112 }
 113
 114 static inline void gic_set_mask(unsigned int intr)
 115 {
 116         gic_write(GIC_REG(SHARED, GIC_SH_SMASK) + GIC_INTR_OFS(intr),
 117                   1ul << GIC_INTR_BIT(intr));
 118 }
 119
 120 static inline void gic_set_polarity(unsigned int intr, unsigned int pol)
 121 {
 122         gic_update_bits(GIC_REG(SHARED, GIC_SH_SET_POLARITY) +
 123                         GIC_INTR_OFS(intr), 1ul << GIC_INTR_BIT(intr),
 124                         (unsigned long)pol << GIC_INTR_BIT(intr));
 125 }
 126
 127 static inline void gic_set_trigger(unsigned int intr, unsigned int trig)
 128 {
 129         gic_update_bits(GIC_REG(SHARED, GIC_SH_SET_TRIGGER) +
 130                         GIC_INTR_OFS(intr), 1ul << GIC_INTR_BIT(intr),
 131                         (unsigned long)trig << GIC_INTR_BIT(intr));
 132 }
 133
 134 static inline void gic_set_dual_edge(unsigned int intr, unsigned int dual)
 135 {
 136         gic_update_bits(GIC_REG(SHARED, GIC_SH_SET_DUAL) + GIC_INTR_OFS(intr),
 137                         1ul << GIC_INTR_BIT(intr),
 138                         (unsigned long)dual << GIC_INTR_BIT(intr));
 139 }
 140
 141 static inline void gic_map_to_pin(unsigned int intr, unsigned int pin)
 142 {
 143         gic_write32(GIC_REG(SHARED, GIC_SH_INTR_MAP_TO_PIN_BASE) +
 144                     GIC_SH_MAP_TO_PIN(intr), GIC_MAP_TO_PIN_MSK | pin);
 145 }
 146
 147 static inline void gic_map_to_vpe(unsigned int intr, unsigned int vpe)
 148 {
 149         gic_write(GIC_REG(SHARED, GIC_SH_INTR_MAP_TO_VPE_BASE) +
 150                   GIC_SH_MAP_TO_VPE_REG_OFF(intr, vpe),
 151                   GIC_SH_MAP_TO_VPE_REG_BIT(vpe));
 152 }
 153
 154 #ifdef CONFIG_CLKSRC_MIPS_GIC
 155 cycle_t gic_read_count(void)
 156 {
 157         unsigned int hi, hi2, lo;
 158
 159         if (mips_cm_is64)
 160                 return (cycle_t)gic_read(GIC_REG(SHARED, GIC_SH_COUNTER));
 161
 162         do {
 163                 hi = gic_read32(GIC_REG(SHARED, GIC_SH_COUNTER_63_32));
 164                 lo = gic_read32(GIC_REG(SHARED, GIC_SH_COUNTER_31_00));
 165                 hi2 = gic_read32(GIC_REG(SHARED, GIC_SH_COUNTER_63_32));
 166         } while (hi2 != hi);
 167
 168         return (((cycle_t) hi) << 32) + lo;
 169 }
 170
 171 unsigned int gic_get_count_width(void)
 172 {
 173         unsigned int bits, config;
 174
 175         config = gic_read(GIC_REG(SHARED, GIC_SH_CONFIG));
 176         bits = 32 + 4 * ((config & GIC_SH_CONFIG_COUNTBITS_MSK) >>
 177                          GIC_SH_CONFIG_COUNTBITS_SHF);
 178
 179         return bits;
 180 }
 181
 182 void gic_write_compare(cycle_t cnt)
 183 {
 184         if (mips_cm_is64) {
 185                 gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE), cnt);
 186         } else {
 187                 gic_write32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_HI),
 188                                         (int)(cnt >> 32));
 189                 gic_write32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_LO),
 190                                         (int)(cnt & 0xffffffff));
 191         }
 192 }
 193
 194 void gic_write_cpu_compare(cycle_t cnt, int cpu)
 195 {
 196         unsigned long flags;
 197
 198         local_irq_save(flags);
 199
 200         gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR), cpu);
 201
 202         if (mips_cm_is64) {
 203                 gic_write(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE), cnt);
 204         } else {
 205                 gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE_HI),
 206                                         (int)(cnt >> 32));
 207                 gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE_LO),
 208                                         (int)(cnt & 0xffffffff));
 209         }
 210
 211         local_irq_restore(flags);
 212 }
 213
 214 cycle_t gic_read_compare(void)
 215 {
 216         unsigned int hi, lo;
 217
 218         if (mips_cm_is64)
 219                 return (cycle_t)gic_read(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE));
 220
 221         hi = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_HI));
 222         lo = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_LO));
 223
 224         return (((cycle_t) hi) << 32) + lo;
 225 }
 226
 227 void gic_start_count(void)
 228 {
 229         u32 gicconfig;
 230
 231         /* Start the counter */
 232         gicconfig = gic_read(GIC_REG(SHARED, GIC_SH_CONFIG));
 233         gicconfig &= ~(1 << GIC_SH_CONFIG_COUNTSTOP_SHF);
 234         gic_write(GIC_REG(SHARED, GIC_SH_CONFIG), gicconfig);
 235 }
 236
 237 void gic_stop_count(void)
 238 {
 239         u32 gicconfig;
 240
 241         /* Stop the counter */
 242         gicconfig = gic_read(GIC_REG(SHARED, GIC_SH_CONFIG));
 243         gicconfig |= 1 << GIC_SH_CONFIG_COUNTSTOP_SHF;
 244         gic_write(GIC_REG(SHARED, GIC_SH_CONFIG), gicconfig);
 245 }
 246
 247 #endif
 248
 249 static bool gic_local_irq_is_routable(int intr)
 250 {
 251         u32 vpe_ctl;
 252
 253         /* All local interrupts are routable in EIC mode. */
 254         if (cpu_has_veic)
 255                 return true;
 256
 257         vpe_ctl = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_CTL));
 258         switch (intr) {
 259         case GIC_LOCAL_INT_TIMER:
 260                 return vpe_ctl & GIC_VPE_CTL_TIMER_RTBL_MSK;
 261         case GIC_LOCAL_INT_PERFCTR:
 262                 return vpe_ctl & GIC_VPE_CTL_PERFCNT_RTBL_MSK;
 263         case GIC_LOCAL_INT_FDC:
 264                 return vpe_ctl & GIC_VPE_CTL_FDC_RTBL_MSK;
 265         case GIC_LOCAL_INT_SWINT0:
 266         case GIC_LOCAL_INT_SWINT1:
 267                 return vpe_ctl & GIC_VPE_CTL_SWINT_RTBL_MSK;
 268         default:
 269                 return true;
 270         }
 271 }
 272
 273 static void gic_bind_eic_interrupt(int irq, int set)
 274 {
 275         /* Convert irq vector # to hw int # */
 276         irq -= GIC_PIN_TO_VEC_OFFSET;
 277
 278         /* Set irq to use shadow set */
 279         gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_EIC_SHADOW_SET_BASE) +
 280                   GIC_VPE_EIC_SS(irq), set);
 281 }
 282
 283 void gic_send_ipi(unsigned int intr)
 284 {
 285         gic_write(GIC_REG(SHARED, GIC_SH_WEDGE), GIC_SH_WEDGE_SET(intr));
 286 }
 287
 288 int gic_get_c0_compare_int(void)
 289 {
 290         if (!gic_local_irq_is_routable(GIC_LOCAL_INT_TIMER))
 291                 return MIPS_CPU_IRQ_BASE + cp0_compare_irq;
 292         return irq_create_mapping(gic_irq_domain,
 293                                   GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_TIMER));
 294 }
 295
 296 int gic_get_c0_perfcount_int(void)
 297 {
 298         if (!gic_local_irq_is_routable(GIC_LOCAL_INT_PERFCTR)) {
 299                 /* Is the performance counter shared with the timer? */
 300                 if (cp0_perfcount_irq < 0)
 301                         return -1;
 302                 return MIPS_CPU_IRQ_BASE + cp0_perfcount_irq;
 303         }
 304         return irq_create_mapping(gic_irq_domain,
 305                                   GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_PERFCTR));
 306 }
 307
 308 int gic_get_c0_fdc_int(void)
 309 {
 310         if (!gic_local_irq_is_routable(GIC_LOCAL_INT_FDC)) {
 311                 /* Is the FDC IRQ even present? */
 312                 if (cp0_fdc_irq < 0)
 313                         return -1;
 314                 return MIPS_CPU_IRQ_BASE + cp0_fdc_irq;
 315         }
 316
 317         return irq_create_mapping(gic_irq_domain,
 318                                   GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_FDC));
 319 }
 320
 321 int gic_get_usm_range(struct resource *gic_usm_res)
 322 {
 323         if (!gic_present)
 324                 return -1;
 325
 326         gic_usm_res->start = __gic_base_addr + USM_VISIBLE_SECTION_OFS;
 327         gic_usm_res->end = gic_usm_res->start + (USM_VISIBLE_SECTION_SIZE - 1);
 328
 329         return 0;
 330 }
 331
 332 static void gic_handle_shared_int(bool chained)
 333 {
 334         unsigned int i, intr, virq, gic_reg_step = mips_cm_is64 ? 8 : 4;
 335         unsigned long *pcpu_mask;
 336         unsigned long pending_reg, intrmask_reg;
 337         DECLARE_BITMAP(pending, GIC_MAX_INTRS);
 338         DECLARE_BITMAP(intrmask, GIC_MAX_INTRS);
 339
 340         /* Get per-cpu bitmaps */
 341         pcpu_mask = pcpu_masks[smp_processor_id()].pcpu_mask;
 342
 343         pending_reg = GIC_REG(SHARED, GIC_SH_PEND);
 344         intrmask_reg = GIC_REG(SHARED, GIC_SH_MASK);
 345
 346         for (i = 0; i < BITS_TO_LONGS(gic_shared_intrs); i++) {
 347                 pending[i] = gic_read(pending_reg);
 348                 intrmask[i] = gic_read(intrmask_reg);
 349                 pending_reg += gic_reg_step;
 350                 intrmask_reg += gic_reg_step;
 351
 352                 if (!config_enabled(CONFIG_64BIT) || mips_cm_is64)
 353                         continue;
 354
 355                 pending[i] |= (u64)gic_read(pending_reg) << 32;
 356                 intrmask[i] |= (u64)gic_read(intrmask_reg) << 32;
 357                 pending_reg += gic_reg_step;
 358                 intrmask_reg += gic_reg_step;
 359         }
 360
 361         bitmap_and(pending, pending, intrmask, gic_shared_intrs);
 362         bitmap_and(pending, pending, pcpu_mask, gic_shared_intrs);
 363
 364         intr = find_first_bit(pending, gic_shared_intrs);
 365         while (intr != gic_shared_intrs) {
 366                 virq = irq_linear_revmap(gic_irq_domain,
 367                                          GIC_SHARED_TO_HWIRQ(intr));
 368                 if (chained)
 369                         generic_handle_irq(virq);
 370                 else
 371                         do_IRQ(virq);
 372
 373                 /* go to next pending bit */
 374                 bitmap_clear(pending, intr, 1);
 375                 intr = find_first_bit(pending, gic_shared_intrs);
 376         }
 377 }
 378
 379 static void gic_mask_irq(struct irq_data *d)
 380 {
 381         gic_reset_mask(GIC_HWIRQ_TO_SHARED(d->hwirq));
 382 }
 383
 384 static void gic_unmask_irq(struct irq_data *d)
 385 {
 386         gic_set_mask(GIC_HWIRQ_TO_SHARED(d->hwirq));
 387 }
 388
 389 static void gic_ack_irq(struct irq_data *d)
 390 {
 391         unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
 392
 393         gic_write(GIC_REG(SHARED, GIC_SH_WEDGE), GIC_SH_WEDGE_CLR(irq));
 394 }
 395
 396 static int gic_set_type(struct irq_data *d, unsigned int type)
 397 {
 398         unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
 399         unsigned long flags;
 400         bool is_edge;
 401
 402         spin_lock_irqsave(&gic_lock, flags);
 403         switch (type & IRQ_TYPE_SENSE_MASK) {
 404         case IRQ_TYPE_EDGE_FALLING:
 405                 gic_set_polarity(irq, GIC_POL_NEG);
 406                 gic_set_trigger(irq, GIC_TRIG_EDGE);
 407                 gic_set_dual_edge(irq, GIC_TRIG_DUAL_DISABLE);
 408                 is_edge = true;
 409                 break;
 410         case IRQ_TYPE_EDGE_RISING:
 411                 gic_set_polarity(irq, GIC_POL_POS);
 412                 gic_set_trigger(irq, GIC_TRIG_EDGE);
 413                 gic_set_dual_edge(irq, GIC_TRIG_DUAL_DISABLE);
 414                 is_edge = true;
 415                 break;
 416         case IRQ_TYPE_EDGE_BOTH:
 417                 /* polarity is irrelevant in this case */
 418                 gic_set_trigger(irq, GIC_TRIG_EDGE);
 419                 gic_set_dual_edge(irq, GIC_TRIG_DUAL_ENABLE);
 420                 is_edge = true;
 421                 break;
 422         case IRQ_TYPE_LEVEL_LOW:
 423                 gic_set_polarity(irq, GIC_POL_NEG);
 424                 gic_set_trigger(irq, GIC_TRIG_LEVEL);
 425                 gic_set_dual_edge(irq, GIC_TRIG_DUAL_DISABLE);
 426                 is_edge = false;
 427                 break;
 428         case IRQ_TYPE_LEVEL_HIGH:
 429         default:
 430                 gic_set_polarity(irq, GIC_POL_POS);
 431                 gic_set_trigger(irq, GIC_TRIG_LEVEL);
 432                 gic_set_dual_edge(irq, GIC_TRIG_DUAL_DISABLE);
 433                 is_edge = false;
 434                 break;
 435         }
 436
 437         if (is_edge)
 438                 irq_set_chip_handler_name_locked(d, &gic_edge_irq_controller,
 439                                                  handle_edge_irq, NULL);
 440         else
 441                 irq_set_chip_handler_name_locked(d, &gic_level_irq_controller,
 442                                                  handle_level_irq, NULL);
 443         spin_unlock_irqrestore(&gic_lock, flags);
 444
 445         return 0;
 446 }
 447
 448 #ifdef CONFIG_SMP
 449 static int gic_set_affinity(struct irq_data *d, const struct cpumask *cpumask,
 450                             bool force)
 451 {
 452         unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
 453         cpumask_t       tmp = CPU_MASK_NONE;
 454         unsigned long   flags;
 455         int             i;
 456
 457         cpumask_and(&tmp, cpumask, cpu_online_mask);
 458         if (cpumask_empty(&tmp))
 459                 return -EINVAL;
 460
 461         /* Assumption : cpumask refers to a single CPU */
 462         spin_lock_irqsave(&gic_lock, flags);
 463
 464         /* Re-route this IRQ */
 465         gic_map_to_vpe(irq, mips_cm_vp_id(cpumask_first(&tmp)));
 466
 467         /* Update the pcpu_masks */
 468         for (i = 0; i < gic_vpes; i++)
 469                 clear_bit(irq, pcpu_masks[i].pcpu_mask);
 470         set_bit(irq, pcpu_masks[cpumask_first(&tmp)].pcpu_mask);
 471
 472         cpumask_copy(irq_data_get_affinity_mask(d), cpumask);
 473         spin_unlock_irqrestore(&gic_lock, flags);
 474
 475         return IRQ_SET_MASK_OK_NOCOPY;
 476 }
 477 #endif
 478
 479 static struct irq_chip gic_level_irq_controller = {
 480         .name                   =       "MIPS GIC",
 481         .irq_mask               =       gic_mask_irq,
 482         .irq_unmask             =       gic_unmask_irq,
 483         .irq_set_type           =       gic_set_type,
 484 #ifdef CONFIG_SMP
 485         .irq_set_affinity       =       gic_set_affinity,
 486 #endif
 487 };
 488
 489 static struct irq_chip gic_edge_irq_controller = {
 490         .name                   =       "MIPS GIC",
 491         .irq_ack                =       gic_ack_irq,
 492         .irq_mask               =       gic_mask_irq,
 493         .irq_unmask             =       gic_unmask_irq,
 494         .irq_set_type           =       gic_set_type,
 495 #ifdef CONFIG_SMP
 496         .irq_set_affinity       =       gic_set_affinity,
 497 #endif
 498 };
 499
 500 static void gic_handle_local_int(bool chained)
 501 {
 502         unsigned long pending, masked;
 503         unsigned int intr, virq;
 504
 505         pending = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_PEND));
 506         masked = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_MASK));
 507
 508         bitmap_and(&pending, &pending, &masked, GIC_NUM_LOCAL_INTRS);
 509
 510         intr = find_first_bit(&pending, GIC_NUM_LOCAL_INTRS);
 511         while (intr != GIC_NUM_LOCAL_INTRS) {
 512                 virq = irq_linear_revmap(gic_irq_domain,
 513                                          GIC_LOCAL_TO_HWIRQ(intr));
 514                 if (chained)
 515                         generic_handle_irq(virq);
 516                 else
 517                         do_IRQ(virq);
 518
 519                 /* go to next pending bit */
 520                 bitmap_clear(&pending, intr, 1);
 521                 intr = find_first_bit(&pending, GIC_NUM_LOCAL_INTRS);
 522         }
 523 }
 524
 525 static void gic_mask_local_irq(struct irq_data *d)
 526 {
 527         int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
 528
 529         gic_write32(GIC_REG(VPE_LOCAL, GIC_VPE_RMASK), 1 << intr);
 530 }
 531
 532 static void gic_unmask_local_irq(struct irq_data *d)
 533 {
 534         int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
 535
 536         gic_write32(GIC_REG(VPE_LOCAL, GIC_VPE_SMASK), 1 << intr);
 537 }
 538
 539 static struct irq_chip gic_local_irq_controller = {
 540         .name                   =       "MIPS GIC Local",
 541         .irq_mask               =       gic_mask_local_irq,
 542         .irq_unmask             =       gic_unmask_local_irq,
 543 };
 544
 545 static void gic_mask_local_irq_all_vpes(struct irq_data *d)
 546 {
 547         int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
 548         int i;
 549         unsigned long flags;
 550
 551         spin_lock_irqsave(&gic_lock, flags);
 552         for (i = 0; i < gic_vpes; i++) {
 553                 gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR), i);
 554                 gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_RMASK), 1 << intr);
 555         }
 556         spin_unlock_irqrestore(&gic_lock, flags);
 557 }
 558
 559 static void gic_unmask_local_irq_all_vpes(struct irq_data *d)
 560 {
 561         int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
 562         int i;
 563         unsigned long flags;
 564
 565         spin_lock_irqsave(&gic_lock, flags);
 566         for (i = 0; i < gic_vpes; i++) {
 567                 gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR), i);
 568                 gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_SMASK), 1 << intr);
 569         }
 570         spin_unlock_irqrestore(&gic_lock, flags);
 571 }
 572
 573 static struct irq_chip gic_all_vpes_local_irq_controller = {
 574         .name                   =       "MIPS GIC Local",
 575         .irq_mask               =       gic_mask_local_irq_all_vpes,
 576         .irq_unmask             =       gic_unmask_local_irq_all_vpes,
 577 };
 578
 579 static void __gic_irq_dispatch(void)
 580 {
 581         gic_handle_local_int(false);
 582         gic_handle_shared_int(false);
 583 }
 584
 585 static void gic_irq_dispatch(struct irq_desc *desc)
 586 {
 587         gic_handle_local_int(true);
 588         gic_handle_shared_int(true);
 589 }
 590
 591 #ifdef CONFIG_MIPS_GIC_IPI
 592 static int gic_resched_int_base;
 593 static int gic_call_int_base;
 594
 595 unsigned int plat_ipi_resched_int_xlate(unsigned int cpu)
 596 {
 597         return gic_resched_int_base + cpu;
 598 }
 599
 600 unsigned int plat_ipi_call_int_xlate(unsigned int cpu)
 601 {
 602         return gic_call_int_base + cpu;
 603 }
 604
 605 static irqreturn_t ipi_resched_interrupt(int irq, void *dev_id)
 606 {
 607         scheduler_ipi();
 608
 609         return IRQ_HANDLED;
 610 }
 611
 612 static irqreturn_t ipi_call_interrupt(int irq, void *dev_id)
 613 {
 614         generic_smp_call_function_interrupt();
 615
 616         return IRQ_HANDLED;
 617 }
 618
 619 static struct irqaction irq_resched = {
 620         .handler        = ipi_resched_interrupt,
 621         .flags          = IRQF_PERCPU,
 622         .name           = "IPI resched"
 623 };
 624
 625 static struct irqaction irq_call = {
 626         .handler        = ipi_call_interrupt,
 627         .flags          = IRQF_PERCPU,
 628         .name           = "IPI call"
 629 };
 630
 631 static __init void gic_ipi_init_one(unsigned int intr, int cpu,
 632                                     struct irqaction *action)
 633 {
 634         int virq = irq_create_mapping(gic_irq_domain,
 635                                       GIC_SHARED_TO_HWIRQ(intr));
 636         int i;
 637
 638         gic_map_to_vpe(intr, mips_cm_vp_id(cpu));
 639         for (i = 0; i < NR_CPUS; i++)
 640                 clear_bit(intr, pcpu_masks[i].pcpu_mask);
 641         set_bit(intr, pcpu_masks[cpu].pcpu_mask);
 642
 643         irq_set_irq_type(virq, IRQ_TYPE_EDGE_RISING);
 644
 645         irq_set_handler(virq, handle_percpu_irq);
 646         setup_irq(virq, action);
 647 }
 648
 649 static __init void gic_ipi_init(void)
 650 {
 651         int i;
 652
 653         /* Use last 2 * NR_CPUS interrupts as IPIs */
 654         gic_resched_int_base = gic_shared_intrs - nr_cpu_ids;
 655         gic_call_int_base = gic_resched_int_base - nr_cpu_ids;
 656
 657         for (i = 0; i < nr_cpu_ids; i++) {
 658                 gic_ipi_init_one(gic_call_int_base + i, i, &irq_call);
 659                 gic_ipi_init_one(gic_resched_int_base + i, i, &irq_resched);
 660         }
 661 }
 662 #else
 663 static inline void gic_ipi_init(void)
 664 {
 665 }
 666 #endif
 667
 668 static void __init gic_basic_init(void)
 669 {
 670         unsigned int i;
 671
 672         board_bind_eic_interrupt = &gic_bind_eic_interrupt;
 673
 674         /* Setup defaults */
 675         for (i = 0; i < gic_shared_intrs; i++) {
 676                 gic_set_polarity(i, GIC_POL_POS);
 677                 gic_set_trigger(i, GIC_TRIG_LEVEL);
 678                 gic_reset_mask(i);
 679         }
 680
 681         for (i = 0; i < gic_vpes; i++) {
 682                 unsigned int j;
 683
 684                 gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR), i);
 685                 for (j = 0; j < GIC_NUM_LOCAL_INTRS; j++) {
 686                         if (!gic_local_irq_is_routable(j))
 687                                 continue;
 688                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_RMASK), 1 << j);
 689                 }
 690         }
 691 }
 692
 693 static int gic_local_irq_domain_map(struct irq_domain *d, unsigned int virq,
 694                                     irq_hw_number_t hw)
 695 {
 696         int intr = GIC_HWIRQ_TO_LOCAL(hw);
 697         int ret = 0;
 698         int i;
 699         unsigned long flags;
 700
 701         if (!gic_local_irq_is_routable(intr))
 702                 return -EPERM;
 703
 704         /*
 705          * HACK: These are all really percpu interrupts, but the rest
 706          * of the MIPS kernel code does not use the percpu IRQ API for
 707          * the CP0 timer and performance counter interrupts.
 708          */
 709         switch (intr) {
 710         case GIC_LOCAL_INT_TIMER:
 711         case GIC_LOCAL_INT_PERFCTR:
 712         case GIC_LOCAL_INT_FDC:
 713                 irq_set_chip_and_handler(virq,
 714                                          &gic_all_vpes_local_irq_controller,
 715                                          handle_percpu_irq);
 716                 break;
 717         default:
 718                 irq_set_chip_and_handler(virq,
 719                                          &gic_local_irq_controller,
 720                                          handle_percpu_devid_irq);
 721                 irq_set_percpu_devid(virq);
 722                 break;
 723         }
 724
 725         spin_lock_irqsave(&gic_lock, flags);
 726         for (i = 0; i < gic_vpes; i++) {
 727                 u32 val = GIC_MAP_TO_PIN_MSK | gic_cpu_pin;
 728
 729                 gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR), i);
 730
 731                 switch (intr) {
 732                 case GIC_LOCAL_INT_WD:
 733                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_WD_MAP), val);
 734                         break;
 735                 case GIC_LOCAL_INT_COMPARE:
 736                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE_MAP),
 737                                     val);
 738                         break;
 739                 case GIC_LOCAL_INT_TIMER:
 740                         /* CONFIG_MIPS_CMP workaround (see __gic_init) */
 741                         val = GIC_MAP_TO_PIN_MSK | timer_cpu_pin;
 742                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_TIMER_MAP),
 743                                     val);
 744                         break;
 745                 case GIC_LOCAL_INT_PERFCTR:
 746                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_PERFCTR_MAP),
 747                                     val);
 748                         break;
 749                 case GIC_LOCAL_INT_SWINT0:
 750                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_SWINT0_MAP),
 751                                     val);
 752                         break;
 753                 case GIC_LOCAL_INT_SWINT1:
 754                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_SWINT1_MAP),
 755                                     val);
 756                         break;
 757                 case GIC_LOCAL_INT_FDC:
 758                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_FDC_MAP), val);
 759                         break;
 760                 default:
 761                         pr_err("Invalid local IRQ %d\n", intr);
 762                         ret = -EINVAL;
 763                         break;
 764                 }
 765         }
 766         spin_unlock_irqrestore(&gic_lock, flags);
 767
 768         return ret;
 769 }
 770
 771 static int gic_shared_irq_domain_map(struct irq_domain *d, unsigned int virq,
 772                                      irq_hw_number_t hw, unsigned int vpe)
 773 {
 774         int intr = GIC_HWIRQ_TO_SHARED(hw);
 775         unsigned long flags;
 776
 777         irq_set_chip_and_handler(virq, &gic_level_irq_controller,
 778                                  handle_level_irq);
 779
 780         spin_lock_irqsave(&gic_lock, flags);
 781         gic_map_to_pin(intr, gic_cpu_pin);
 782         gic_map_to_vpe(intr, vpe);
 783         set_bit(intr, pcpu_masks[vpe].pcpu_mask);
 784         spin_unlock_irqrestore(&gic_lock, flags);
 785
 786         return 0;
 787 }
 788
 789 static int gic_irq_domain_map(struct irq_domain *d, unsigned int virq,
 790                               irq_hw_number_t hw)
 791 {
 792         if (GIC_HWIRQ_TO_LOCAL(hw) < GIC_NUM_LOCAL_INTRS)
 793                 return gic_local_irq_domain_map(d, virq, hw);
 794         return gic_shared_irq_domain_map(d, virq, hw, 0);
 795 }
 796
 797 static int gic_irq_domain_alloc(struct irq_domain *d, unsigned int virq,
 798                                 unsigned int nr_irqs, void *arg)
 799 {
 800         struct gic_irq_spec *spec = arg;
 801         irq_hw_number_t hwirq, base_hwirq;
 802         int cpu, ret, i;
 803
 804         if (spec->type == GIC_DEVICE) {
 805                 /* verify that it doesn't conflict with an IPI irq */
 806                 if (test_bit(spec->hwirq, ipi_resrv))
 807                         return -EBUSY;
 808         } else {
 809                 base_hwirq = find_first_bit(ipi_resrv, gic_shared_intrs);
 810                 if (base_hwirq == gic_shared_intrs) {
 811                         return -ENOMEM;
 812                 }
 813
 814                 /* check that we have enough space */
 815                 for (i = base_hwirq; i < nr_irqs; i++) {
 816                         if (!test_bit(i, ipi_resrv))
 817                                 return -EBUSY;
 818                 }
 819                 bitmap_clear(ipi_resrv, base_hwirq, nr_irqs);
 820
 821                 /* map the hwirq for each cpu consecutively */
 822                 i = 0;
 823                 for_each_cpu(cpu, spec->ipimask) {
 824                         hwirq = GIC_SHARED_TO_HWIRQ(base_hwirq + i);
 825
 826                         ret = irq_domain_set_hwirq_and_chip(d, virq + i, hwirq,
 827                                                             &gic_edge_irq_controller,
 828                                                             NULL);
 829                         if (ret)
 830                                 goto error;
 831
 832                         ret = gic_shared_irq_domain_map(d, virq + i, hwirq, cpu);
 833                         if (ret)
 834                                 goto error;
 835
 836                         i++;
 837                 }
 838
 839                 /*
 840                  * tell the parent about the base hwirq we allocated so it can
 841                  * set its own domain data
 842                  */
 843                 spec->hwirq = base_hwirq;
 844         }
 845
 846         return 0;
 847 error:
 848         bitmap_set(ipi_resrv, base_hwirq, nr_irqs);
 849         return ret;
 850 }
 851
 852 void gic_irq_domain_free(struct irq_domain *d, unsigned int virq,
 853                          unsigned int nr_irqs)
 854 {
 855         irq_hw_number_t base_hwirq;
 856         struct irq_data *data;
 857
 858         data = irq_get_irq_data(virq);
 859         if (!data)
 860                 return;
 861
 862         base_hwirq = GIC_HWIRQ_TO_SHARED(irqd_to_hwirq(data));
 863         bitmap_set(ipi_resrv, base_hwirq, nr_irqs);
 864 }
 865
 866 int gic_irq_domain_match(struct irq_domain *d, struct device_node *node,
 867                          enum irq_domain_bus_token bus_token)
 868 {
 869         /* this domain should'nt be accessed directly */
 870         return 0;
 871 }
 872
 873 static const struct irq_domain_ops gic_irq_domain_ops = {
 874         .map = gic_irq_domain_map,
 875         .alloc = gic_irq_domain_alloc,
 876         .free = gic_irq_domain_free,
 877         .match = gic_irq_domain_match,
 878 };
 879
 880 static int gic_dev_domain_xlate(struct irq_domain *d, struct device_node *ctrlr,
 881                                 const u32 *intspec, unsigned int intsize,
 882                                 irq_hw_number_t *out_hwirq,
 883                                 unsigned int *out_type)
 884 {
 885         if (intsize != 3)
 886                 return -EINVAL;
 887
 888         if (intspec[0] == GIC_SHARED)
 889                 *out_hwirq = GIC_SHARED_TO_HWIRQ(intspec[1]);
 890         else if (intspec[0] == GIC_LOCAL)
 891                 *out_hwirq = GIC_LOCAL_TO_HWIRQ(intspec[1]);
 892         else
 893                 return -EINVAL;
 894         *out_type = intspec[2] & IRQ_TYPE_SENSE_MASK;
 895
 896         return 0;
 897 }
 898
 899 static int gic_dev_domain_alloc(struct irq_domain *d, unsigned int virq,
 900                                 unsigned int nr_irqs, void *arg)
 901 {
 902         struct irq_fwspec *fwspec = arg;
 903         struct gic_irq_spec spec = {
 904                 .type = GIC_DEVICE,
 905                 .hwirq = fwspec->param[1],
 906         };
 907         int i, ret;
 908         bool is_shared = fwspec->param[0] == GIC_SHARED;
 909
 910         if (is_shared) {
 911                 ret = irq_domain_alloc_irqs_parent(d, virq, nr_irqs, &spec);
 912                 if (ret)
 913                         return ret;
 914         }
 915
 916         for (i = 0; i < nr_irqs; i++) {
 917                 irq_hw_number_t hwirq;
 918
 919                 if (is_shared)
 920                         hwirq = GIC_SHARED_TO_HWIRQ(spec.hwirq + i);
 921                 else
 922                         hwirq = GIC_LOCAL_TO_HWIRQ(spec.hwirq + i);
 923
 924                 ret = irq_domain_set_hwirq_and_chip(d, virq + i,
 925                                                     hwirq,
 926                                                     &gic_level_irq_controller,
 927                                                     NULL);
 928                 if (ret)
 929                         return ret;
 930         }
 931
 932         return 0;
 933 }
 934
 935 void gic_dev_domain_free(struct irq_domain *d, unsigned int virq,
 936                          unsigned int nr_irqs)
 937 {
 938         /* no real allocation is done for dev irqs, so no need to free anything */
 939         return;
 940 }
 941
 942 static struct irq_domain_ops gic_dev_domain_ops = {
 943         .xlate = gic_dev_domain_xlate,
 944         .alloc = gic_dev_domain_alloc,
 945         .free = gic_dev_domain_free,
 946 };
 947
 948 static int gic_ipi_domain_xlate(struct irq_domain *d, struct device_node *ctrlr,
 949                                 const u32 *intspec, unsigned int intsize,
 950                                 irq_hw_number_t *out_hwirq,
 951                                 unsigned int *out_type)
 952 {
 953         /*
 954          * There's nothing to translate here. hwirq is dynamically allocated and
 955          * the irq type is always edge triggered.
 956          * */
 957         *out_hwirq = 0;
 958         *out_type = IRQ_TYPE_EDGE_RISING;
 959
 960         return 0;
 961 }
 962
 963 static int gic_ipi_domain_alloc(struct irq_domain *d, unsigned int virq,
 964                                 unsigned int nr_irqs, void *arg)
 965 {
 966         struct cpumask *ipimask = arg;
 967         struct gic_irq_spec spec = {
 968                 .type = GIC_IPI,
 969                 .ipimask = ipimask
 970         };
 971         int ret, i;
 972
 973         ret = irq_domain_alloc_irqs_parent(d, virq, nr_irqs, &spec);
 974         if (ret)
 975                 return ret;
 976
 977         /* the parent should have set spec.hwirq to the base_hwirq it allocated */
 978         for (i = 0; i < nr_irqs; i++) {
 979                 ret = irq_domain_set_hwirq_and_chip(d, virq + i,
 980                                                     GIC_SHARED_TO_HWIRQ(spec.hwirq + i),
 981                                                     &gic_edge_irq_controller,
 982                                                     NULL);
 983                 if (ret)
 984                         goto error;
 985
 986                 ret = irq_set_irq_type(virq + i, IRQ_TYPE_EDGE_RISING);
 987                 if (ret)
 988                         goto error;
 989         }
 990
 991         return 0;
 992 error:
 993         irq_domain_free_irqs_parent(d, virq, nr_irqs);
 994         return ret;
 995 }
 996
 997 void gic_ipi_domain_free(struct irq_domain *d, unsigned int virq,
 998                          unsigned int nr_irqs)
 999 {
1000         irq_domain_free_irqs_parent(d, virq, nr_irqs);
1001 }
1002
1003 int gic_ipi_domain_match(struct irq_domain *d, struct device_node *node,
1004                          enum irq_domain_bus_token bus_token)
1005 {
1006         bool is_ipi;
1007
1008         switch (bus_token) {
1009         case DOMAIN_BUS_IPI:
1010                 is_ipi = d->bus_token == bus_token;
1011                 return to_of_node(d->fwnode) == node && is_ipi;
1012                 break;
1013         default:
1014                 return 0;
1015         }
1016 }
1017
1018 static struct irq_domain_ops gic_ipi_domain_ops = {
1019         .xlate = gic_ipi_domain_xlate,
1020         .alloc = gic_ipi_domain_alloc,
1021         .free = gic_ipi_domain_free,
1022         .match = gic_ipi_domain_match,
1023 };
1024
1025 static void __init __gic_init(unsigned long gic_base_addr,
1026                               unsigned long gic_addrspace_size,
1027                               unsigned int cpu_vec, unsigned int irqbase,
1028                               struct device_node *node)
1029 {
1030         unsigned int gicconfig;
1031
1032         __gic_base_addr = gic_base_addr;
1033
1034         gic_base = ioremap_nocache(gic_base_addr, gic_addrspace_size);
1035
1036         gicconfig = gic_read(GIC_REG(SHARED, GIC_SH_CONFIG));
1037         gic_shared_intrs = (gicconfig & GIC_SH_CONFIG_NUMINTRS_MSK) >>
1038                    GIC_SH_CONFIG_NUMINTRS_SHF;
1039         gic_shared_intrs = ((gic_shared_intrs + 1) * 8);
1040
1041         gic_vpes = (gicconfig & GIC_SH_CONFIG_NUMVPES_MSK) >>
1042                   GIC_SH_CONFIG_NUMVPES_SHF;
1043         gic_vpes = gic_vpes + 1;
1044
1045         if (cpu_has_veic) {
1046                 /* Always use vector 1 in EIC mode */
1047                 gic_cpu_pin = 0;
1048                 timer_cpu_pin = gic_cpu_pin;
1049                 set_vi_handler(gic_cpu_pin + GIC_PIN_TO_VEC_OFFSET,
1050                                __gic_irq_dispatch);
1051         } else {
1052                 gic_cpu_pin = cpu_vec - GIC_CPU_PIN_OFFSET;
1053                 irq_set_chained_handler(MIPS_CPU_IRQ_BASE + cpu_vec,
1054                                         gic_irq_dispatch);
1055                 /*
1056                  * With the CMP implementation of SMP (deprecated), other CPUs
1057                  * are started by the bootloader and put into a timer based
1058                  * waiting poll loop. We must not re-route those CPU's local
1059                  * timer interrupts as the wait instruction will never finish,
1060                  * so just handle whatever CPU interrupt it is routed to by
1061                  * default.
1062                  *
1063                  * This workaround should be removed when CMP support is
1064                  * dropped.
1065                  */
1066                 if (IS_ENABLED(CONFIG_MIPS_CMP) &&
1067                     gic_local_irq_is_routable(GIC_LOCAL_INT_TIMER)) {
1068                         timer_cpu_pin = gic_read32(GIC_REG(VPE_LOCAL,
1069                                                          GIC_VPE_TIMER_MAP)) &
1070                                         GIC_MAP_MSK;
1071                         irq_set_chained_handler(MIPS_CPU_IRQ_BASE +
1072                                                 GIC_CPU_PIN_OFFSET +
1073                                                 timer_cpu_pin,
1074                                                 gic_irq_dispatch);
1075                 } else {
1076                         timer_cpu_pin = gic_cpu_pin;
1077                 }
1078         }
1079
1080         gic_irq_domain = irq_domain_add_simple(node, GIC_NUM_LOCAL_INTRS +
1081                                                gic_shared_intrs, irqbase,
1082                                                &gic_irq_domain_ops, NULL);
1083         if (!gic_irq_domain)
1084                 panic("Failed to add GIC IRQ domain");
1085
1086         gic_dev_domain = irq_domain_add_hierarchy(gic_irq_domain, 0,
1087                                                   GIC_NUM_LOCAL_INTRS + gic_shared_intrs,
1088                                                   node, &gic_dev_domain_ops, NULL);
1089         if (!gic_dev_domain)
1090                 panic("Failed to add GIC DEV domain");
1091
1092         gic_ipi_domain = irq_domain_add_hierarchy(gic_irq_domain,
1093                                                   IRQ_DOMAIN_FLAG_IPI_PER_CPU,
1094                                                   GIC_NUM_LOCAL_INTRS + gic_shared_intrs,
1095                                                   node, &gic_ipi_domain_ops, NULL);
1096         if (!gic_ipi_domain)
1097                 panic("Failed to add GIC IPI domain");
1098
1099         gic_ipi_domain->bus_token = DOMAIN_BUS_IPI;
1100
1101         /* Make the last 2 * gic_vpes available for IPIs */
1102         bitmap_set(ipi_resrv, gic_shared_intrs - 2 * gic_vpes, 2 * gic_vpes);
1103
1104         gic_basic_init();
1105
1106         gic_ipi_init();
1107 }
1108
1109 void __init gic_init(unsigned long gic_base_addr,
1110                      unsigned long gic_addrspace_size,
1111                      unsigned int cpu_vec, unsigned int irqbase)
1112 {
1113         __gic_init(gic_base_addr, gic_addrspace_size, cpu_vec, irqbase, NULL);
1114 }
1115
1116 static int __init gic_of_init(struct device_node *node,
1117                               struct device_node *parent)
1118 {
1119         struct resource res;
1120         unsigned int cpu_vec, i = 0, reserved = 0;
1121         phys_addr_t gic_base;
1122         size_t gic_len;
1123
1124         /* Find the first available CPU vector. */
1125         while (!of_property_read_u32_index(node, "mti,reserved-cpu-vectors",
1126                                            i++, &cpu_vec))
1127                 reserved |= BIT(cpu_vec);
1128         for (cpu_vec = 2; cpu_vec < 8; cpu_vec++) {
1129                 if (!(reserved & BIT(cpu_vec)))
1130                         break;
1131         }
1132         if (cpu_vec == 8) {
1133                 pr_err("No CPU vectors available for GIC\n");
1134                 return -ENODEV;
1135         }
1136
1137         if (of_address_to_resource(node, 0, &res)) {
1138                 /*
1139                  * Probe the CM for the GIC base address if not specified
1140                  * in the device-tree.
1141                  */
1142                 if (mips_cm_present()) {
1143                         gic_base = read_gcr_gic_base() &
1144                                 ~CM_GCR_GIC_BASE_GICEN_MSK;
1145                         gic_len = 0x20000;
1146                 } else {
1147                         pr_err("Failed to get GIC memory range\n");
1148                         return -ENODEV;
1149                 }
1150         } else {
1151                 gic_base = res.start;
1152                 gic_len = resource_size(&res);
1153         }
1154
1155         if (mips_cm_present())
1156                 write_gcr_gic_base(gic_base | CM_GCR_GIC_BASE_GICEN_MSK);
1157         gic_present = true;
1158
1159         __gic_init(gic_base, gic_len, cpu_vec, 0, node);
1160
1161         return 0;
1162 }
1163 IRQCHIP_DECLARE(mips_gic, "mti,gic", gic_of_init);