drivers/gpu/drm/i915/i915_irq.c

   1 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
   2  */
   3 /*
   4  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  20  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  22  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  23  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  24  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  25  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  */
  28
  29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  30
  31 #include <linux/sysrq.h>
  32 #include <linux/slab.h>
  33 #include <drm/drmP.h>
  34 #include <drm/i915_drm.h>
  35 #include "i915_drv.h"
  36 #include "i915_trace.h"
  37 #include "intel_drv.h"
  38
  39 static const u32 hpd_ibx[] = {
  40         [HPD_CRT] = SDE_CRT_HOTPLUG,
  41         [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
  42         [HPD_PORT_B] = SDE_PORTB_HOTPLUG,
  43         [HPD_PORT_C] = SDE_PORTC_HOTPLUG,
  44         [HPD_PORT_D] = SDE_PORTD_HOTPLUG
  45 };
  46
  47 static const u32 hpd_cpt[] = {
  48         [HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
  49         [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
  50         [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
  51         [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
  52         [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
  53 };
  54
  55 static const u32 hpd_mask_i915[] = {
  56         [HPD_CRT] = CRT_HOTPLUG_INT_EN,
  57         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
  58         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
  59         [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
  60         [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
  61         [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
  62 };
  63
  64 static const u32 hpd_status_gen4[] = {
  65         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
  66         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
  67         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
  68         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
  69         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
  70         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
  71 };
  72
  73 static const u32 hpd_status_i915[] = { /* i915 and valleyview are the same */
  74         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
  75         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
  76         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
  77         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
  78         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
  79         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
  80 };
  81
  82 /* For display hotplug interrupt */
  83 static void
  84 ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
  85 {
  86         assert_spin_locked(&dev_priv->irq_lock);
  87
  88         if (dev_priv->pc8.irqs_disabled) {
  89                 WARN(1, "IRQs disabled\n");
  90                 dev_priv->pc8.regsave.deimr &= ~mask;
  91                 return;
  92         }
  93
  94         if ((dev_priv->irq_mask & mask) != 0) {
  95                 dev_priv->irq_mask &= ~mask;
  96                 I915_WRITE(DEIMR, dev_priv->irq_mask);
  97                 POSTING_READ(DEIMR);
  98         }
  99 }
 100
 101 static void
 102 ironlake_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
 103 {
 104         assert_spin_locked(&dev_priv->irq_lock);
 105
 106         if (dev_priv->pc8.irqs_disabled) {
 107                 WARN(1, "IRQs disabled\n");
 108                 dev_priv->pc8.regsave.deimr |= mask;
 109                 return;
 110         }
 111
 112         if ((dev_priv->irq_mask & mask) != mask) {
 113                 dev_priv->irq_mask |= mask;
 114                 I915_WRITE(DEIMR, dev_priv->irq_mask);
 115                 POSTING_READ(DEIMR);
 116         }
 117 }
 118
 119 /**
 120  * ilk_update_gt_irq - update GTIMR
 121  * @dev_priv: driver private
 122  * @interrupt_mask: mask of interrupt bits to update
 123  * @enabled_irq_mask: mask of interrupt bits to enable
 124  */
 125 static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
 126                               uint32_t interrupt_mask,
 127                               uint32_t enabled_irq_mask)
 128 {
 129         assert_spin_locked(&dev_priv->irq_lock);
 130
 131         if (dev_priv->pc8.irqs_disabled) {
 132                 WARN(1, "IRQs disabled\n");
 133                 dev_priv->pc8.regsave.gtimr &= ~interrupt_mask;
 134                 dev_priv->pc8.regsave.gtimr |= (~enabled_irq_mask &
 135                                                 interrupt_mask);
 136                 return;
 137         }
 138
 139         dev_priv->gt_irq_mask &= ~interrupt_mask;
 140         dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
 141         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 142         POSTING_READ(GTIMR);
 143 }
 144
 145 void ilk_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 146 {
 147         ilk_update_gt_irq(dev_priv, mask, mask);
 148 }
 149
 150 void ilk_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 151 {
 152         ilk_update_gt_irq(dev_priv, mask, 0);
 153 }
 154
 155 /**
 156   * snb_update_pm_irq - update GEN6_PMIMR
 157   * @dev_priv: driver private
 158   * @interrupt_mask: mask of interrupt bits to update
 159   * @enabled_irq_mask: mask of interrupt bits to enable
 160   */
 161 static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
 162                               uint32_t interrupt_mask,
 163                               uint32_t enabled_irq_mask)
 164 {
 165         uint32_t new_val;
 166
 167         assert_spin_locked(&dev_priv->irq_lock);
 168
 169         if (dev_priv->pc8.irqs_disabled) {
 170                 WARN(1, "IRQs disabled\n");
 171                 dev_priv->pc8.regsave.gen6_pmimr &= ~interrupt_mask;
 172                 dev_priv->pc8.regsave.gen6_pmimr |= (~enabled_irq_mask &
 173                                                      interrupt_mask);
 174                 return;
 175         }
 176
 177         new_val = dev_priv->pm_irq_mask;
 178         new_val &= ~interrupt_mask;
 179         new_val |= (~enabled_irq_mask & interrupt_mask);
 180
 181         if (new_val != dev_priv->pm_irq_mask) {
 182                 dev_priv->pm_irq_mask = new_val;
 183                 I915_WRITE(GEN6_PMIMR, dev_priv->pm_irq_mask);
 184                 POSTING_READ(GEN6_PMIMR);
 185         }
 186 }
 187
 188 void snb_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 189 {
 190         snb_update_pm_irq(dev_priv, mask, mask);
 191 }
 192
 193 void snb_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 194 {
 195         snb_update_pm_irq(dev_priv, mask, 0);
 196 }
 197
 198 static bool ivb_can_enable_err_int(struct drm_device *dev)
 199 {
 200         struct drm_i915_private *dev_priv = dev->dev_private;
 201         struct intel_crtc *crtc;
 202         enum pipe pipe;
 203
 204         assert_spin_locked(&dev_priv->irq_lock);
 205
 206         for_each_pipe(pipe) {
 207                 crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
 208
 209                 if (crtc->cpu_fifo_underrun_disabled)
 210                         return false;
 211         }
 212
 213         return true;
 214 }
 215
 216 static bool cpt_can_enable_serr_int(struct drm_device *dev)
 217 {
 218         struct drm_i915_private *dev_priv = dev->dev_private;
 219         enum pipe pipe;
 220         struct intel_crtc *crtc;
 221
 222         assert_spin_locked(&dev_priv->irq_lock);
 223
 224         for_each_pipe(pipe) {
 225                 crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
 226
 227                 if (crtc->pch_fifo_underrun_disabled)
 228                         return false;
 229         }
 230
 231         return true;
 232 }
 233
 234 static void ironlake_set_fifo_underrun_reporting(struct drm_device *dev,
 235                                                  enum pipe pipe, bool enable)
 236 {
 237         struct drm_i915_private *dev_priv = dev->dev_private;
 238         uint32_t bit = (pipe == PIPE_A) ? DE_PIPEA_FIFO_UNDERRUN :
 239                                           DE_PIPEB_FIFO_UNDERRUN;
 240
 241         if (enable)
 242                 ironlake_enable_display_irq(dev_priv, bit);
 243         else
 244                 ironlake_disable_display_irq(dev_priv, bit);
 245 }
 246
 247 static void ivybridge_set_fifo_underrun_reporting(struct drm_device *dev,
 248                                                   enum pipe pipe, bool enable)
 249 {
 250         struct drm_i915_private *dev_priv = dev->dev_private;
 251         if (enable) {
 252                 I915_WRITE(GEN7_ERR_INT, ERR_INT_FIFO_UNDERRUN(pipe));
 253
 254                 if (!ivb_can_enable_err_int(dev))
 255                         return;
 256
 257                 ironlake_enable_display_irq(dev_priv, DE_ERR_INT_IVB);
 258         } else {
 259                 bool was_enabled = !(I915_READ(DEIMR) & DE_ERR_INT_IVB);
 260
 261                 /* Change the state _after_ we've read out the current one. */
 262                 ironlake_disable_display_irq(dev_priv, DE_ERR_INT_IVB);
 263
 264                 if (!was_enabled &&
 265                     (I915_READ(GEN7_ERR_INT) & ERR_INT_FIFO_UNDERRUN(pipe))) {
 266                         DRM_DEBUG_KMS("uncleared fifo underrun on pipe %c\n",
 267                                       pipe_name(pipe));
 268                 }
 269         }
 270 }
 271
 272 /**
 273  * ibx_display_interrupt_update - update SDEIMR
 274  * @dev_priv: driver private
 275  * @interrupt_mask: mask of interrupt bits to update
 276  * @enabled_irq_mask: mask of interrupt bits to enable
 277  */
 278 static void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
 279                                          uint32_t interrupt_mask,
 280                                          uint32_t enabled_irq_mask)
 281 {
 282         uint32_t sdeimr = I915_READ(SDEIMR);
 283         sdeimr &= ~interrupt_mask;
 284         sdeimr |= (~enabled_irq_mask & interrupt_mask);
 285
 286         assert_spin_locked(&dev_priv->irq_lock);
 287
 288         if (dev_priv->pc8.irqs_disabled &&
 289             (interrupt_mask & SDE_HOTPLUG_MASK_CPT)) {
 290                 WARN(1, "IRQs disabled\n");
 291                 dev_priv->pc8.regsave.sdeimr &= ~interrupt_mask;
 292                 dev_priv->pc8.regsave.sdeimr |= (~enabled_irq_mask &
 293                                                  interrupt_mask);
 294                 return;
 295         }
 296
 297         I915_WRITE(SDEIMR, sdeimr);
 298         POSTING_READ(SDEIMR);
 299 }
 300 #define ibx_enable_display_interrupt(dev_priv, bits) \
 301         ibx_display_interrupt_update((dev_priv), (bits), (bits))
 302 #define ibx_disable_display_interrupt(dev_priv, bits) \
 303         ibx_display_interrupt_update((dev_priv), (bits), 0)
 304
 305 static void ibx_set_fifo_underrun_reporting(struct drm_device *dev,
 306                                             enum transcoder pch_transcoder,
 307                                             bool enable)
 308 {
 309         struct drm_i915_private *dev_priv = dev->dev_private;
 310         uint32_t bit = (pch_transcoder == TRANSCODER_A) ?
 311                        SDE_TRANSA_FIFO_UNDER : SDE_TRANSB_FIFO_UNDER;
 312
 313         if (enable)
 314                 ibx_enable_display_interrupt(dev_priv, bit);
 315         else
 316                 ibx_disable_display_interrupt(dev_priv, bit);
 317 }
 318
 319 static void cpt_set_fifo_underrun_reporting(struct drm_device *dev,
 320                                             enum transcoder pch_transcoder,
 321                                             bool enable)
 322 {
 323         struct drm_i915_private *dev_priv = dev->dev_private;
 324
 325         if (enable) {
 326                 I915_WRITE(SERR_INT,
 327                            SERR_INT_TRANS_FIFO_UNDERRUN(pch_transcoder));
 328
 329                 if (!cpt_can_enable_serr_int(dev))
 330                         return;
 331
 332                 ibx_enable_display_interrupt(dev_priv, SDE_ERROR_CPT);
 333         } else {
 334                 uint32_t tmp = I915_READ(SERR_INT);
 335                 bool was_enabled = !(I915_READ(SDEIMR) & SDE_ERROR_CPT);
 336
 337                 /* Change the state _after_ we've read out the current one. */
 338                 ibx_disable_display_interrupt(dev_priv, SDE_ERROR_CPT);
 339
 340                 if (!was_enabled &&
 341                     (tmp & SERR_INT_TRANS_FIFO_UNDERRUN(pch_transcoder))) {
 342                         DRM_DEBUG_KMS("uncleared pch fifo underrun on pch transcoder %c\n",
 343                                       transcoder_name(pch_transcoder));
 344                 }
 345         }
 346 }
 347
 348 /**
 349  * intel_set_cpu_fifo_underrun_reporting - enable/disable FIFO underrun messages
 350  * @dev: drm device
 351  * @pipe: pipe
 352  * @enable: true if we want to report FIFO underrun errors, false otherwise
 353  *
 354  * This function makes us disable or enable CPU fifo underruns for a specific
 355  * pipe. Notice that on some Gens (e.g. IVB, HSW), disabling FIFO underrun
 356  * reporting for one pipe may also disable all the other CPU error interruts for
 357  * the other pipes, due to the fact that there's just one interrupt mask/enable
 358  * bit for all the pipes.
 359  *
 360  * Returns the previous state of underrun reporting.
 361  */
 362 bool intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
 363                                            enum pipe pipe, bool enable)
 364 {
 365         struct drm_i915_private *dev_priv = dev->dev_private;
 366         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
 367         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 368         unsigned long flags;
 369         bool ret;
 370
 371         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 372
 373         ret = !intel_crtc->cpu_fifo_underrun_disabled;
 374
 375         if (enable == ret)
 376                 goto done;
 377
 378         intel_crtc->cpu_fifo_underrun_disabled = !enable;
 379
 380         if (IS_GEN5(dev) || IS_GEN6(dev))
 381                 ironlake_set_fifo_underrun_reporting(dev, pipe, enable);
 382         else if (IS_GEN7(dev))
 383                 ivybridge_set_fifo_underrun_reporting(dev, pipe, enable);
 384
 385 done:
 386         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 387         return ret;
 388 }
 389
 390 /**
 391  * intel_set_pch_fifo_underrun_reporting - enable/disable FIFO underrun messages
 392  * @dev: drm device
 393  * @pch_transcoder: the PCH transcoder (same as pipe on IVB and older)
 394  * @enable: true if we want to report FIFO underrun errors, false otherwise
 395  *
 396  * This function makes us disable or enable PCH fifo underruns for a specific
 397  * PCH transcoder. Notice that on some PCHs (e.g. CPT/PPT), disabling FIFO
 398  * underrun reporting for one transcoder may also disable all the other PCH
 399  * error interruts for the other transcoders, due to the fact that there's just
 400  * one interrupt mask/enable bit for all the transcoders.
 401  *
 402  * Returns the previous state of underrun reporting.
 403  */
 404 bool intel_set_pch_fifo_underrun_reporting(struct drm_device *dev,
 405                                            enum transcoder pch_transcoder,
 406                                            bool enable)
 407 {
 408         struct drm_i915_private *dev_priv = dev->dev_private;
 409         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pch_transcoder];
 410         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 411         unsigned long flags;
 412         bool ret;
 413
 414         /*
 415          * NOTE: Pre-LPT has a fixed cpu pipe -> pch transcoder mapping, but LPT
 416          * has only one pch transcoder A that all pipes can use. To avoid racy
 417          * pch transcoder -> pipe lookups from interrupt code simply store the
 418          * underrun statistics in crtc A. Since we never expose this anywhere
 419          * nor use it outside of the fifo underrun code here using the "wrong"
 420          * crtc on LPT won't cause issues.
 421          */
 422
 423         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 424
 425         ret = !intel_crtc->pch_fifo_underrun_disabled;
 426
 427         if (enable == ret)
 428                 goto done;
 429
 430         intel_crtc->pch_fifo_underrun_disabled = !enable;
 431
 432         if (HAS_PCH_IBX(dev))
 433                 ibx_set_fifo_underrun_reporting(dev, pch_transcoder, enable);
 434         else
 435                 cpt_set_fifo_underrun_reporting(dev, pch_transcoder, enable);
 436
 437 done:
 438         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 439         return ret;
 440 }
 441
 442
 443 void
 444 i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
 445 {
 446         u32 reg = PIPESTAT(pipe);
 447         u32 pipestat = I915_READ(reg) & 0x7fff0000;
 448
 449         assert_spin_locked(&dev_priv->irq_lock);
 450
 451         if ((pipestat & mask) == mask)
 452                 return;
 453
 454         /* Enable the interrupt, clear any pending status */
 455         pipestat |= mask | (mask >> 16);
 456         I915_WRITE(reg, pipestat);
 457         POSTING_READ(reg);
 458 }
 459
 460 void
 461 i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
 462 {
 463         u32 reg = PIPESTAT(pipe);
 464         u32 pipestat = I915_READ(reg) & 0x7fff0000;
 465
 466         assert_spin_locked(&dev_priv->irq_lock);
 467
 468         if ((pipestat & mask) == 0)
 469                 return;
 470
 471         pipestat &= ~mask;
 472         I915_WRITE(reg, pipestat);
 473         POSTING_READ(reg);
 474 }
 475
 476 /**
 477  * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
 478  */
 479 static void i915_enable_asle_pipestat(struct drm_device *dev)
 480 {
 481         drm_i915_private_t *dev_priv = dev->dev_private;
 482         unsigned long irqflags;
 483
 484         if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
 485                 return;
 486
 487         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
 488
 489         i915_enable_pipestat(dev_priv, 1, PIPE_LEGACY_BLC_EVENT_ENABLE);
 490         if (INTEL_INFO(dev)->gen >= 4)
 491                 i915_enable_pipestat(dev_priv, 0, PIPE_LEGACY_BLC_EVENT_ENABLE);
 492
 493         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 494 }
 495
 496 /**
 497  * i915_pipe_enabled - check if a pipe is enabled
 498  * @dev: DRM device
 499  * @pipe: pipe to check
 500  *
 501  * Reading certain registers when the pipe is disabled can hang the chip.
 502  * Use this routine to make sure the PLL is running and the pipe is active
 503  * before reading such registers if unsure.
 504  */
 505 static int
 506 i915_pipe_enabled(struct drm_device *dev, int pipe)
 507 {
 508         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 509
 510         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
 511                 /* Locking is horribly broken here, but whatever. */
 512                 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
 513                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 514
 515                 return intel_crtc->active;
 516         } else {
 517                 return I915_READ(PIPECONF(pipe)) & PIPECONF_ENABLE;
 518         }
 519 }
 520
 521 /* Called from drm generic code, passed a 'crtc', which
 522  * we use as a pipe index
 523  */
 524 static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
 525 {
 526         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 527         unsigned long high_frame;
 528         unsigned long low_frame;
 529         u32 high1, high2, low;
 530
 531         if (!i915_pipe_enabled(dev, pipe)) {
 532                 DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
 533                                 "pipe %c\n", pipe_name(pipe));
 534                 return 0;
 535         }
 536
 537         high_frame = PIPEFRAME(pipe);
 538         low_frame = PIPEFRAMEPIXEL(pipe);
 539
 540         /*
 541          * High & low register fields aren't synchronized, so make sure
 542          * we get a low value that's stable across two reads of the high
 543          * register.
 544          */
 545         do {
 546                 high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
 547                 low   = I915_READ(low_frame)  & PIPE_FRAME_LOW_MASK;
 548                 high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
 549         } while (high1 != high2);
 550
 551         high1 >>= PIPE_FRAME_HIGH_SHIFT;
 552         low >>= PIPE_FRAME_LOW_SHIFT;
 553         return (high1 << 8) | low;
 554 }
 555
 556 static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
 557 {
 558         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 559         int reg = PIPE_FRMCOUNT_GM45(pipe);
 560
 561         if (!i915_pipe_enabled(dev, pipe)) {
 562                 DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
 563                                  "pipe %c\n", pipe_name(pipe));
 564                 return 0;
 565         }
 566
 567         return I915_READ(reg);
 568 }
 569
 570 static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
 571                              int *vpos, int *hpos)
 572 {
 573         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 574         u32 vbl = 0, position = 0;
 575         int vbl_start, vbl_end, htotal, vtotal;
 576         bool in_vbl = true;
 577         int ret = 0;
 578         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
 579                                                                       pipe);
 580
 581         if (!i915_pipe_enabled(dev, pipe)) {
 582                 DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
 583                                  "pipe %c\n", pipe_name(pipe));
 584                 return 0;
 585         }
 586
 587         /* Get vtotal. */
 588         vtotal = 1 + ((I915_READ(VTOTAL(cpu_transcoder)) >> 16) & 0x1fff);
 589
 590         if (INTEL_INFO(dev)->gen >= 4) {
 591                 /* No obvious pixelcount register. Only query vertical
 592                  * scanout position from Display scan line register.
 593                  */
 594                 position = I915_READ(PIPEDSL(pipe));
 595
 596                 /* Decode into vertical scanout position. Don't have
 597                  * horizontal scanout position.
 598                  */
 599                 *vpos = position & 0x1fff;
 600                 *hpos = 0;
 601         } else {
 602                 /* Have access to pixelcount since start of frame.
 603                  * We can split this into vertical and horizontal
 604                  * scanout position.
 605                  */
 606                 position = (I915_READ(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
 607
 608                 htotal = 1 + ((I915_READ(HTOTAL(cpu_transcoder)) >> 16) & 0x1fff);
 609                 *vpos = position / htotal;
 610                 *hpos = position - (*vpos * htotal);
 611         }
 612
 613         /* Query vblank area. */
 614         vbl = I915_READ(VBLANK(cpu_transcoder));
 615
 616         /* Test position against vblank region. */
 617         vbl_start = vbl & 0x1fff;
 618         vbl_end = (vbl >> 16) & 0x1fff;
 619
 620         if ((*vpos < vbl_start) || (*vpos > vbl_end))
 621                 in_vbl = false;
 622
 623         /* Inside "upper part" of vblank area? Apply corrective offset: */
 624         if (in_vbl && (*vpos >= vbl_start))
 625                 *vpos = *vpos - vtotal;
 626
 627         /* Readouts valid? */
 628         if (vbl > 0)
 629                 ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
 630
 631         /* In vblank? */
 632         if (in_vbl)
 633                 ret |= DRM_SCANOUTPOS_INVBL;
 634
 635         return ret;
 636 }
 637
 638 static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
 639                               int *max_error,
 640                               struct timeval *vblank_time,
 641                               unsigned flags)
 642 {
 643         struct drm_crtc *crtc;
 644
 645         if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) {
 646                 DRM_ERROR("Invalid crtc %d\n", pipe);
 647                 return -EINVAL;
 648         }
 649
 650         /* Get drm_crtc to timestamp: */
 651         crtc = intel_get_crtc_for_pipe(dev, pipe);
 652         if (crtc == NULL) {
 653                 DRM_ERROR("Invalid crtc %d\n", pipe);
 654                 return -EINVAL;
 655         }
 656
 657         if (!crtc->enabled) {
 658                 DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
 659                 return -EBUSY;
 660         }
 661
 662         /* Helper routine in DRM core does all the work: */
 663         return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
 664                                                      vblank_time, flags,
 665                                                      crtc);
 666 }
 667
 668 static int intel_hpd_irq_event(struct drm_device *dev, struct drm_connector *connector)
 669 {
 670         enum drm_connector_status old_status;
 671
 672         WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
 673         old_status = connector->status;
 674
 675         connector->status = connector->funcs->detect(connector, false);
 676         DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %d to %d\n",
 677                       connector->base.id,
 678                       drm_get_connector_name(connector),
 679                       old_status, connector->status);
 680         return (old_status != connector->status);
 681 }
 682
 683 /*
 684  * Handle hotplug events outside the interrupt handler proper.
 685  */
 686 #define I915_REENABLE_HOTPLUG_DELAY (2*60*1000)
 687
 688 static void i915_hotplug_work_func(struct work_struct *work)
 689 {
 690         drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
 691                                                     hotplug_work);
 692         struct drm_device *dev = dev_priv->dev;
 693         struct drm_mode_config *mode_config = &dev->mode_config;
 694         struct intel_connector *intel_connector;
 695         struct intel_encoder *intel_encoder;
 696         struct drm_connector *connector;
 697         unsigned long irqflags;
 698         bool hpd_disabled = false;
 699         bool changed = false;
 700         u32 hpd_event_bits;
 701
 702         /* HPD irq before everything is fully set up. */
 703         if (!dev_priv->enable_hotplug_processing)
 704                 return;
 705
 706         mutex_lock(&mode_config->mutex);
 707         DRM_DEBUG_KMS("running encoder hotplug functions\n");
 708
 709         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
 710
 711         hpd_event_bits = dev_priv->hpd_event_bits;
 712         dev_priv->hpd_event_bits = 0;
 713         list_for_each_entry(connector, &mode_config->connector_list, head) {
 714                 intel_connector = to_intel_connector(connector);
 715                 intel_encoder = intel_connector->encoder;
 716                 if (intel_encoder->hpd_pin > HPD_NONE &&
 717                     dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_MARK_DISABLED &&
 718                     connector->polled == DRM_CONNECTOR_POLL_HPD) {
 719                         DRM_INFO("HPD interrupt storm detected on connector %s: "
 720                                  "switching from hotplug detection to polling\n",
 721                                 drm_get_connector_name(connector));
 722                         dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark = HPD_DISABLED;
 723                         connector->polled = DRM_CONNECTOR_POLL_CONNECT
 724                                 | DRM_CONNECTOR_POLL_DISCONNECT;
 725                         hpd_disabled = true;
 726                 }
 727                 if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
 728                         DRM_DEBUG_KMS("Connector %s (pin %i) received hotplug event.\n",
 729                                       drm_get_connector_name(connector), intel_encoder->hpd_pin);
 730                 }
 731         }
 732          /* if there were no outputs to poll, poll was disabled,
 733           * therefore make sure it's enabled when disabling HPD on
 734           * some connectors */
 735         if (hpd_disabled) {
 736                 drm_kms_helper_poll_enable(dev);
 737                 mod_timer(&dev_priv->hotplug_reenable_timer,
 738                           jiffies + msecs_to_jiffies(I915_REENABLE_HOTPLUG_DELAY));
 739         }
 740
 741         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 742
 743         list_for_each_entry(connector, &mode_config->connector_list, head) {
 744                 intel_connector = to_intel_connector(connector);
 745                 intel_encoder = intel_connector->encoder;
 746                 if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
 747                         if (intel_encoder->hot_plug)
 748                                 intel_encoder->hot_plug(intel_encoder);
 749                         if (intel_hpd_irq_event(dev, connector))
 750                                 changed = true;
 751                 }
 752         }
 753         mutex_unlock(&mode_config->mutex);
 754
 755         if (changed)
 756                 drm_kms_helper_hotplug_event(dev);
 757 }
 758
 759 static void ironlake_rps_change_irq_handler(struct drm_device *dev)
 760 {
 761         drm_i915_private_t *dev_priv = dev->dev_private;
 762         u32 busy_up, busy_down, max_avg, min_avg;
 763         u8 new_delay;
 764
 765         spin_lock(&mchdev_lock);
 766
 767         I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
 768
 769         new_delay = dev_priv->ips.cur_delay;
 770
 771         I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
 772         busy_up = I915_READ(RCPREVBSYTUPAVG);
 773         busy_down = I915_READ(RCPREVBSYTDNAVG);
 774         max_avg = I915_READ(RCBMAXAVG);
 775         min_avg = I915_READ(RCBMINAVG);
 776
 777         /* Handle RCS change request from hw */
 778         if (busy_up > max_avg) {
 779                 if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
 780                         new_delay = dev_priv->ips.cur_delay - 1;
 781                 if (new_delay < dev_priv->ips.max_delay)
 782                         new_delay = dev_priv->ips.max_delay;
 783         } else if (busy_down < min_avg) {
 784                 if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
 785                         new_delay = dev_priv->ips.cur_delay + 1;
 786                 if (new_delay > dev_priv->ips.min_delay)
 787                         new_delay = dev_priv->ips.min_delay;
 788         }
 789
 790         if (ironlake_set_drps(dev, new_delay))
 791                 dev_priv->ips.cur_delay = new_delay;
 792
 793         spin_unlock(&mchdev_lock);
 794
 795         return;
 796 }
 797
 798 static void notify_ring(struct drm_device *dev,
 799                         struct intel_ring_buffer *ring)
 800 {
 801         if (ring->obj == NULL)
 802                 return;
 803
 804         trace_i915_gem_request_complete(ring, ring->get_seqno(ring, false));
 805
 806         wake_up_all(&ring->irq_queue);
 807         i915_queue_hangcheck(dev);
 808 }
 809
 810 static void gen6_pm_rps_work(struct work_struct *work)
 811 {
 812         drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
 813                                                     rps.work);
 814         u32 pm_iir;
 815         u8 new_delay;
 816
 817         spin_lock_irq(&dev_priv->irq_lock);
 818         pm_iir = dev_priv->rps.pm_iir;
 819         dev_priv->rps.pm_iir = 0;
 820         /* Make sure not to corrupt PMIMR state used by ringbuffer code */
 821         snb_enable_pm_irq(dev_priv, GEN6_PM_RPS_EVENTS);
 822         spin_unlock_irq(&dev_priv->irq_lock);
 823
 824         /* Make sure we didn't queue anything we're not going to process. */
 825         WARN_ON(pm_iir & ~GEN6_PM_RPS_EVENTS);
 826
 827         if ((pm_iir & GEN6_PM_RPS_EVENTS) == 0)
 828                 return;
 829
 830         mutex_lock(&dev_priv->rps.hw_lock);
 831
 832         if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
 833                 new_delay = dev_priv->rps.cur_delay + 1;
 834
 835                 /*
 836                  * For better performance, jump directly
 837                  * to RPe if we're below it.
 838                  */
 839                 if (IS_VALLEYVIEW(dev_priv->dev) &&
 840                     dev_priv->rps.cur_delay < dev_priv->rps.rpe_delay)
 841                         new_delay = dev_priv->rps.rpe_delay;
 842         } else
 843                 new_delay = dev_priv->rps.cur_delay - 1;
 844
 845         /* sysfs frequency interfaces may have snuck in while servicing the
 846          * interrupt
 847          */
 848         if (new_delay >= dev_priv->rps.min_delay &&
 849             new_delay <= dev_priv->rps.max_delay) {
 850                 if (IS_VALLEYVIEW(dev_priv->dev))
 851                         valleyview_set_rps(dev_priv->dev, new_delay);
 852                 else
 853                         gen6_set_rps(dev_priv->dev, new_delay);
 854         }
 855
 856         if (IS_VALLEYVIEW(dev_priv->dev)) {
 857                 /*
 858                  * On VLV, when we enter RC6 we may not be at the minimum
 859                  * voltage level, so arm a timer to check.  It should only
 860                  * fire when there's activity or once after we've entered
 861                  * RC6, and then won't be re-armed until the next RPS interrupt.
 862                  */
 863                 mod_delayed_work(dev_priv->wq, &dev_priv->rps.vlv_work,
 864                                  msecs_to_jiffies(100));
 865         }
 866
 867         mutex_unlock(&dev_priv->rps.hw_lock);
 868 }
 869
 870
 871 /**
 872  * ivybridge_parity_work - Workqueue called when a parity error interrupt
 873  * occurred.
 874  * @work: workqueue struct
 875  *
 876  * Doesn't actually do anything except notify userspace. As a consequence of
 877  * this event, userspace should try to remap the bad rows since statistically
 878  * it is likely the same row is more likely to go bad again.
 879  */
 880 static void ivybridge_parity_work(struct work_struct *work)
 881 {
 882         drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
 883                                                     l3_parity.error_work);
 884         u32 error_status, row, bank, subbank;
 885         char *parity_event[5];
 886         uint32_t misccpctl;
 887         unsigned long flags;
 888
 889         /* We must turn off DOP level clock gating to access the L3 registers.
 890          * In order to prevent a get/put style interface, acquire struct mutex
 891          * any time we access those registers.
 892          */
 893         mutex_lock(&dev_priv->dev->struct_mutex);
 894
 895         misccpctl = I915_READ(GEN7_MISCCPCTL);
 896         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
 897         POSTING_READ(GEN7_MISCCPCTL);
 898
 899         error_status = I915_READ(GEN7_L3CDERRST1);
 900         row = GEN7_PARITY_ERROR_ROW(error_status);
 901         bank = GEN7_PARITY_ERROR_BANK(error_status);
 902         subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
 903
 904         I915_WRITE(GEN7_L3CDERRST1, GEN7_PARITY_ERROR_VALID |
 905                                     GEN7_L3CDERRST1_ENABLE);
 906         POSTING_READ(GEN7_L3CDERRST1);
 907
 908         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
 909
 910         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 911         ilk_enable_gt_irq(dev_priv, GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
 912         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 913
 914         mutex_unlock(&dev_priv->dev->struct_mutex);
 915
 916         parity_event[0] = I915_L3_PARITY_UEVENT "=1";
 917         parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
 918         parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
 919         parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
 920         parity_event[4] = NULL;
 921
 922         kobject_uevent_env(&dev_priv->dev->primary->kdev.kobj,
 923                            KOBJ_CHANGE, parity_event);
 924
 925         DRM_DEBUG("Parity error: Row = %d, Bank = %d, Sub bank = %d.\n",
 926                   row, bank, subbank);
 927
 928         kfree(parity_event[3]);
 929         kfree(parity_event[2]);
 930         kfree(parity_event[1]);
 931 }
 932
 933 static void ivybridge_parity_error_irq_handler(struct drm_device *dev)
 934 {
 935         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 936
 937         if (!HAS_L3_GPU_CACHE(dev))
 938                 return;
 939
 940         spin_lock(&dev_priv->irq_lock);
 941         ilk_disable_gt_irq(dev_priv, GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
 942         spin_unlock(&dev_priv->irq_lock);
 943
 944         queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
 945 }
 946
 947 static void ilk_gt_irq_handler(struct drm_device *dev,
 948                                struct drm_i915_private *dev_priv,
 949                                u32 gt_iir)
 950 {
 951         if (gt_iir &
 952             (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
 953                 notify_ring(dev, &dev_priv->ring[RCS]);
 954         if (gt_iir & ILK_BSD_USER_INTERRUPT)
 955                 notify_ring(dev, &dev_priv->ring[VCS]);
 956 }
 957
 958 static void snb_gt_irq_handler(struct drm_device *dev,
 959                                struct drm_i915_private *dev_priv,
 960                                u32 gt_iir)
 961 {
 962
 963         if (gt_iir &
 964             (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
 965                 notify_ring(dev, &dev_priv->ring[RCS]);
 966         if (gt_iir & GT_BSD_USER_INTERRUPT)
 967                 notify_ring(dev, &dev_priv->ring[VCS]);
 968         if (gt_iir & GT_BLT_USER_INTERRUPT)
 969                 notify_ring(dev, &dev_priv->ring[BCS]);
 970
 971         if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
 972                       GT_BSD_CS_ERROR_INTERRUPT |
 973                       GT_RENDER_CS_MASTER_ERROR_INTERRUPT)) {
 974                 DRM_ERROR("GT error interrupt 0x%08x\n", gt_iir);
 975                 i915_handle_error(dev, false);
 976         }
 977
 978         if (gt_iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
 979                 ivybridge_parity_error_irq_handler(dev);
 980 }
 981
 982 #define HPD_STORM_DETECT_PERIOD 1000
 983 #define HPD_STORM_THRESHOLD 5
 984
 985 static inline void intel_hpd_irq_handler(struct drm_device *dev,
 986                                          u32 hotplug_trigger,
 987                                          const u32 *hpd)
 988 {
 989         drm_i915_private_t *dev_priv = dev->dev_private;
 990         int i;
 991         bool storm_detected = false;
 992
 993         if (!hotplug_trigger)
 994                 return;
 995
 996         spin_lock(&dev_priv->irq_lock);
 997         for (i = 1; i < HPD_NUM_PINS; i++) {
 998
 999                 WARN(((hpd[i] & hotplug_trigger) &&
1000                       dev_priv->hpd_stats[i].hpd_mark != HPD_ENABLED),
1001                      "Received HPD interrupt although disabled\n");
1002
1003                 if (!(hpd[i] & hotplug_trigger) ||
1004                     dev_priv->hpd_stats[i].hpd_mark != HPD_ENABLED)
1005                         continue;
1006
1007                 dev_priv->hpd_event_bits |= (1 << i);
1008                 if (!time_in_range(jiffies, dev_priv->hpd_stats[i].hpd_last_jiffies,
1009                                    dev_priv->hpd_stats[i].hpd_last_jiffies
1010                                    + msecs_to_jiffies(HPD_STORM_DETECT_PERIOD))) {
1011                         dev_priv->hpd_stats[i].hpd_last_jiffies = jiffies;
1012                         dev_priv->hpd_stats[i].hpd_cnt = 0;
1013                         DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: 0\n", i);
1014                 } else if (dev_priv->hpd_stats[i].hpd_cnt > HPD_STORM_THRESHOLD) {
1015                         dev_priv->hpd_stats[i].hpd_mark = HPD_MARK_DISABLED;
1016                         dev_priv->hpd_event_bits &= ~(1 << i);
1017                         DRM_DEBUG_KMS("HPD interrupt storm detected on PIN %d\n", i);
1018                         storm_detected = true;
1019                 } else {
1020                         dev_priv->hpd_stats[i].hpd_cnt++;
1021                         DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: %d\n", i,
1022                                       dev_priv->hpd_stats[i].hpd_cnt);
1023                 }
1024         }
1025
1026         if (storm_detected)
1027                 dev_priv->display.hpd_irq_setup(dev);
1028         spin_unlock(&dev_priv->irq_lock);
1029
1030         /*
1031          * Our hotplug handler can grab modeset locks (by calling down into the
1032          * fb helpers). Hence it must not be run on our own dev-priv->wq work
1033          * queue for otherwise the flush_work in the pageflip code will
1034          * deadlock.
1035          */
1036         schedule_work(&dev_priv->hotplug_work);
1037 }
1038
1039 static void gmbus_irq_handler(struct drm_device *dev)
1040 {
1041         struct drm_i915_private *dev_priv = (drm_i915_private_t *) dev->dev_private;
1042
1043         wake_up_all(&dev_priv->gmbus_wait_queue);
1044 }
1045
1046 static void dp_aux_irq_handler(struct drm_device *dev)
1047 {
1048         struct drm_i915_private *dev_priv = (drm_i915_private_t *) dev->dev_private;
1049
1050         wake_up_all(&dev_priv->gmbus_wait_queue);
1051 }
1052
1053 /* The RPS events need forcewake, so we add them to a work queue and mask their
1054  * IMR bits until the work is done. Other interrupts can be processed without
1055  * the work queue. */
1056 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
1057 {
1058         if (pm_iir & GEN6_PM_RPS_EVENTS) {
1059                 spin_lock(&dev_priv->irq_lock);
1060                 dev_priv->rps.pm_iir |= pm_iir & GEN6_PM_RPS_EVENTS;
1061                 snb_disable_pm_irq(dev_priv, pm_iir & GEN6_PM_RPS_EVENTS);
1062                 spin_unlock(&dev_priv->irq_lock);
1063
1064                 queue_work(dev_priv->wq, &dev_priv->rps.work);
1065         }
1066
1067         if (HAS_VEBOX(dev_priv->dev)) {
1068                 if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1069                         notify_ring(dev_priv->dev, &dev_priv->ring[VECS]);
1070
1071                 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) {
1072                         DRM_ERROR("VEBOX CS error interrupt 0x%08x\n", pm_iir);
1073                         i915_handle_error(dev_priv->dev, false);
1074                 }
1075         }
1076 }
1077
1078 static irqreturn_t valleyview_irq_handler(int irq, void *arg)
1079 {
1080         struct drm_device *dev = (struct drm_device *) arg;
1081         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1082         u32 iir, gt_iir, pm_iir;
1083         irqreturn_t ret = IRQ_NONE;
1084         unsigned long irqflags;
1085         int pipe;
1086         u32 pipe_stats[I915_MAX_PIPES];
1087
1088         atomic_inc(&dev_priv->irq_received);
1089
1090         while (true) {
1091                 iir = I915_READ(VLV_IIR);
1092                 gt_iir = I915_READ(GTIIR);
1093                 pm_iir = I915_READ(GEN6_PMIIR);
1094
1095                 if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1096                         goto out;
1097
1098                 ret = IRQ_HANDLED;
1099
1100                 snb_gt_irq_handler(dev, dev_priv, gt_iir);
1101
1102                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1103                 for_each_pipe(pipe) {
1104                         int reg = PIPESTAT(pipe);
1105                         pipe_stats[pipe] = I915_READ(reg);
1106
1107                         /*
1108                          * Clear the PIPE*STAT regs before the IIR
1109                          */
1110                         if (pipe_stats[pipe] & 0x8000ffff) {
1111                                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1112                                         DRM_DEBUG_DRIVER("pipe %c underrun\n",
1113                                                          pipe_name(pipe));
1114                                 I915_WRITE(reg, pipe_stats[pipe]);
1115                         }
1116                 }
1117                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1118
1119                 for_each_pipe(pipe) {
1120                         if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
1121                                 drm_handle_vblank(dev, pipe);
1122
1123                         if (pipe_stats[pipe] & PLANE_FLIPDONE_INT_STATUS_VLV) {
1124                                 intel_prepare_page_flip(dev, pipe);
1125                                 intel_finish_page_flip(dev, pipe);
1126                         }
1127                 }
1128
1129                 /* Consume port.  Then clear IIR or we'll miss events */
1130                 if (iir & I915_DISPLAY_PORT_INTERRUPT) {
1131                         u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
1132                         u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1133
1134                         DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
1135                                          hotplug_status);
1136
1137                         intel_hpd_irq_handler(dev, hotplug_trigger, hpd_status_i915);
1138
1139                         I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1140                         I915_READ(PORT_HOTPLUG_STAT);
1141                 }
1142
1143                 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1144                         gmbus_irq_handler(dev);
1145
1146                 if (pm_iir)
1147                         gen6_rps_irq_handler(dev_priv, pm_iir);
1148
1149                 I915_WRITE(GTIIR, gt_iir);
1150                 I915_WRITE(GEN6_PMIIR, pm_iir);
1151                 I915_WRITE(VLV_IIR, iir);
1152         }
1153
1154 out:
1155         return ret;
1156 }
1157
1158 static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
1159 {
1160         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1161         int pipe;
1162         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1163
1164         intel_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);
1165
1166         if (pch_iir & SDE_AUDIO_POWER_MASK) {
1167                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
1168                                SDE_AUDIO_POWER_SHIFT);
1169                 DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
1170                                  port_name(port));
1171         }
1172
1173         if (pch_iir & SDE_AUX_MASK)
1174                 dp_aux_irq_handler(dev);
1175
1176         if (pch_iir & SDE_GMBUS)
1177                 gmbus_irq_handler(dev);
1178
1179         if (pch_iir & SDE_AUDIO_HDCP_MASK)
1180                 DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
1181
1182         if (pch_iir & SDE_AUDIO_TRANS_MASK)
1183                 DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
1184
1185         if (pch_iir & SDE_POISON)
1186                 DRM_ERROR("PCH poison interrupt\n");
1187
1188         if (pch_iir & SDE_FDI_MASK)
1189                 for_each_pipe(pipe)
1190                         DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
1191                                          pipe_name(pipe),
1192                                          I915_READ(FDI_RX_IIR(pipe)));
1193
1194         if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
1195                 DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
1196
1197         if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
1198                 DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
1199
1200         if (pch_iir & SDE_TRANSA_FIFO_UNDER)
1201                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A,
1202                                                           false))
1203                         DRM_DEBUG_DRIVER("PCH transcoder A FIFO underrun\n");
1204
1205         if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1206                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B,
1207                                                           false))
1208                         DRM_DEBUG_DRIVER("PCH transcoder B FIFO underrun\n");
1209 }
1210
1211 static void ivb_err_int_handler(struct drm_device *dev)
1212 {
1213         struct drm_i915_private *dev_priv = dev->dev_private;
1214         u32 err_int = I915_READ(GEN7_ERR_INT);
1215
1216         if (err_int & ERR_INT_POISON)
1217                 DRM_ERROR("Poison interrupt\n");
1218
1219         if (err_int & ERR_INT_FIFO_UNDERRUN_A)
1220                 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_A, false))
1221                         DRM_DEBUG_DRIVER("Pipe A FIFO underrun\n");
1222
1223         if (err_int & ERR_INT_FIFO_UNDERRUN_B)
1224                 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_B, false))
1225                         DRM_DEBUG_DRIVER("Pipe B FIFO underrun\n");
1226
1227         if (err_int & ERR_INT_FIFO_UNDERRUN_C)
1228                 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_C, false))
1229                         DRM_DEBUG_DRIVER("Pipe C FIFO underrun\n");
1230
1231         I915_WRITE(GEN7_ERR_INT, err_int);
1232 }
1233
1234 static void cpt_serr_int_handler(struct drm_device *dev)
1235 {
1236         struct drm_i915_private *dev_priv = dev->dev_private;
1237         u32 serr_int = I915_READ(SERR_INT);
1238
1239         if (serr_int & SERR_INT_POISON)
1240                 DRM_ERROR("PCH poison interrupt\n");
1241
1242         if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
1243                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A,
1244                                                           false))
1245                         DRM_DEBUG_DRIVER("PCH transcoder A FIFO underrun\n");
1246
1247         if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
1248                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B,
1249                                                           false))
1250                         DRM_DEBUG_DRIVER("PCH transcoder B FIFO underrun\n");
1251
1252         if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
1253                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_C,
1254                                                           false))
1255                         DRM_DEBUG_DRIVER("PCH transcoder C FIFO underrun\n");
1256
1257         I915_WRITE(SERR_INT, serr_int);
1258 }
1259
1260 static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
1261 {
1262         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1263         int pipe;
1264         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
1265
1266         intel_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);
1267
1268         if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
1269                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
1270                                SDE_AUDIO_POWER_SHIFT_CPT);
1271                 DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
1272                                  port_name(port));
1273         }
1274
1275         if (pch_iir & SDE_AUX_MASK_CPT)
1276                 dp_aux_irq_handler(dev);
1277
1278         if (pch_iir & SDE_GMBUS_CPT)
1279                 gmbus_irq_handler(dev);
1280
1281         if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
1282                 DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
1283
1284         if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
1285                 DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
1286
1287         if (pch_iir & SDE_FDI_MASK_CPT)
1288                 for_each_pipe(pipe)
1289                         DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
1290                                          pipe_name(pipe),
1291                                          I915_READ(FDI_RX_IIR(pipe)));
1292
1293         if (pch_iir & SDE_ERROR_CPT)
1294                 cpt_serr_int_handler(dev);
1295 }
1296
1297 static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
1298 {
1299         struct drm_i915_private *dev_priv = dev->dev_private;
1300
1301         if (de_iir & DE_AUX_CHANNEL_A)
1302                 dp_aux_irq_handler(dev);
1303
1304         if (de_iir & DE_GSE)
1305                 intel_opregion_asle_intr(dev);
1306
1307         if (de_iir & DE_PIPEA_VBLANK)
1308                 drm_handle_vblank(dev, 0);
1309
1310         if (de_iir & DE_PIPEB_VBLANK)
1311                 drm_handle_vblank(dev, 1);
1312
1313         if (de_iir & DE_POISON)
1314                 DRM_ERROR("Poison interrupt\n");
1315
1316         if (de_iir & DE_PIPEA_FIFO_UNDERRUN)
1317                 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_A, false))
1318                         DRM_DEBUG_DRIVER("Pipe A FIFO underrun\n");
1319
1320         if (de_iir & DE_PIPEB_FIFO_UNDERRUN)
1321                 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_B, false))
1322                         DRM_DEBUG_DRIVER("Pipe B FIFO underrun\n");
1323
1324         if (de_iir & DE_PLANEA_FLIP_DONE) {
1325                 intel_prepare_page_flip(dev, 0);
1326                 intel_finish_page_flip_plane(dev, 0);
1327         }
1328
1329         if (de_iir & DE_PLANEB_FLIP_DONE) {
1330                 intel_prepare_page_flip(dev, 1);
1331                 intel_finish_page_flip_plane(dev, 1);
1332         }
1333
1334         /* check event from PCH */
1335         if (de_iir & DE_PCH_EVENT) {
1336                 u32 pch_iir = I915_READ(SDEIIR);
1337
1338                 if (HAS_PCH_CPT(dev))
1339                         cpt_irq_handler(dev, pch_iir);
1340                 else
1341                         ibx_irq_handler(dev, pch_iir);
1342
1343                 /* should clear PCH hotplug event before clear CPU irq */
1344                 I915_WRITE(SDEIIR, pch_iir);
1345         }
1346
1347         if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
1348                 ironlake_rps_change_irq_handler(dev);
1349 }
1350
1351 static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
1352 {
1353         struct drm_i915_private *dev_priv = dev->dev_private;
1354         int i;
1355
1356         if (de_iir & DE_ERR_INT_IVB)
1357                 ivb_err_int_handler(dev);
1358
1359         if (de_iir & DE_AUX_CHANNEL_A_IVB)
1360                 dp_aux_irq_handler(dev);
1361
1362         if (de_iir & DE_GSE_IVB)
1363                 intel_opregion_asle_intr(dev);
1364
1365         for (i = 0; i < 3; i++) {
1366                 if (de_iir & (DE_PIPEA_VBLANK_IVB << (5 * i)))
1367                         drm_handle_vblank(dev, i);
1368                 if (de_iir & (DE_PLANEA_FLIP_DONE_IVB << (5 * i))) {
1369                         intel_prepare_page_flip(dev, i);
1370                         intel_finish_page_flip_plane(dev, i);
1371                 }
1372         }
1373
1374         /* check event from PCH */
1375         if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
1376                 u32 pch_iir = I915_READ(SDEIIR);
1377
1378                 cpt_irq_handler(dev, pch_iir);
1379
1380                 /* clear PCH hotplug event before clear CPU irq */
1381                 I915_WRITE(SDEIIR, pch_iir);
1382         }
1383 }
1384
1385 static irqreturn_t ironlake_irq_handler(int irq, void *arg)
1386 {
1387         struct drm_device *dev = (struct drm_device *) arg;
1388         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1389         u32 de_iir, gt_iir, de_ier, sde_ier = 0;
1390         irqreturn_t ret = IRQ_NONE;
1391         bool err_int_reenable = false;
1392
1393         atomic_inc(&dev_priv->irq_received);
1394
1395         /* We get interrupts on unclaimed registers, so check for this before we
1396          * do any I915_{READ,WRITE}. */
1397         intel_uncore_check_errors(dev);
1398
1399         /* disable master interrupt before clearing iir  */
1400         de_ier = I915_READ(DEIER);
1401         I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
1402         POSTING_READ(DEIER);
1403
1404         /* Disable south interrupts. We'll only write to SDEIIR once, so further
1405          * interrupts will will be stored on its back queue, and then we'll be
1406          * able to process them after we restore SDEIER (as soon as we restore
1407          * it, we'll get an interrupt if SDEIIR still has something to process
1408          * due to its back queue). */
1409         if (!HAS_PCH_NOP(dev)) {
1410                 sde_ier = I915_READ(SDEIER);
1411                 I915_WRITE(SDEIER, 0);
1412                 POSTING_READ(SDEIER);
1413         }
1414
1415         /* On Haswell, also mask ERR_INT because we don't want to risk
1416          * generating "unclaimed register" interrupts from inside the interrupt
1417          * handler. */
1418         if (IS_HASWELL(dev)) {
1419                 spin_lock(&dev_priv->irq_lock);
1420                 err_int_reenable = ~dev_priv->irq_mask & DE_ERR_INT_IVB;
1421                 if (err_int_reenable)
1422                         ironlake_disable_display_irq(dev_priv, DE_ERR_INT_IVB);
1423                 spin_unlock(&dev_priv->irq_lock);
1424         }
1425
1426         gt_iir = I915_READ(GTIIR);
1427         if (gt_iir) {
1428                 if (INTEL_INFO(dev)->gen >= 6)
1429                         snb_gt_irq_handler(dev, dev_priv, gt_iir);
1430                 else
1431                         ilk_gt_irq_handler(dev, dev_priv, gt_iir);
1432                 I915_WRITE(GTIIR, gt_iir);
1433                 ret = IRQ_HANDLED;
1434         }
1435
1436         de_iir = I915_READ(DEIIR);
1437         if (de_iir) {
1438                 if (INTEL_INFO(dev)->gen >= 7)
1439                         ivb_display_irq_handler(dev, de_iir);
1440                 else
1441                         ilk_display_irq_handler(dev, de_iir);
1442                 I915_WRITE(DEIIR, de_iir);
1443                 ret = IRQ_HANDLED;
1444         }
1445
1446         if (INTEL_INFO(dev)->gen >= 6) {
1447                 u32 pm_iir = I915_READ(GEN6_PMIIR);
1448                 if (pm_iir) {
1449                         gen6_rps_irq_handler(dev_priv, pm_iir);
1450                         I915_WRITE(GEN6_PMIIR, pm_iir);
1451                         ret = IRQ_HANDLED;
1452                 }
1453         }
1454
1455         if (err_int_reenable) {
1456                 spin_lock(&dev_priv->irq_lock);
1457                 if (ivb_can_enable_err_int(dev))
1458                         ironlake_enable_display_irq(dev_priv, DE_ERR_INT_IVB);
1459                 spin_unlock(&dev_priv->irq_lock);
1460         }
1461
1462         I915_WRITE(DEIER, de_ier);
1463         POSTING_READ(DEIER);
1464         if (!HAS_PCH_NOP(dev)) {
1465                 I915_WRITE(SDEIER, sde_ier);
1466                 POSTING_READ(SDEIER);
1467         }
1468
1469         return ret;
1470 }
1471
1472 /**
1473  * i915_error_work_func - do process context error handling work
1474  * @work: work struct
1475  *
1476  * Fire an error uevent so userspace can see that a hang or error
1477  * was detected.
1478  */
1479 static void i915_error_work_func(struct work_struct *work)
1480 {
1481         struct i915_gpu_error *error = container_of(work, struct i915_gpu_error,
1482                                                     work);
1483         drm_i915_private_t *dev_priv = container_of(error, drm_i915_private_t,
1484                                                     gpu_error);
1485         struct drm_device *dev = dev_priv->dev;
1486         struct intel_ring_buffer *ring;
1487         char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
1488         char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
1489         char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
1490         int i, ret;
1491
1492         kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, error_event);
1493
1494         /*
1495          * Note that there's only one work item which does gpu resets, so we
1496          * need not worry about concurrent gpu resets potentially incrementing
1497          * error->reset_counter twice. We only need to take care of another
1498          * racing irq/hangcheck declaring the gpu dead for a second time. A
1499          * quick check for that is good enough: schedule_work ensures the
1500          * correct ordering between hang detection and this work item, and since
1501          * the reset in-progress bit is only ever set by code outside of this
1502          * work we don't need to worry about any other races.
1503          */
1504         if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
1505                 DRM_DEBUG_DRIVER("resetting chip\n");
1506                 kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE,
1507                                    reset_event);
1508
1509                 ret = i915_reset(dev);
1510
1511                 if (ret == 0) {
1512                         /*
1513                          * After all the gem state is reset, increment the reset
1514                          * counter and wake up everyone waiting for the reset to
1515                          * complete.
1516                          *
1517                          * Since unlock operations are a one-sided barrier only,
1518                          * we need to insert a barrier here to order any seqno
1519                          * updates before
1520                          * the counter increment.
1521                          */
1522                         smp_mb__before_atomic_inc();
1523                         atomic_inc(&dev_priv->gpu_error.reset_counter);
1524
1525                         kobject_uevent_env(&dev->primary->kdev.kobj,
1526                                            KOBJ_CHANGE, reset_done_event);
1527                 } else {
1528                         atomic_set(&error->reset_counter, I915_WEDGED);
1529                 }
1530
1531                 for_each_ring(ring, dev_priv, i)
1532                         wake_up_all(&ring->irq_queue);
1533
1534                 intel_display_handle_reset(dev);
1535
1536                 wake_up_all(&dev_priv->gpu_error.reset_queue);
1537         }
1538 }
1539
1540 static void i915_report_and_clear_eir(struct drm_device *dev)
1541 {
1542         struct drm_i915_private *dev_priv = dev->dev_private;
1543         uint32_t instdone[I915_NUM_INSTDONE_REG];
1544         u32 eir = I915_READ(EIR);
1545         int pipe, i;
1546
1547         if (!eir)
1548                 return;
1549
1550         pr_err("render error detected, EIR: 0x%08x\n", eir);
1551
1552         i915_get_extra_instdone(dev, instdone);
1553
1554         if (IS_G4X(dev)) {
1555                 if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
1556                         u32 ipeir = I915_READ(IPEIR_I965);
1557
1558                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
1559                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
1560                         for (i = 0; i < ARRAY_SIZE(instdone); i++)
1561                                 pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
1562                         pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
1563                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
1564                         I915_WRITE(IPEIR_I965, ipeir);
1565                         POSTING_READ(IPEIR_I965);
1566                 }
1567                 if (eir & GM45_ERROR_PAGE_TABLE) {
1568                         u32 pgtbl_err = I915_READ(PGTBL_ER);
1569                         pr_err("page table error\n");
1570                         pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
1571                         I915_WRITE(PGTBL_ER, pgtbl_err);
1572                         POSTING_READ(PGTBL_ER);
1573                 }
1574         }
1575
1576         if (!IS_GEN2(dev)) {
1577                 if (eir & I915_ERROR_PAGE_TABLE) {
1578                         u32 pgtbl_err = I915_READ(PGTBL_ER);
1579                         pr_err("page table error\n");
1580                         pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
1581                         I915_WRITE(PGTBL_ER, pgtbl_err);
1582                         POSTING_READ(PGTBL_ER);
1583                 }
1584         }
1585
1586         if (eir & I915_ERROR_MEMORY_REFRESH) {
1587                 pr_err("memory refresh error:\n");
1588                 for_each_pipe(pipe)
1589                         pr_err("pipe %c stat: 0x%08x\n",
1590                                pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
1591                 /* pipestat has already been acked */
1592         }
1593         if (eir & I915_ERROR_INSTRUCTION) {
1594                 pr_err("instruction error\n");
1595                 pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
1596                 for (i = 0; i < ARRAY_SIZE(instdone); i++)
1597                         pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
1598                 if (INTEL_INFO(dev)->gen < 4) {
1599                         u32 ipeir = I915_READ(IPEIR);
1600
1601                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
1602                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
1603                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
1604                         I915_WRITE(IPEIR, ipeir);
1605                         POSTING_READ(IPEIR);
1606                 } else {
1607                         u32 ipeir = I915_READ(IPEIR_I965);
1608
1609                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
1610                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
1611                         pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
1612                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
1613                         I915_WRITE(IPEIR_I965, ipeir);
1614                         POSTING_READ(IPEIR_I965);
1615                 }
1616         }
1617
1618         I915_WRITE(EIR, eir);
1619         POSTING_READ(EIR);
1620         eir = I915_READ(EIR);
1621         if (eir) {
1622                 /*
1623                  * some errors might have become stuck,
1624                  * mask them.
1625                  */
1626                 DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
1627                 I915_WRITE(EMR, I915_READ(EMR) | eir);
1628                 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
1629         }
1630 }
1631
1632 /**
1633  * i915_handle_error - handle an error interrupt
1634  * @dev: drm device
1635  *
1636  * Do some basic checking of regsiter state at error interrupt time and
1637  * dump it to the syslog.  Also call i915_capture_error_state() to make
1638  * sure we get a record and make it available in debugfs.  Fire a uevent
1639  * so userspace knows something bad happened (should trigger collection
1640  * of a ring dump etc.).
1641  */
1642 void i915_handle_error(struct drm_device *dev, bool wedged)
1643 {
1644         struct drm_i915_private *dev_priv = dev->dev_private;
1645         struct intel_ring_buffer *ring;
1646         int i;
1647
1648         i915_capture_error_state(dev);
1649         i915_report_and_clear_eir(dev);
1650
1651         if (wedged) {
1652                 atomic_set_mask(I915_RESET_IN_PROGRESS_FLAG,
1653                                 &dev_priv->gpu_error.reset_counter);
1654
1655                 /*
1656                  * Wakeup waiting processes so that the reset work item
1657                  * doesn't deadlock trying to grab various locks.
1658                  */
1659                 for_each_ring(ring, dev_priv, i)
1660                         wake_up_all(&ring->irq_queue);
1661         }
1662
1663         /*
1664          * Our reset work can grab modeset locks (since it needs to reset the
1665          * state of outstanding pagelips). Hence it must not be run on our own
1666          * dev-priv->wq work queue for otherwise the flush_work in the pageflip
1667          * code will deadlock.
1668          */
1669         schedule_work(&dev_priv->gpu_error.work);
1670 }
1671
1672 static void __always_unused i915_pageflip_stall_check(struct drm_device *dev, int pipe)
1673 {
1674         drm_i915_private_t *dev_priv = dev->dev_private;
1675         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
1676         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1677         struct drm_i915_gem_object *obj;
1678         struct intel_unpin_work *work;
1679         unsigned long flags;
1680         bool stall_detected;
1681
1682         /* Ignore early vblank irqs */
1683         if (intel_crtc == NULL)
1684                 return;
1685
1686         spin_lock_irqsave(&dev->event_lock, flags);
1687         work = intel_crtc->unpin_work;
1688
1689         if (work == NULL ||
1690             atomic_read(&work->pending) >= INTEL_FLIP_COMPLETE ||
1691             !work->enable_stall_check) {
1692                 /* Either the pending flip IRQ arrived, or we're too early. Don't check */
1693                 spin_unlock_irqrestore(&dev->event_lock, flags);
1694                 return;
1695         }
1696
1697         /* Potential stall - if we see that the flip has happened, assume a missed interrupt */
1698         obj = work->pending_flip_obj;
1699         if (INTEL_INFO(dev)->gen >= 4) {
1700                 int dspsurf = DSPSURF(intel_crtc->plane);
1701                 stall_detected = I915_HI_DISPBASE(I915_READ(dspsurf)) ==
1702                                         i915_gem_obj_ggtt_offset(obj);
1703         } else {
1704                 int dspaddr = DSPADDR(intel_crtc->plane);
1705                 stall_detected = I915_READ(dspaddr) == (i915_gem_obj_ggtt_offset(obj) +
1706                                                         crtc->y * crtc->fb->pitches[0] +
1707                                                         crtc->x * crtc->fb->bits_per_pixel/8);
1708         }
1709
1710         spin_unlock_irqrestore(&dev->event_lock, flags);
1711
1712         if (stall_detected) {
1713                 DRM_DEBUG_DRIVER("Pageflip stall detected\n");
1714                 intel_prepare_page_flip(dev, intel_crtc->plane);
1715         }
1716 }
1717
1718 /* Called from drm generic code, passed 'crtc' which
1719  * we use as a pipe index
1720  */
1721 static int i915_enable_vblank(struct drm_device *dev, int pipe)
1722 {
1723         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1724         unsigned long irqflags;
1725
1726         if (!i915_pipe_enabled(dev, pipe))
1727                 return -EINVAL;
1728
1729         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1730         if (INTEL_INFO(dev)->gen >= 4)
1731                 i915_enable_pipestat(dev_priv, pipe,
1732                                      PIPE_START_VBLANK_INTERRUPT_ENABLE);
1733         else
1734                 i915_enable_pipestat(dev_priv, pipe,
1735                                      PIPE_VBLANK_INTERRUPT_ENABLE);
1736
1737         /* maintain vblank delivery even in deep C-states */
1738         if (dev_priv->info->gen == 3)
1739                 I915_WRITE(INSTPM, _MASKED_BIT_DISABLE(INSTPM_AGPBUSY_DIS));
1740         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1741
1742         return 0;
1743 }
1744
1745 static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
1746 {
1747         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1748         unsigned long irqflags;
1749         uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
1750                                                      DE_PIPE_VBLANK_ILK(pipe);
1751
1752         if (!i915_pipe_enabled(dev, pipe))
1753                 return -EINVAL;
1754
1755         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1756         ironlake_enable_display_irq(dev_priv, bit);
1757         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1758
1759         return 0;
1760 }
1761
1762 static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
1763 {
1764         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1765         unsigned long irqflags;
1766         u32 imr;
1767
1768         if (!i915_pipe_enabled(dev, pipe))
1769                 return -EINVAL;
1770
1771         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1772         imr = I915_READ(VLV_IMR);
1773         if (pipe == 0)
1774                 imr &= ~I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
1775         else
1776                 imr &= ~I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
1777         I915_WRITE(VLV_IMR, imr);
1778         i915_enable_pipestat(dev_priv, pipe,
1779                              PIPE_START_VBLANK_INTERRUPT_ENABLE);
1780         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1781
1782         return 0;
1783 }
1784
1785 /* Called from drm generic code, passed 'crtc' which
1786  * we use as a pipe index
1787  */
1788 static void i915_disable_vblank(struct drm_device *dev, int pipe)
1789 {
1790         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1791         unsigned long irqflags;
1792
1793         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1794         if (dev_priv->info->gen == 3)
1795                 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_DIS));
1796
1797         i915_disable_pipestat(dev_priv, pipe,
1798                               PIPE_VBLANK_INTERRUPT_ENABLE |
1799                               PIPE_START_VBLANK_INTERRUPT_ENABLE);
1800         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1801 }
1802
1803 static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
1804 {
1805         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1806         unsigned long irqflags;
1807         uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
1808                                                      DE_PIPE_VBLANK_ILK(pipe);
1809
1810         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1811         ironlake_disable_display_irq(dev_priv, bit);
1812         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1813 }
1814
1815 static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
1816 {
1817         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1818         unsigned long irqflags;
1819         u32 imr;
1820
1821         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1822         i915_disable_pipestat(dev_priv, pipe,
1823                               PIPE_START_VBLANK_INTERRUPT_ENABLE);
1824         imr = I915_READ(VLV_IMR);
1825         if (pipe == 0)
1826                 imr |= I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
1827         else
1828                 imr |= I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
1829         I915_WRITE(VLV_IMR, imr);
1830         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1831 }
1832
1833 static u32
1834 ring_last_seqno(struct intel_ring_buffer *ring)
1835 {
1836         return list_entry(ring->request_list.prev,
1837                           struct drm_i915_gem_request, list)->seqno;
1838 }
1839
1840 static bool
1841 ring_idle(struct intel_ring_buffer *ring, u32 seqno)
1842 {
1843         return (list_empty(&ring->request_list) ||
1844                 i915_seqno_passed(seqno, ring_last_seqno(ring)));
1845 }
1846
1847 static struct intel_ring_buffer *
1848 semaphore_waits_for(struct intel_ring_buffer *ring, u32 *seqno)
1849 {
1850         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1851         u32 cmd, ipehr, acthd, acthd_min;
1852
1853         ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
1854         if ((ipehr & ~(0x3 << 16)) !=
1855             (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE | MI_SEMAPHORE_REGISTER))
1856                 return NULL;
1857
1858         /* ACTHD is likely pointing to the dword after the actual command,
1859          * so scan backwards until we find the MBOX.
1860          */
1861         acthd = intel_ring_get_active_head(ring) & HEAD_ADDR;
1862         acthd_min = max((int)acthd - 3 * 4, 0);
1863         do {
1864                 cmd = ioread32(ring->virtual_start + acthd);
1865                 if (cmd == ipehr)
1866                         break;
1867
1868                 acthd -= 4;
1869                 if (acthd < acthd_min)
1870                         return NULL;
1871         } while (1);
1872
1873         *seqno = ioread32(ring->virtual_start+acthd+4)+1;
1874         return &dev_priv->ring[(ring->id + (((ipehr >> 17) & 1) + 1)) % 3];
1875 }
1876
1877 static int semaphore_passed(struct intel_ring_buffer *ring)
1878 {
1879         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1880         struct intel_ring_buffer *signaller;
1881         u32 seqno, ctl;
1882
1883         ring->hangcheck.deadlock = true;
1884
1885         signaller = semaphore_waits_for(ring, &seqno);
1886         if (signaller == NULL || signaller->hangcheck.deadlock)
1887                 return -1;
1888
1889         /* cursory check for an unkickable deadlock */
1890         ctl = I915_READ_CTL(signaller);
1891         if (ctl & RING_WAIT_SEMAPHORE && semaphore_passed(signaller) < 0)
1892                 return -1;
1893
1894         return i915_seqno_passed(signaller->get_seqno(signaller, false), seqno);
1895 }
1896
1897 static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
1898 {
1899         struct intel_ring_buffer *ring;
1900         int i;
1901
1902         for_each_ring(ring, dev_priv, i)
1903                 ring->hangcheck.deadlock = false;
1904 }
1905
1906 static enum intel_ring_hangcheck_action
1907 ring_stuck(struct intel_ring_buffer *ring, u32 acthd)
1908 {
1909         struct drm_device *dev = ring->dev;
1910         struct drm_i915_private *dev_priv = dev->dev_private;
1911         u32 tmp;
1912
1913         if (ring->hangcheck.acthd != acthd)
1914                 return HANGCHECK_ACTIVE;
1915
1916         if (IS_GEN2(dev))
1917                 return HANGCHECK_HUNG;
1918
1919         /* Is the chip hanging on a WAIT_FOR_EVENT?
1920          * If so we can simply poke the RB_WAIT bit
1921          * and break the hang. This should work on
1922          * all but the second generation chipsets.
1923          */
1924         tmp = I915_READ_CTL(ring);
1925         if (tmp & RING_WAIT) {
1926                 DRM_ERROR("Kicking stuck wait on %s\n",
1927                           ring->name);
1928                 I915_WRITE_CTL(ring, tmp);
1929                 return HANGCHECK_KICK;
1930         }
1931
1932         if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
1933                 switch (semaphore_passed(ring)) {
1934                 default:
1935                         return HANGCHECK_HUNG;
1936                 case 1:
1937                         DRM_ERROR("Kicking stuck semaphore on %s\n",
1938                                   ring->name);
1939                         I915_WRITE_CTL(ring, tmp);
1940                         return HANGCHECK_KICK;
1941                 case 0:
1942                         return HANGCHECK_WAIT;
1943                 }
1944         }
1945
1946         return HANGCHECK_HUNG;
1947 }
1948
1949 /**
1950  * This is called when the chip hasn't reported back with completed
1951  * batchbuffers in a long time. We keep track per ring seqno progress and
1952  * if there are no progress, hangcheck score for that ring is increased.
1953  * Further, acthd is inspected to see if the ring is stuck. On stuck case
1954  * we kick the ring. If we see no progress on three subsequent calls
1955  * we assume chip is wedged and try to fix it by resetting the chip.
1956  */
1957 static void i915_hangcheck_elapsed(unsigned long data)
1958 {
1959         struct drm_device *dev = (struct drm_device *)data;
1960         drm_i915_private_t *dev_priv = dev->dev_private;
1961         struct intel_ring_buffer *ring;
1962         int i;
1963         int busy_count = 0, rings_hung = 0;
1964         bool stuck[I915_NUM_RINGS] = { 0 };
1965 #define BUSY 1
1966 #define KICK 5
1967 #define HUNG 20
1968 #define FIRE 30
1969
1970         if (!i915_enable_hangcheck)
1971                 return;
1972
1973         for_each_ring(ring, dev_priv, i) {
1974                 u32 seqno, acthd;
1975                 bool busy = true;
1976
1977                 semaphore_clear_deadlocks(dev_priv);
1978
1979                 seqno = ring->get_seqno(ring, false);
1980                 acthd = intel_ring_get_active_head(ring);
1981
1982                 if (ring->hangcheck.seqno == seqno) {
1983                         if (ring_idle(ring, seqno)) {
1984                                 if (waitqueue_active(&ring->irq_queue)) {
1985                                         /* Issue a wake-up to catch stuck h/w. */
1986                                         DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
1987                                                   ring->name);
1988                                         wake_up_all(&ring->irq_queue);
1989                                         ring->hangcheck.score += HUNG;
1990                                 } else
1991                                         busy = false;
1992                         } else {
1993                                 /* We always increment the hangcheck score
1994                                  * if the ring is busy and still processing
1995                                  * the same request, so that no single request
1996                                  * can run indefinitely (such as a chain of
1997                                  * batches). The only time we do not increment
1998                                  * the hangcheck score on this ring, if this
1999                                  * ring is in a legitimate wait for another
2000                                  * ring. In that case the waiting ring is a
2001                                  * victim and we want to be sure we catch the
2002                                  * right culprit. Then every time we do kick
2003                                  * the ring, add a small increment to the
2004                                  * score so that we can catch a batch that is
2005                                  * being repeatedly kicked and so responsible
2006                                  * for stalling the machine.
2007                                  */
2008                                 ring->hangcheck.action = ring_stuck(ring,
2009                                                                     acthd);
2010
2011                                 switch (ring->hangcheck.action) {
2012                                 case HANGCHECK_WAIT:
2013                                         break;
2014                                 case HANGCHECK_ACTIVE:
2015                                         ring->hangcheck.score += BUSY;
2016                                         break;
2017                                 case HANGCHECK_KICK:
2018                                         ring->hangcheck.score += KICK;
2019                                         break;
2020                                 case HANGCHECK_HUNG:
2021                                         ring->hangcheck.score += HUNG;
2022                                         stuck[i] = true;
2023                                         break;
2024                                 }
2025                         }
2026                 } else {
2027                         /* Gradually reduce the count so that we catch DoS
2028                          * attempts across multiple batches.
2029                          */
2030                         if (ring->hangcheck.score > 0)
2031                                 ring->hangcheck.score--;
2032                 }
2033
2034                 ring->hangcheck.seqno = seqno;
2035                 ring->hangcheck.acthd = acthd;
2036                 busy_count += busy;
2037         }
2038
2039         for_each_ring(ring, dev_priv, i) {
2040                 if (ring->hangcheck.score > FIRE) {
2041                         DRM_INFO("%s on %s\n",
2042                                  stuck[i] ? "stuck" : "no progress",
2043                                  ring->name);
2044                         rings_hung++;
2045                 }
2046         }
2047
2048         if (rings_hung)
2049                 return i915_handle_error(dev, true);
2050
2051         if (busy_count)
2052                 /* Reset timer case chip hangs without another request
2053                  * being added */
2054                 i915_queue_hangcheck(dev);
2055 }
2056
2057 void i915_queue_hangcheck(struct drm_device *dev)
2058 {
2059         struct drm_i915_private *dev_priv = dev->dev_private;
2060         if (!i915_enable_hangcheck)
2061                 return;
2062
2063         mod_timer(&dev_priv->gpu_error.hangcheck_timer,
2064                   round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
2065 }
2066
2067 static void ibx_irq_preinstall(struct drm_device *dev)
2068 {
2069         struct drm_i915_private *dev_priv = dev->dev_private;
2070
2071         if (HAS_PCH_NOP(dev))
2072                 return;
2073
2074         /* south display irq */
2075         I915_WRITE(SDEIMR, 0xffffffff);
2076         /*
2077          * SDEIER is also touched by the interrupt handler to work around missed
2078          * PCH interrupts. Hence we can't update it after the interrupt handler
2079          * is enabled - instead we unconditionally enable all PCH interrupt
2080          * sources here, but then only unmask them as needed with SDEIMR.
2081          */
2082         I915_WRITE(SDEIER, 0xffffffff);
2083         POSTING_READ(SDEIER);
2084 }
2085
2086 static void gen5_gt_irq_preinstall(struct drm_device *dev)
2087 {
2088         struct drm_i915_private *dev_priv = dev->dev_private;
2089
2090         /* and GT */
2091         I915_WRITE(GTIMR, 0xffffffff);
2092         I915_WRITE(GTIER, 0x0);
2093         POSTING_READ(GTIER);
2094
2095         if (INTEL_INFO(dev)->gen >= 6) {
2096                 /* and PM */
2097                 I915_WRITE(GEN6_PMIMR, 0xffffffff);
2098                 I915_WRITE(GEN6_PMIER, 0x0);
2099                 POSTING_READ(GEN6_PMIER);
2100         }
2101 }
2102
2103 /* drm_dma.h hooks
2104 */
2105 static void ironlake_irq_preinstall(struct drm_device *dev)
2106 {
2107         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2108
2109         atomic_set(&dev_priv->irq_received, 0);
2110
2111         I915_WRITE(HWSTAM, 0xeffe);
2112
2113         I915_WRITE(DEIMR, 0xffffffff);
2114         I915_WRITE(DEIER, 0x0);
2115         POSTING_READ(DEIER);
2116
2117         gen5_gt_irq_preinstall(dev);
2118
2119         ibx_irq_preinstall(dev);
2120 }
2121
2122 static void valleyview_irq_preinstall(struct drm_device *dev)
2123 {
2124         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2125         int pipe;
2126
2127         atomic_set(&dev_priv->irq_received, 0);
2128
2129         /* VLV magic */
2130         I915_WRITE(VLV_IMR, 0);
2131         I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
2132         I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
2133         I915_WRITE(RING_IMR(BLT_RING_BASE), 0);
2134
2135         /* and GT */
2136         I915_WRITE(GTIIR, I915_READ(GTIIR));
2137         I915_WRITE(GTIIR, I915_READ(GTIIR));
2138
2139         gen5_gt_irq_preinstall(dev);
2140
2141         I915_WRITE(DPINVGTT, 0xff);
2142
2143         I915_WRITE(PORT_HOTPLUG_EN, 0);
2144         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2145         for_each_pipe(pipe)
2146                 I915_WRITE(PIPESTAT(pipe), 0xffff);
2147         I915_WRITE(VLV_IIR, 0xffffffff);
2148         I915_WRITE(VLV_IMR, 0xffffffff);
2149         I915_WRITE(VLV_IER, 0x0);
2150         POSTING_READ(VLV_IER);
2151 }
2152
2153 static void ibx_hpd_irq_setup(struct drm_device *dev)
2154 {
2155         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2156         struct drm_mode_config *mode_config = &dev->mode_config;
2157         struct intel_encoder *intel_encoder;
2158         u32 hotplug_irqs, hotplug, enabled_irqs = 0;
2159
2160         if (HAS_PCH_IBX(dev)) {
2161                 hotplug_irqs = SDE_HOTPLUG_MASK;
2162                 list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
2163                         if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
2164                                 enabled_irqs |= hpd_ibx[intel_encoder->hpd_pin];
2165         } else {
2166                 hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
2167                 list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
2168                         if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
2169                                 enabled_irqs |= hpd_cpt[intel_encoder->hpd_pin];
2170         }
2171
2172         ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
2173
2174         /*
2175          * Enable digital hotplug on the PCH, and configure the DP short pulse
2176          * duration to 2ms (which is the minimum in the Display Port spec)
2177          *
2178          * This register is the same on all known PCH chips.
2179          */
2180         hotplug = I915_READ(PCH_PORT_HOTPLUG);
2181         hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
2182         hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
2183         hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
2184         hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
2185         I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
2186 }
2187
2188 static void ibx_irq_postinstall(struct drm_device *dev)
2189 {
2190         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2191         u32 mask;
2192
2193         if (HAS_PCH_NOP(dev))
2194                 return;
2195
2196         if (HAS_PCH_IBX(dev)) {
2197                 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_TRANSB_FIFO_UNDER |
2198                        SDE_TRANSA_FIFO_UNDER | SDE_POISON;
2199         } else {
2200                 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT | SDE_ERROR_CPT;
2201
2202                 I915_WRITE(SERR_INT, I915_READ(SERR_INT));
2203         }
2204
2205         I915_WRITE(SDEIIR, I915_READ(SDEIIR));
2206         I915_WRITE(SDEIMR, ~mask);
2207 }
2208
2209 static void gen5_gt_irq_postinstall(struct drm_device *dev)
2210 {
2211         struct drm_i915_private *dev_priv = dev->dev_private;
2212         u32 pm_irqs, gt_irqs;
2213
2214         pm_irqs = gt_irqs = 0;
2215
2216         dev_priv->gt_irq_mask = ~0;
2217         if (HAS_L3_GPU_CACHE(dev)) {
2218                 /* L3 parity interrupt is always unmasked. */
2219                 dev_priv->gt_irq_mask = ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
2220                 gt_irqs |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
2221         }
2222
2223         gt_irqs |= GT_RENDER_USER_INTERRUPT;
2224         if (IS_GEN5(dev)) {
2225                 gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
2226                            ILK_BSD_USER_INTERRUPT;
2227         } else {
2228                 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
2229         }
2230
2231         I915_WRITE(GTIIR, I915_READ(GTIIR));
2232         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
2233         I915_WRITE(GTIER, gt_irqs);
2234         POSTING_READ(GTIER);
2235
2236         if (INTEL_INFO(dev)->gen >= 6) {
2237                 pm_irqs |= GEN6_PM_RPS_EVENTS;
2238
2239                 if (HAS_VEBOX(dev))
2240                         pm_irqs |= PM_VEBOX_USER_INTERRUPT;
2241
2242                 dev_priv->pm_irq_mask = 0xffffffff;
2243                 I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR));
2244                 I915_WRITE(GEN6_PMIMR, dev_priv->pm_irq_mask);
2245                 I915_WRITE(GEN6_PMIER, pm_irqs);
2246                 POSTING_READ(GEN6_PMIER);
2247         }
2248 }
2249
2250 static int ironlake_irq_postinstall(struct drm_device *dev)
2251 {
2252         unsigned long irqflags;
2253         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2254         u32 display_mask, extra_mask;
2255
2256         if (INTEL_INFO(dev)->gen >= 7) {
2257                 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
2258                                 DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
2259                                 DE_PLANEB_FLIP_DONE_IVB |
2260                                 DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB |
2261                                 DE_ERR_INT_IVB);
2262                 extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
2263                               DE_PIPEA_VBLANK_IVB);
2264
2265                 I915_WRITE(GEN7_ERR_INT, I915_READ(GEN7_ERR_INT));
2266         } else {
2267                 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
2268                                 DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
2269                                 DE_AUX_CHANNEL_A | DE_PIPEB_FIFO_UNDERRUN |
2270                                 DE_PIPEA_FIFO_UNDERRUN | DE_POISON);
2271                 extra_mask = DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT;
2272         }
2273
2274         dev_priv->irq_mask = ~display_mask;
2275
2276         /* should always can generate irq */
2277         I915_WRITE(DEIIR, I915_READ(DEIIR));
2278         I915_WRITE(DEIMR, dev_priv->irq_mask);
2279         I915_WRITE(DEIER, display_mask | extra_mask);
2280         POSTING_READ(DEIER);
2281
2282         gen5_gt_irq_postinstall(dev);
2283
2284         ibx_irq_postinstall(dev);
2285
2286         if (IS_IRONLAKE_M(dev)) {
2287                 /* Enable PCU event interrupts
2288                  *
2289                  * spinlocking not required here for correctness since interrupt
2290                  * setup is guaranteed to run in single-threaded context. But we
2291                  * need it to make the assert_spin_locked happy. */
2292                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2293                 ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
2294                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2295         }
2296
2297         return 0;
2298 }
2299
2300 static int valleyview_irq_postinstall(struct drm_device *dev)
2301 {
2302         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2303         u32 enable_mask;
2304         u32 pipestat_enable = PLANE_FLIP_DONE_INT_EN_VLV;
2305         unsigned long irqflags;
2306
2307         enable_mask = I915_DISPLAY_PORT_INTERRUPT;
2308         enable_mask |= I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2309                 I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT |
2310                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2311                 I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
2312
2313         /*
2314          *Leave vblank interrupts masked initially.  enable/disable will
2315          * toggle them based on usage.
2316          */
2317         dev_priv->irq_mask = (~enable_mask) |
2318                 I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT |
2319                 I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
2320
2321         I915_WRITE(PORT_HOTPLUG_EN, 0);
2322         POSTING_READ(PORT_HOTPLUG_EN);
2323
2324         I915_WRITE(VLV_IMR, dev_priv->irq_mask);
2325         I915_WRITE(VLV_IER, enable_mask);
2326         I915_WRITE(VLV_IIR, 0xffffffff);
2327         I915_WRITE(PIPESTAT(0), 0xffff);
2328         I915_WRITE(PIPESTAT(1), 0xffff);
2329         POSTING_READ(VLV_IER);
2330
2331         /* Interrupt setup is already guaranteed to be single-threaded, this is
2332          * just to make the assert_spin_locked check happy. */
2333         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2334         i915_enable_pipestat(dev_priv, 0, pipestat_enable);
2335         i915_enable_pipestat(dev_priv, 0, PIPE_GMBUS_EVENT_ENABLE);
2336         i915_enable_pipestat(dev_priv, 1, pipestat_enable);
2337         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2338
2339         I915_WRITE(VLV_IIR, 0xffffffff);
2340         I915_WRITE(VLV_IIR, 0xffffffff);
2341
2342         gen5_gt_irq_postinstall(dev);
2343
2344         /* ack & enable invalid PTE error interrupts */
2345 #if 0 /* FIXME: add support to irq handler for checking these bits */
2346         I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
2347         I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
2348 #endif
2349
2350         I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
2351
2352         return 0;
2353 }
2354
2355 static void valleyview_irq_uninstall(struct drm_device *dev)
2356 {
2357         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2358         int pipe;
2359
2360         if (!dev_priv)
2361                 return;
2362
2363         del_timer_sync(&dev_priv->hotplug_reenable_timer);
2364
2365         for_each_pipe(pipe)
2366                 I915_WRITE(PIPESTAT(pipe), 0xffff);
2367
2368         I915_WRITE(HWSTAM, 0xffffffff);
2369         I915_WRITE(PORT_HOTPLUG_EN, 0);
2370         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2371         for_each_pipe(pipe)
2372                 I915_WRITE(PIPESTAT(pipe), 0xffff);
2373         I915_WRITE(VLV_IIR, 0xffffffff);
2374         I915_WRITE(VLV_IMR, 0xffffffff);
2375         I915_WRITE(VLV_IER, 0x0);
2376         POSTING_READ(VLV_IER);
2377 }
2378
2379 static void ironlake_irq_uninstall(struct drm_device *dev)
2380 {
2381         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2382
2383         if (!dev_priv)
2384                 return;
2385
2386         del_timer_sync(&dev_priv->hotplug_reenable_timer);
2387
2388         I915_WRITE(HWSTAM, 0xffffffff);
2389
2390         I915_WRITE(DEIMR, 0xffffffff);
2391         I915_WRITE(DEIER, 0x0);
2392         I915_WRITE(DEIIR, I915_READ(DEIIR));
2393         if (IS_GEN7(dev))
2394                 I915_WRITE(GEN7_ERR_INT, I915_READ(GEN7_ERR_INT));
2395
2396         I915_WRITE(GTIMR, 0xffffffff);
2397         I915_WRITE(GTIER, 0x0);
2398         I915_WRITE(GTIIR, I915_READ(GTIIR));
2399
2400         if (HAS_PCH_NOP(dev))
2401                 return;
2402
2403         I915_WRITE(SDEIMR, 0xffffffff);
2404         I915_WRITE(SDEIER, 0x0);
2405         I915_WRITE(SDEIIR, I915_READ(SDEIIR));
2406         if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
2407                 I915_WRITE(SERR_INT, I915_READ(SERR_INT));
2408 }
2409
2410 static void i8xx_irq_preinstall(struct drm_device * dev)
2411 {
2412         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2413         int pipe;
2414
2415         atomic_set(&dev_priv->irq_received, 0);
2416
2417         for_each_pipe(pipe)
2418                 I915_WRITE(PIPESTAT(pipe), 0);
2419         I915_WRITE16(IMR, 0xffff);
2420         I915_WRITE16(IER, 0x0);
2421         POSTING_READ16(IER);
2422 }
2423
2424 static int i8xx_irq_postinstall(struct drm_device *dev)
2425 {
2426         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2427
2428         I915_WRITE16(EMR,
2429                      ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
2430
2431         /* Unmask the interrupts that we always want on. */
2432         dev_priv->irq_mask =
2433                 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2434                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2435                   I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2436                   I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
2437                   I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2438         I915_WRITE16(IMR, dev_priv->irq_mask);
2439
2440         I915_WRITE16(IER,
2441                      I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2442                      I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2443                      I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
2444                      I915_USER_INTERRUPT);
2445         POSTING_READ16(IER);
2446
2447         return 0;
2448 }
2449
2450 /*
2451  * Returns true when a page flip has completed.
2452  */
2453 static bool i8xx_handle_vblank(struct drm_device *dev,
2454                                int pipe, u16 iir)
2455 {
2456         drm_i915_private_t *dev_priv = dev->dev_private;
2457         u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(pipe);
2458
2459         if (!drm_handle_vblank(dev, pipe))
2460                 return false;
2461
2462         if ((iir & flip_pending) == 0)
2463                 return false;
2464
2465         intel_prepare_page_flip(dev, pipe);
2466
2467         /* We detect FlipDone by looking for the change in PendingFlip from '1'
2468          * to '0' on the following vblank, i.e. IIR has the Pendingflip
2469          * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
2470          * the flip is completed (no longer pending). Since this doesn't raise
2471          * an interrupt per se, we watch for the change at vblank.
2472          */
2473         if (I915_READ16(ISR) & flip_pending)
2474                 return false;
2475
2476         intel_finish_page_flip(dev, pipe);
2477
2478         return true;
2479 }
2480
2481 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
2482 {
2483         struct drm_device *dev = (struct drm_device *) arg;
2484         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2485         u16 iir, new_iir;
2486         u32 pipe_stats[2];
2487         unsigned long irqflags;
2488         int pipe;
2489         u16 flip_mask =
2490                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2491                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
2492
2493         atomic_inc(&dev_priv->irq_received);
2494
2495         iir = I915_READ16(IIR);
2496         if (iir == 0)
2497                 return IRQ_NONE;
2498
2499         while (iir & ~flip_mask) {
2500                 /* Can't rely on pipestat interrupt bit in iir as it might
2501                  * have been cleared after the pipestat interrupt was received.
2502                  * It doesn't set the bit in iir again, but it still produces
2503                  * interrupts (for non-MSI).
2504                  */
2505                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2506                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
2507                         i915_handle_error(dev, false);
2508
2509                 for_each_pipe(pipe) {
2510                         int reg = PIPESTAT(pipe);
2511                         pipe_stats[pipe] = I915_READ(reg);
2512
2513                         /*
2514                          * Clear the PIPE*STAT regs before the IIR
2515                          */
2516                         if (pipe_stats[pipe] & 0x8000ffff) {
2517                                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
2518                                         DRM_DEBUG_DRIVER("pipe %c underrun\n",
2519                                                          pipe_name(pipe));
2520                                 I915_WRITE(reg, pipe_stats[pipe]);
2521                         }
2522                 }
2523                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2524
2525                 I915_WRITE16(IIR, iir & ~flip_mask);
2526                 new_iir = I915_READ16(IIR); /* Flush posted writes */
2527
2528                 i915_update_dri1_breadcrumb(dev);
2529
2530                 if (iir & I915_USER_INTERRUPT)
2531                         notify_ring(dev, &dev_priv->ring[RCS]);
2532
2533                 if (pipe_stats[0] & PIPE_VBLANK_INTERRUPT_STATUS &&
2534                     i8xx_handle_vblank(dev, 0, iir))
2535                         flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(0);
2536
2537                 if (pipe_stats[1] & PIPE_VBLANK_INTERRUPT_STATUS &&
2538                     i8xx_handle_vblank(dev, 1, iir))
2539                         flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(1);
2540
2541                 iir = new_iir;
2542         }
2543
2544         return IRQ_HANDLED;
2545 }
2546
2547 static void i8xx_irq_uninstall(struct drm_device * dev)
2548 {
2549         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2550         int pipe;
2551
2552         for_each_pipe(pipe) {
2553                 /* Clear enable bits; then clear status bits */
2554                 I915_WRITE(PIPESTAT(pipe), 0);
2555                 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
2556         }
2557         I915_WRITE16(IMR, 0xffff);
2558         I915_WRITE16(IER, 0x0);
2559         I915_WRITE16(IIR, I915_READ16(IIR));
2560 }
2561
2562 static void i915_irq_preinstall(struct drm_device * dev)
2563 {
2564         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2565         int pipe;
2566
2567         atomic_set(&dev_priv->irq_received, 0);
2568
2569         if (I915_HAS_HOTPLUG(dev)) {
2570                 I915_WRITE(PORT_HOTPLUG_EN, 0);
2571                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2572         }
2573
2574         I915_WRITE16(HWSTAM, 0xeffe);
2575         for_each_pipe(pipe)
2576                 I915_WRITE(PIPESTAT(pipe), 0);
2577         I915_WRITE(IMR, 0xffffffff);
2578         I915_WRITE(IER, 0x0);
2579         POSTING_READ(IER);
2580 }
2581
2582 static int i915_irq_postinstall(struct drm_device *dev)
2583 {
2584         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2585         u32 enable_mask;
2586
2587         I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
2588
2589         /* Unmask the interrupts that we always want on. */
2590         dev_priv->irq_mask =
2591                 ~(I915_ASLE_INTERRUPT |
2592                   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2593                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2594                   I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2595                   I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
2596                   I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2597
2598         enable_mask =
2599                 I915_ASLE_INTERRUPT |
2600                 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2601                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2602                 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
2603                 I915_USER_INTERRUPT;
2604
2605         if (I915_HAS_HOTPLUG(dev)) {
2606                 I915_WRITE(PORT_HOTPLUG_EN, 0);
2607                 POSTING_READ(PORT_HOTPLUG_EN);
2608
2609                 /* Enable in IER... */
2610                 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
2611                 /* and unmask in IMR */
2612                 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
2613         }
2614
2615         I915_WRITE(IMR, dev_priv->irq_mask);
2616         I915_WRITE(IER, enable_mask);
2617         POSTING_READ(IER);
2618
2619         i915_enable_asle_pipestat(dev);
2620
2621         return 0;
2622 }
2623
2624 /*
2625  * Returns true when a page flip has completed.
2626  */
2627 static bool i915_handle_vblank(struct drm_device *dev,
2628                                int plane, int pipe, u32 iir)
2629 {
2630         drm_i915_private_t *dev_priv = dev->dev_private;
2631         u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
2632
2633         if (!drm_handle_vblank(dev, pipe))
2634                 return false;
2635
2636         if ((iir & flip_pending) == 0)
2637                 return false;
2638
2639         intel_prepare_page_flip(dev, plane);
2640
2641         /* We detect FlipDone by looking for the change in PendingFlip from '1'
2642          * to '0' on the following vblank, i.e. IIR has the Pendingflip
2643          * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
2644          * the flip is completed (no longer pending). Since this doesn't raise
2645          * an interrupt per se, we watch for the change at vblank.
2646          */
2647         if (I915_READ(ISR) & flip_pending)
2648                 return false;
2649
2650         intel_finish_page_flip(dev, pipe);
2651
2652         return true;
2653 }
2654
2655 static irqreturn_t i915_irq_handler(int irq, void *arg)
2656 {
2657         struct drm_device *dev = (struct drm_device *) arg;
2658         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2659         u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
2660         unsigned long irqflags;
2661         u32 flip_mask =
2662                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2663                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
2664         int pipe, ret = IRQ_NONE;
2665
2666         atomic_inc(&dev_priv->irq_received);
2667
2668         iir = I915_READ(IIR);
2669         do {
2670                 bool irq_received = (iir & ~flip_mask) != 0;
2671                 bool blc_event = false;
2672
2673                 /* Can't rely on pipestat interrupt bit in iir as it might
2674                  * have been cleared after the pipestat interrupt was received.
2675                  * It doesn't set the bit in iir again, but it still produces
2676                  * interrupts (for non-MSI).
2677                  */
2678                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2679                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
2680                         i915_handle_error(dev, false);
2681
2682                 for_each_pipe(pipe) {
2683                         int reg = PIPESTAT(pipe);
2684                         pipe_stats[pipe] = I915_READ(reg);
2685
2686                         /* Clear the PIPE*STAT regs before the IIR */
2687                         if (pipe_stats[pipe] & 0x8000ffff) {
2688                                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
2689                                         DRM_DEBUG_DRIVER("pipe %c underrun\n",
2690                                                          pipe_name(pipe));
2691                                 I915_WRITE(reg, pipe_stats[pipe]);
2692                                 irq_received = true;
2693                         }
2694                 }
2695                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2696
2697                 if (!irq_received)
2698                         break;
2699
2700                 /* Consume port.  Then clear IIR or we'll miss events */
2701                 if ((I915_HAS_HOTPLUG(dev)) &&
2702                     (iir & I915_DISPLAY_PORT_INTERRUPT)) {
2703                         u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
2704                         u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
2705
2706                         DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
2707                                   hotplug_status);
2708
2709                         intel_hpd_irq_handler(dev, hotplug_trigger, hpd_status_i915);
2710
2711                         I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
2712                         POSTING_READ(PORT_HOTPLUG_STAT);
2713                 }
2714
2715                 I915_WRITE(IIR, iir & ~flip_mask);
2716                 new_iir = I915_READ(IIR); /* Flush posted writes */
2717
2718                 if (iir & I915_USER_INTERRUPT)
2719                         notify_ring(dev, &dev_priv->ring[RCS]);
2720
2721                 for_each_pipe(pipe) {
2722                         int plane = pipe;
2723                         if (IS_MOBILE(dev))
2724                                 plane = !plane;
2725
2726                         if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
2727                             i915_handle_vblank(dev, plane, pipe, iir))
2728                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
2729
2730                         if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
2731                                 blc_event = true;
2732                 }
2733
2734                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
2735                         intel_opregion_asle_intr(dev);
2736
2737                 /* With MSI, interrupts are only generated when iir
2738                  * transitions from zero to nonzero.  If another bit got
2739                  * set while we were handling the existing iir bits, then
2740                  * we would never get another interrupt.
2741                  *
2742                  * This is fine on non-MSI as well, as if we hit this path
2743                  * we avoid exiting the interrupt handler only to generate
2744                  * another one.
2745                  *
2746                  * Note that for MSI this could cause a stray interrupt report
2747                  * if an interrupt landed in the time between writing IIR and
2748                  * the posting read.  This should be rare enough to never
2749                  * trigger the 99% of 100,000 interrupts test for disabling
2750                  * stray interrupts.
2751                  */
2752                 ret = IRQ_HANDLED;
2753                 iir = new_iir;
2754         } while (iir & ~flip_mask);
2755
2756         i915_update_dri1_breadcrumb(dev);
2757
2758         return ret;
2759 }
2760
2761 static void i915_irq_uninstall(struct drm_device * dev)
2762 {
2763         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2764         int pipe;
2765
2766         del_timer_sync(&dev_priv->hotplug_reenable_timer);
2767
2768         if (I915_HAS_HOTPLUG(dev)) {
2769                 I915_WRITE(PORT_HOTPLUG_EN, 0);
2770                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2771         }
2772
2773         I915_WRITE16(HWSTAM, 0xffff);
2774         for_each_pipe(pipe) {
2775                 /* Clear enable bits; then clear status bits */
2776                 I915_WRITE(PIPESTAT(pipe), 0);
2777                 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
2778         }
2779         I915_WRITE(IMR, 0xffffffff);
2780         I915_WRITE(IER, 0x0);
2781
2782         I915_WRITE(IIR, I915_READ(IIR));
2783 }
2784
2785 static void i965_irq_preinstall(struct drm_device * dev)
2786 {
2787         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2788         int pipe;
2789
2790         atomic_set(&dev_priv->irq_received, 0);
2791
2792         I915_WRITE(PORT_HOTPLUG_EN, 0);
2793         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2794
2795         I915_WRITE(HWSTAM, 0xeffe);
2796         for_each_pipe(pipe)
2797                 I915_WRITE(PIPESTAT(pipe), 0);
2798         I915_WRITE(IMR, 0xffffffff);
2799         I915_WRITE(IER, 0x0);
2800         POSTING_READ(IER);
2801 }
2802
2803 static int i965_irq_postinstall(struct drm_device *dev)
2804 {
2805         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2806         u32 enable_mask;
2807         u32 error_mask;
2808         unsigned long irqflags;
2809
2810         /* Unmask the interrupts that we always want on. */
2811         dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
2812                                I915_DISPLAY_PORT_INTERRUPT |
2813                                I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2814                                I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2815                                I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2816                                I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
2817                                I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2818
2819         enable_mask = ~dev_priv->irq_mask;
2820         enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2821                          I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
2822         enable_mask |= I915_USER_INTERRUPT;
2823
2824         if (IS_G4X(dev))
2825                 enable_mask |= I915_BSD_USER_INTERRUPT;
2826
2827         /* Interrupt setup is already guaranteed to be single-threaded, this is
2828          * just to make the assert_spin_locked check happy. */
2829         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2830         i915_enable_pipestat(dev_priv, 0, PIPE_GMBUS_EVENT_ENABLE);
2831         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2832
2833         /*
2834          * Enable some error detection, note the instruction error mask
2835          * bit is reserved, so we leave it masked.
2836          */
2837         if (IS_G4X(dev)) {
2838                 error_mask = ~(GM45_ERROR_PAGE_TABLE |
2839                                GM45_ERROR_MEM_PRIV |
2840                                GM45_ERROR_CP_PRIV |
2841                                I915_ERROR_MEMORY_REFRESH);
2842         } else {
2843                 error_mask = ~(I915_ERROR_PAGE_TABLE |
2844                                I915_ERROR_MEMORY_REFRESH);
2845         }
2846         I915_WRITE(EMR, error_mask);
2847
2848         I915_WRITE(IMR, dev_priv->irq_mask);
2849         I915_WRITE(IER, enable_mask);
2850         POSTING_READ(IER);
2851
2852         I915_WRITE(PORT_HOTPLUG_EN, 0);
2853         POSTING_READ(PORT_HOTPLUG_EN);
2854
2855         i915_enable_asle_pipestat(dev);
2856
2857         return 0;
2858 }
2859
2860 static void i915_hpd_irq_setup(struct drm_device *dev)
2861 {
2862         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2863         struct drm_mode_config *mode_config = &dev->mode_config;
2864         struct intel_encoder *intel_encoder;
2865         u32 hotplug_en;
2866
2867         assert_spin_locked(&dev_priv->irq_lock);
2868
2869         if (I915_HAS_HOTPLUG(dev)) {
2870                 hotplug_en = I915_READ(PORT_HOTPLUG_EN);
2871                 hotplug_en &= ~HOTPLUG_INT_EN_MASK;
2872                 /* Note HDMI and DP share hotplug bits */
2873                 /* enable bits are the same for all generations */
2874                 list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
2875                         if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
2876                                 hotplug_en |= hpd_mask_i915[intel_encoder->hpd_pin];
2877                 /* Programming the CRT detection parameters tends
2878                    to generate a spurious hotplug event about three
2879                    seconds later.  So just do it once.
2880                 */
2881                 if (IS_G4X(dev))
2882                         hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
2883                 hotplug_en &= ~CRT_HOTPLUG_VOLTAGE_COMPARE_MASK;
2884                 hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
2885
2886                 /* Ignore TV since it's buggy */
2887                 I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
2888         }
2889 }
2890
2891 static irqreturn_t i965_irq_handler(int irq, void *arg)
2892 {
2893         struct drm_device *dev = (struct drm_device *) arg;
2894         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2895         u32 iir, new_iir;
2896         u32 pipe_stats[I915_MAX_PIPES];
2897         unsigned long irqflags;
2898         int irq_received;
2899         int ret = IRQ_NONE, pipe;
2900         u32 flip_mask =
2901                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2902                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
2903
2904         atomic_inc(&dev_priv->irq_received);
2905
2906         iir = I915_READ(IIR);
2907
2908         for (;;) {
2909                 bool blc_event = false;
2910
2911                 irq_received = (iir & ~flip_mask) != 0;
2912
2913                 /* Can't rely on pipestat interrupt bit in iir as it might
2914                  * have been cleared after the pipestat interrupt was received.
2915                  * It doesn't set the bit in iir again, but it still produces
2916                  * interrupts (for non-MSI).
2917                  */
2918                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2919                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
2920                         i915_handle_error(dev, false);
2921
2922                 for_each_pipe(pipe) {
2923                         int reg = PIPESTAT(pipe);
2924                         pipe_stats[pipe] = I915_READ(reg);
2925
2926                         /*
2927                          * Clear the PIPE*STAT regs before the IIR
2928                          */
2929                         if (pipe_stats[pipe] & 0x8000ffff) {
2930                                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
2931                                         DRM_DEBUG_DRIVER("pipe %c underrun\n",
2932                                                          pipe_name(pipe));
2933                                 I915_WRITE(reg, pipe_stats[pipe]);
2934                                 irq_received = 1;
2935                         }
2936                 }
2937                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2938
2939                 if (!irq_received)
2940                         break;
2941
2942                 ret = IRQ_HANDLED;
2943
2944                 /* Consume port.  Then clear IIR or we'll miss events */
2945                 if (iir & I915_DISPLAY_PORT_INTERRUPT) {
2946                         u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
2947                         u32 hotplug_trigger = hotplug_status & (IS_G4X(dev) ?
2948                                                                   HOTPLUG_INT_STATUS_G4X :
2949                                                                   HOTPLUG_INT_STATUS_I915);
2950
2951                         DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
2952                                   hotplug_status);
2953
2954                         intel_hpd_irq_handler(dev, hotplug_trigger,
2955                                               IS_G4X(dev) ? hpd_status_gen4 : hpd_status_i915);
2956
2957                         I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
2958                         I915_READ(PORT_HOTPLUG_STAT);
2959                 }
2960
2961                 I915_WRITE(IIR, iir & ~flip_mask);
2962                 new_iir = I915_READ(IIR); /* Flush posted writes */
2963
2964                 if (iir & I915_USER_INTERRUPT)
2965                         notify_ring(dev, &dev_priv->ring[RCS]);
2966                 if (iir & I915_BSD_USER_INTERRUPT)
2967                         notify_ring(dev, &dev_priv->ring[VCS]);
2968
2969                 for_each_pipe(pipe) {
2970                         if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
2971                             i915_handle_vblank(dev, pipe, pipe, iir))
2972                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
2973
2974                         if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
2975                                 blc_event = true;
2976                 }
2977
2978
2979                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
2980                         intel_opregion_asle_intr(dev);
2981
2982                 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
2983                         gmbus_irq_handler(dev);
2984
2985                 /* With MSI, interrupts are only generated when iir
2986                  * transitions from zero to nonzero.  If another bit got
2987                  * set while we were handling the existing iir bits, then
2988                  * we would never get another interrupt.
2989                  *
2990                  * This is fine on non-MSI as well, as if we hit this path
2991                  * we avoid exiting the interrupt handler only to generate
2992                  * another one.
2993                  *
2994                  * Note that for MSI this could cause a stray interrupt report
2995                  * if an interrupt landed in the time between writing IIR and
2996                  * the posting read.  This should be rare enough to never
2997                  * trigger the 99% of 100,000 interrupts test for disabling
2998                  * stray interrupts.
2999                  */
3000                 iir = new_iir;
3001         }
3002
3003         i915_update_dri1_breadcrumb(dev);
3004
3005         return ret;
3006 }
3007
3008 static void i965_irq_uninstall(struct drm_device * dev)
3009 {
3010         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
3011         int pipe;
3012
3013         if (!dev_priv)
3014                 return;
3015
3016         del_timer_sync(&dev_priv->hotplug_reenable_timer);
3017
3018         I915_WRITE(PORT_HOTPLUG_EN, 0);
3019         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3020
3021         I915_WRITE(HWSTAM, 0xffffffff);
3022         for_each_pipe(pipe)
3023                 I915_WRITE(PIPESTAT(pipe), 0);
3024         I915_WRITE(IMR, 0xffffffff);
3025         I915_WRITE(IER, 0x0);
3026
3027         for_each_pipe(pipe)
3028                 I915_WRITE(PIPESTAT(pipe),
3029                            I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
3030         I915_WRITE(IIR, I915_READ(IIR));
3031 }
3032
3033 static void i915_reenable_hotplug_timer_func(unsigned long data)
3034 {
3035         drm_i915_private_t *dev_priv = (drm_i915_private_t *)data;
3036         struct drm_device *dev = dev_priv->dev;
3037         struct drm_mode_config *mode_config = &dev->mode_config;
3038         unsigned long irqflags;
3039         int i;
3040
3041         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3042         for (i = (HPD_NONE + 1); i < HPD_NUM_PINS; i++) {
3043                 struct drm_connector *connector;
3044
3045                 if (dev_priv->hpd_stats[i].hpd_mark != HPD_DISABLED)
3046                         continue;
3047
3048                 dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;
3049
3050                 list_for_each_entry(connector, &mode_config->connector_list, head) {
3051                         struct intel_connector *intel_connector = to_intel_connector(connector);
3052
3053                         if (intel_connector->encoder->hpd_pin == i) {
3054                                 if (connector->polled != intel_connector->polled)
3055                                         DRM_DEBUG_DRIVER("Reenabling HPD on connector %s\n",
3056                                                          drm_get_connector_name(connector));
3057                                 connector->polled = intel_connector->polled;
3058                                 if (!connector->polled)
3059                                         connector->polled = DRM_CONNECTOR_POLL_HPD;
3060                         }
3061                 }
3062         }
3063         if (dev_priv->display.hpd_irq_setup)
3064                 dev_priv->display.hpd_irq_setup(dev);
3065         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3066 }
3067
3068 void intel_irq_init(struct drm_device *dev)
3069 {
3070         struct drm_i915_private *dev_priv = dev->dev_private;
3071
3072         INIT_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func);
3073         INIT_WORK(&dev_priv->gpu_error.work, i915_error_work_func);
3074         INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
3075         INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
3076
3077         setup_timer(&dev_priv->gpu_error.hangcheck_timer,
3078                     i915_hangcheck_elapsed,
3079                     (unsigned long) dev);
3080         setup_timer(&dev_priv->hotplug_reenable_timer, i915_reenable_hotplug_timer_func,
3081                     (unsigned long) dev_priv);
3082
3083         pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
3084
3085         dev->driver->get_vblank_counter = i915_get_vblank_counter;
3086         dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
3087         if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
3088                 dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
3089                 dev->driver->get_vblank_counter = gm45_get_vblank_counter;
3090         }
3091
3092         if (drm_core_check_feature(dev, DRIVER_MODESET))
3093                 dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
3094         else
3095                 dev->driver->get_vblank_timestamp = NULL;
3096         dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
3097
3098         if (IS_VALLEYVIEW(dev)) {
3099                 dev->driver->irq_handler = valleyview_irq_handler;
3100                 dev->driver->irq_preinstall = valleyview_irq_preinstall;
3101                 dev->driver->irq_postinstall = valleyview_irq_postinstall;
3102                 dev->driver->irq_uninstall = valleyview_irq_uninstall;
3103                 dev->driver->enable_vblank = valleyview_enable_vblank;
3104                 dev->driver->disable_vblank = valleyview_disable_vblank;
3105                 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
3106         } else if (HAS_PCH_SPLIT(dev)) {
3107                 dev->driver->irq_handler = ironlake_irq_handler;
3108                 dev->driver->irq_preinstall = ironlake_irq_preinstall;
3109                 dev->driver->irq_postinstall = ironlake_irq_postinstall;
3110                 dev->driver->irq_uninstall = ironlake_irq_uninstall;
3111                 dev->driver->enable_vblank = ironlake_enable_vblank;
3112                 dev->driver->disable_vblank = ironlake_disable_vblank;
3113                 dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
3114         } else {
3115                 if (INTEL_INFO(dev)->gen == 2) {
3116                         dev->driver->irq_preinstall = i8xx_irq_preinstall;
3117                         dev->driver->irq_postinstall = i8xx_irq_postinstall;
3118                         dev->driver->irq_handler = i8xx_irq_handler;
3119                         dev->driver->irq_uninstall = i8xx_irq_uninstall;
3120                 } else if (INTEL_INFO(dev)->gen == 3) {
3121                         dev->driver->irq_preinstall = i915_irq_preinstall;
3122                         dev->driver->irq_postinstall = i915_irq_postinstall;
3123                         dev->driver->irq_uninstall = i915_irq_uninstall;
3124                         dev->driver->irq_handler = i915_irq_handler;
3125                         dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
3126                 } else {
3127                         dev->driver->irq_preinstall = i965_irq_preinstall;
3128                         dev->driver->irq_postinstall = i965_irq_postinstall;
3129                         dev->driver->irq_uninstall = i965_irq_uninstall;
3130                         dev->driver->irq_handler = i965_irq_handler;
3131                         dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
3132                 }
3133                 dev->driver->enable_vblank = i915_enable_vblank;
3134                 dev->driver->disable_vblank = i915_disable_vblank;
3135         }
3136 }
3137
3138 void intel_hpd_init(struct drm_device *dev)
3139 {
3140         struct drm_i915_private *dev_priv = dev->dev_private;
3141         struct drm_mode_config *mode_config = &dev->mode_config;
3142         struct drm_connector *connector;
3143         unsigned long irqflags;
3144         int i;
3145
3146         for (i = 1; i < HPD_NUM_PINS; i++) {
3147                 dev_priv->hpd_stats[i].hpd_cnt = 0;
3148                 dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;
3149         }
3150         list_for_each_entry(connector, &mode_config->connector_list, head) {
3151                 struct intel_connector *intel_connector = to_intel_connector(connector);
3152                 connector->polled = intel_connector->polled;
3153                 if (!connector->polled && I915_HAS_HOTPLUG(dev) && intel_connector->encoder->hpd_pin > HPD_NONE)
3154                         connector->polled = DRM_CONNECTOR_POLL_HPD;
3155         }
3156
3157         /* Interrupt setup is already guaranteed to be single-threaded, this is
3158          * just to make the assert_spin_locked checks happy. */
3159         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3160         if (dev_priv->display.hpd_irq_setup)
3161                 dev_priv->display.hpd_irq_setup(dev);
3162         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3163 }
3164
3165 /* Disable interrupts so we can allow Package C8+. */
3166 void hsw_pc8_disable_interrupts(struct drm_device *dev)
3167 {
3168         struct drm_i915_private *dev_priv = dev->dev_private;
3169         unsigned long irqflags;
3170
3171         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3172
3173         dev_priv->pc8.regsave.deimr = I915_READ(DEIMR);
3174         dev_priv->pc8.regsave.sdeimr = I915_READ(SDEIMR);
3175         dev_priv->pc8.regsave.gtimr = I915_READ(GTIMR);
3176         dev_priv->pc8.regsave.gtier = I915_READ(GTIER);
3177         dev_priv->pc8.regsave.gen6_pmimr = I915_READ(GEN6_PMIMR);
3178
3179         ironlake_disable_display_irq(dev_priv, ~DE_PCH_EVENT_IVB);
3180         ibx_disable_display_interrupt(dev_priv, ~SDE_HOTPLUG_MASK_CPT);
3181         ilk_disable_gt_irq(dev_priv, 0xffffffff);
3182         snb_disable_pm_irq(dev_priv, 0xffffffff);
3183
3184         dev_priv->pc8.irqs_disabled = true;
3185
3186         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3187 }
3188
3189 /* Restore interrupts so we can recover from Package C8+. */
3190 void hsw_pc8_restore_interrupts(struct drm_device *dev)
3191 {
3192         struct drm_i915_private *dev_priv = dev->dev_private;
3193         unsigned long irqflags;
3194         uint32_t val, expected;
3195
3196         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3197
3198         val = I915_READ(DEIMR);
3199         expected = ~DE_PCH_EVENT_IVB;
3200         WARN(val != expected, "DEIMR is 0x%08x, not 0x%08x\n", val, expected);
3201
3202         val = I915_READ(SDEIMR) & ~SDE_HOTPLUG_MASK_CPT;
3203         expected = ~SDE_HOTPLUG_MASK_CPT;
3204         WARN(val != expected, "SDEIMR non-HPD bits are 0x%08x, not 0x%08x\n",
3205              val, expected);
3206
3207         val = I915_READ(GTIMR);
3208         expected = 0xffffffff;
3209         WARN(val != expected, "GTIMR is 0x%08x, not 0x%08x\n", val, expected);
3210
3211         val = I915_READ(GEN6_PMIMR);
3212         expected = 0xffffffff;
3213         WARN(val != expected, "GEN6_PMIMR is 0x%08x, not 0x%08x\n", val,
3214              expected);
3215
3216         dev_priv->pc8.irqs_disabled = false;
3217
3218         ironlake_enable_display_irq(dev_priv, ~dev_priv->pc8.regsave.deimr);
3219         ibx_enable_display_interrupt(dev_priv,
3220                                      ~dev_priv->pc8.regsave.sdeimr &
3221                                      ~SDE_HOTPLUG_MASK_CPT);
3222         ilk_enable_gt_irq(dev_priv, ~dev_priv->pc8.regsave.gtimr);
3223         snb_enable_pm_irq(dev_priv, ~dev_priv->pc8.regsave.gen6_pmimr);
3224         I915_WRITE(GTIER, dev_priv->pc8.regsave.gtier);
3225
3226         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3227 }