]> git.karo-electronics.de Git - karo-tx-linux.git/blobdiff - drivers/gpu/drm/i915/intel_pm.c
drm/i915: implement WaDisableL3CacheAging on VLV
[karo-tx-linux.git] / drivers / gpu / drm / i915 / intel_pm.c
index 72f41aaa71ff330637bdc82d4ee8e0c04091bdf2..a9e2c546de9a1896559c8551d3bc73f6036f6ebe 100644 (file)
@@ -2404,10 +2404,10 @@ static void gen6_enable_rps(struct drm_device *dev)
        struct intel_ring_buffer *ring;
        u32 rp_state_cap;
        u32 gt_perf_status;
-       u32 pcu_mbox, rc6_mask = 0;
+       u32 rc6vids, pcu_mbox, rc6_mask = 0;
        u32 gtfifodbg;
        int rc6_mode;
-       int i;
+       int i, ret;
 
        WARN_ON(!mutex_is_locked(&dev->struct_mutex));
 
@@ -2503,30 +2503,16 @@ static void gen6_enable_rps(struct drm_device *dev)
                   GEN6_RP_UP_BUSY_AVG |
                   (IS_HASWELL(dev) ? GEN7_RP_DOWN_IDLE_AVG : GEN6_RP_DOWN_IDLE_CONT));
 
-       if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
-                    500))
-               DRM_ERROR("timeout waiting for pcode mailbox to become idle\n");
-
-       I915_WRITE(GEN6_PCODE_DATA, 0);
-       I915_WRITE(GEN6_PCODE_MAILBOX,
-                  GEN6_PCODE_READY |
-                  GEN6_PCODE_WRITE_MIN_FREQ_TABLE);
-       if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
-                    500))
-               DRM_ERROR("timeout waiting for pcode mailbox to finish\n");
-
-       /* Check for overclock support */
-       if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
-                    500))
-               DRM_ERROR("timeout waiting for pcode mailbox to become idle\n");
-       I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_READ_OC_PARAMS);
-       pcu_mbox = I915_READ(GEN6_PCODE_DATA);
-       if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
-                    500))
-               DRM_ERROR("timeout waiting for pcode mailbox to finish\n");
-       if (pcu_mbox & (1<<31)) { /* OC supported */
-               dev_priv->rps.max_delay = pcu_mbox & 0xff;
-               DRM_DEBUG_DRIVER("overclocking supported, adjusting frequency max to %dMHz\n", pcu_mbox * 50);
+       ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
+       if (!ret) {
+               pcu_mbox = 0;
+               ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
+               if (ret && pcu_mbox & (1<<31)) { /* OC supported */
+                       dev_priv->rps.max_delay = pcu_mbox & 0xff;
+                       DRM_DEBUG_DRIVER("overclocking supported, adjusting frequency max to %dMHz\n", pcu_mbox * 50);
+               }
+       } else {
+               DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
        }
 
        gen6_set_rps(dev_priv->dev, (gt_perf_status & 0xff00) >> 8);
@@ -2540,6 +2526,20 @@ static void gen6_enable_rps(struct drm_device *dev)
        /* enable all PM interrupts */
        I915_WRITE(GEN6_PMINTRMSK, 0);
 
+       rc6vids = 0;
+       ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
+       if (IS_GEN6(dev) && ret) {
+               DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
+       } else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
+               DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
+                         GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
+               rc6vids &= 0xffff00;
+               rc6vids |= GEN6_ENCODE_RC6_VID(450);
+               ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
+               if (ret)
+                       DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
+       }
+
        gen6_gt_force_wake_put(dev_priv);
 }
 
@@ -2581,17 +2581,11 @@ static void gen6_update_ring_freq(struct drm_device *dev)
                else
                        ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
                ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
+               ia_freq <<= GEN6_PCODE_FREQ_IA_RATIO_SHIFT;
 
-               I915_WRITE(GEN6_PCODE_DATA,
-                          (ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT) |
-                          gpu_freq);
-               I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY |
-                          GEN6_PCODE_WRITE_MIN_FREQ_TABLE);
-               if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) &
-                             GEN6_PCODE_READY) == 0, 10)) {
-                       DRM_ERROR("pcode write of freq table timed out\n");
-                       continue;
-               }
+               sandybridge_pcode_write(dev_priv,
+                                       GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
+                                       ia_freq | gpu_freq);
        }
 }
 
@@ -3330,17 +3324,27 @@ void intel_enable_gt_powersave(struct drm_device *dev)
        }
 }
 
+static void ibx_init_clock_gating(struct drm_device *dev)
+{
+       struct drm_i915_private *dev_priv = dev->dev_private;
+
+       /*
+        * On Ibex Peak and Cougar Point, we need to disable clock
+        * gating for the panel power sequencer or it will fail to
+        * start up when no ports are active.
+        */
+       I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
+}
+
 static void ironlake_init_clock_gating(struct drm_device *dev)
 {
        struct drm_i915_private *dev_priv = dev->dev_private;
-       uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
+       uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
 
        /* Required for FBC */
-       dspclk_gate |= DPFCUNIT_CLOCK_GATE_DISABLE |
-               DPFCRUNIT_CLOCK_GATE_DISABLE |
-               DPFDUNIT_CLOCK_GATE_DISABLE;
-       /* Required for CxSR */
-       dspclk_gate |= DPARBUNIT_CLOCK_GATE_DISABLE;
+       dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
+                  ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
+                  ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
 
        I915_WRITE(PCH_3DCGDIS0,
                   MARIUNIT_CLOCK_GATE_DISABLE |
@@ -3348,8 +3352,6 @@ static void ironlake_init_clock_gating(struct drm_device *dev)
        I915_WRITE(PCH_3DCGDIS1,
                   VFMUNIT_CLOCK_GATE_DISABLE);
 
-       I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
-
        /*
         * According to the spec the following bits should be set in
         * order to enable memory self-refresh
@@ -3360,9 +3362,7 @@ static void ironlake_init_clock_gating(struct drm_device *dev)
        I915_WRITE(ILK_DISPLAY_CHICKEN2,
                   (I915_READ(ILK_DISPLAY_CHICKEN2) |
                    ILK_DPARB_GATE | ILK_VSDPFD_FULL));
-       I915_WRITE(ILK_DSPCLK_GATE,
-                  (I915_READ(ILK_DSPCLK_GATE) |
-                   ILK_DPARB_CLK_GATE));
+       dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
        I915_WRITE(DISP_ARB_CTL,
                   (I915_READ(DISP_ARB_CTL) |
                    DISP_FBC_WM_DIS));
@@ -3384,28 +3384,51 @@ static void ironlake_init_clock_gating(struct drm_device *dev)
                I915_WRITE(ILK_DISPLAY_CHICKEN2,
                           I915_READ(ILK_DISPLAY_CHICKEN2) |
                           ILK_DPARB_GATE);
-               I915_WRITE(ILK_DSPCLK_GATE,
-                          I915_READ(ILK_DSPCLK_GATE) |
-                          ILK_DPFC_DIS1 |
-                          ILK_DPFC_DIS2 |
-                          ILK_CLK_FBC);
        }
 
+       I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
+
        I915_WRITE(ILK_DISPLAY_CHICKEN2,
                   I915_READ(ILK_DISPLAY_CHICKEN2) |
                   ILK_ELPIN_409_SELECT);
        I915_WRITE(_3D_CHICKEN2,
                   _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
                   _3D_CHICKEN2_WM_READ_PIPELINED);
+
+       /* WaDisableRenderCachePipelinedFlush */
+       I915_WRITE(CACHE_MODE_0,
+                  _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
+
+       ibx_init_clock_gating(dev);
+}
+
+static void cpt_init_clock_gating(struct drm_device *dev)
+{
+       struct drm_i915_private *dev_priv = dev->dev_private;
+       int pipe;
+
+       /*
+        * On Ibex Peak and Cougar Point, we need to disable clock
+        * gating for the panel power sequencer or it will fail to
+        * start up when no ports are active.
+        */
+       I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
+       I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
+                  DPLS_EDP_PPS_FIX_DIS);
+       /* WADP0ClockGatingDisable */
+       for_each_pipe(pipe) {
+               I915_WRITE(TRANS_CHICKEN1(pipe),
+                          TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
+       }
 }
 
 static void gen6_init_clock_gating(struct drm_device *dev)
 {
        struct drm_i915_private *dev_priv = dev->dev_private;
        int pipe;
-       uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
+       uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
 
-       I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
+       I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
 
        I915_WRITE(ILK_DISPLAY_CHICKEN2,
                   I915_READ(ILK_DISPLAY_CHICKEN2) |
@@ -3460,10 +3483,10 @@ static void gen6_init_clock_gating(struct drm_device *dev)
        I915_WRITE(ILK_DISPLAY_CHICKEN2,
                   I915_READ(ILK_DISPLAY_CHICKEN2) |
                   ILK_DPARB_GATE | ILK_VSDPFD_FULL);
-       I915_WRITE(ILK_DSPCLK_GATE,
-                  I915_READ(ILK_DSPCLK_GATE) |
-                  ILK_DPARB_CLK_GATE  |
-                  ILK_DPFD_CLK_GATE);
+       I915_WRITE(ILK_DSPCLK_GATE_D,
+                  I915_READ(ILK_DSPCLK_GATE_D) |
+                  ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
+                  ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
 
        I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) |
                   GEN6_MBCTL_ENABLE_BOOT_FETCH);
@@ -3479,6 +3502,8 @@ static void gen6_init_clock_gating(struct drm_device *dev)
         * platforms I checked have a 0 for this. (Maybe BIOS overrides?) */
        I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_DISABLE(0xffff));
        I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_ENABLE(GEN6_GT_MODE_HI));
+
+       cpt_init_clock_gating(dev);
 }
 
 static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
@@ -3497,9 +3522,6 @@ static void haswell_init_clock_gating(struct drm_device *dev)
 {
        struct drm_i915_private *dev_priv = dev->dev_private;
        int pipe;
-       uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
-
-       I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
 
        I915_WRITE(WM3_LP_ILK, 0);
        I915_WRITE(WM2_LP_ILK, 0);
@@ -3510,12 +3532,6 @@ static void haswell_init_clock_gating(struct drm_device *dev)
         */
        I915_WRITE(GEN6_UCGCTL2, GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
 
-       I915_WRITE(ILK_DSPCLK_GATE, IVB_VRHUNIT_CLK_GATE);
-
-       I915_WRITE(IVB_CHICKEN3,
-                  CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
-                  CHICKEN3_DGMG_DONE_FIX_DISABLE);
-
        /* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
        I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
                   GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
@@ -3559,17 +3575,19 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
 {
        struct drm_i915_private *dev_priv = dev->dev_private;
        int pipe;
-       uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
        uint32_t snpcr;
 
-       I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
-
        I915_WRITE(WM3_LP_ILK, 0);
        I915_WRITE(WM2_LP_ILK, 0);
        I915_WRITE(WM1_LP_ILK, 0);
 
-       I915_WRITE(ILK_DSPCLK_GATE, IVB_VRHUNIT_CLK_GATE);
+       I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
+
+       /* WaDisableEarlyCull */
+       I915_WRITE(_3D_CHICKEN3,
+                  _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
 
+       /* WaDisableBackToBackFlipFix */
        I915_WRITE(IVB_CHICKEN3,
                   CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
                   CHICKEN3_DGMG_DONE_FIX_DISABLE);
@@ -3584,6 +3602,10 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
        I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
                        GEN7_WA_L3_CHICKEN_MODE);
 
+       /* WaForceL3Serialization */
+       I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
+                  ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
+
        /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
         * gating disable must be set.  Failure to set it results in
         * flickering pixels due to Z write ordering failures after
@@ -3626,22 +3648,26 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
        snpcr &= ~GEN6_MBC_SNPCR_MASK;
        snpcr |= GEN6_MBC_SNPCR_MED;
        I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
+
+       cpt_init_clock_gating(dev);
 }
 
 static void valleyview_init_clock_gating(struct drm_device *dev)
 {
        struct drm_i915_private *dev_priv = dev->dev_private;
        int pipe;
-       uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
-
-       I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
 
        I915_WRITE(WM3_LP_ILK, 0);
        I915_WRITE(WM2_LP_ILK, 0);
        I915_WRITE(WM1_LP_ILK, 0);
 
-       I915_WRITE(ILK_DSPCLK_GATE, IVB_VRHUNIT_CLK_GATE);
+       I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
+
+       /* WaDisableEarlyCull */
+       I915_WRITE(_3D_CHICKEN3,
+                  _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
 
+       /* WaDisableBackToBackFlipFix */
        I915_WRITE(IVB_CHICKEN3,
                   CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
                   CHICKEN3_DGMG_DONE_FIX_DISABLE);
@@ -3651,9 +3677,13 @@ static void valleyview_init_clock_gating(struct drm_device *dev)
                   GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
 
        /* WaApplyL3ControlAndL3ChickenMode requires those two on Ivy Bridge */
-       I915_WRITE(GEN7_L3CNTLREG1, GEN7_WA_FOR_GEN7_L3_CONTROL);
+       I915_WRITE(GEN7_L3CNTLREG1, I915_READ(GEN7_L3CNTLREG1) | GEN7_L3AGDIS);
        I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
 
+       /* WaForceL3Serialization */
+       I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
+                  ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
+
        /* This is required by WaCatErrorRejectionIssue */
        I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
                   I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
@@ -3728,6 +3758,10 @@ static void g4x_init_clock_gating(struct drm_device *dev)
        if (IS_GM45(dev))
                dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
        I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
+
+       /* WaDisableRenderCachePipelinedFlush */
+       I915_WRITE(CACHE_MODE_0,
+                  _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
 }
 
 static void crestline_init_clock_gating(struct drm_device *dev)
@@ -3783,44 +3817,11 @@ static void i830_init_clock_gating(struct drm_device *dev)
        I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
 }
 
-static void ibx_init_clock_gating(struct drm_device *dev)
-{
-       struct drm_i915_private *dev_priv = dev->dev_private;
-
-       /*
-        * On Ibex Peak and Cougar Point, we need to disable clock
-        * gating for the panel power sequencer or it will fail to
-        * start up when no ports are active.
-        */
-       I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
-}
-
-static void cpt_init_clock_gating(struct drm_device *dev)
-{
-       struct drm_i915_private *dev_priv = dev->dev_private;
-       int pipe;
-
-       /*
-        * On Ibex Peak and Cougar Point, we need to disable clock
-        * gating for the panel power sequencer or it will fail to
-        * start up when no ports are active.
-        */
-       I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
-       I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
-                  DPLS_EDP_PPS_FIX_DIS);
-       /* Without this, mode sets may fail silently on FDI */
-       for_each_pipe(pipe)
-               I915_WRITE(TRANS_CHICKEN2(pipe), TRANS_AUTOTRAIN_GEN_STALL_DIS);
-}
-
 void intel_init_clock_gating(struct drm_device *dev)
 {
        struct drm_i915_private *dev_priv = dev->dev_private;
 
        dev_priv->display.init_clock_gating(dev);
-
-       if (dev_priv->display.init_pch_clock_gating)
-               dev_priv->display.init_pch_clock_gating(dev);
 }
 
 /* Starting with Haswell, we have different power wells for
@@ -3846,7 +3847,7 @@ void intel_init_power_wells(struct drm_device *dev)
 
                if ((well & HSW_PWR_WELL_STATE) == 0) {
                        I915_WRITE(power_wells[i], well & HSW_PWR_WELL_ENABLE);
-                       if (wait_for(I915_READ(power_wells[i] & HSW_PWR_WELL_STATE), 20))
+                       if (wait_for((I915_READ(power_wells[i]) & HSW_PWR_WELL_STATE), 20))
                                DRM_ERROR("Error enabling power well %lx\n", power_wells[i]);
                }
        }
@@ -3884,11 +3885,6 @@ void intel_init_pm(struct drm_device *dev)
 
        /* For FIFO watermark updates */
        if (HAS_PCH_SPLIT(dev)) {
-               if (HAS_PCH_IBX(dev))
-                       dev_priv->display.init_pch_clock_gating = ibx_init_clock_gating;
-               else if (HAS_PCH_CPT(dev))
-                       dev_priv->display.init_pch_clock_gating = cpt_init_clock_gating;
-
                if (IS_GEN5(dev)) {
                        if (I915_READ(MLTR_ILK) & ILK_SRLT_MASK)
                                dev_priv->display.update_wm = ironlake_update_wm;
@@ -3999,6 +3995,12 @@ static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv)
                DRM_ERROR("GT thread status wait timed out\n");
 }
 
+static void __gen6_gt_force_wake_reset(struct drm_i915_private *dev_priv)
+{
+       I915_WRITE_NOTRACE(FORCEWAKE, 0);
+       POSTING_READ(ECOBUS); /* something from same cacheline, but !FORCEWAKE */
+}
+
 static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
 {
        u32 forcewake_ack;
@@ -4012,7 +4014,7 @@ static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
                            FORCEWAKE_ACK_TIMEOUT_MS))
                DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
 
-       I915_WRITE_NOTRACE(FORCEWAKE, 1);
+       I915_WRITE_NOTRACE(FORCEWAKE, FORCEWAKE_KERNEL);
        POSTING_READ(ECOBUS); /* something from same cacheline, but !FORCEWAKE */
 
        if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & 1),
@@ -4022,6 +4024,12 @@ static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
        __gen6_gt_wait_for_thread_c0(dev_priv);
 }
 
+static void __gen6_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv)
+{
+       I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_DISABLE(0xffff));
+       POSTING_READ(ECOBUS); /* something from same cacheline, but !FORCEWAKE */
+}
+
 static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv)
 {
        u32 forcewake_ack;
@@ -4035,7 +4043,7 @@ static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv)
                            FORCEWAKE_ACK_TIMEOUT_MS))
                DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
 
-       I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_ENABLE(1));
+       I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
        POSTING_READ(ECOBUS); /* something from same cacheline, but !FORCEWAKE */
 
        if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & 1),
@@ -4079,7 +4087,7 @@ static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
 
 static void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv)
 {
-       I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_DISABLE(1));
+       I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
        /* gen6_gt_check_fifodbg doubles as the POSTING_READ */
        gen6_gt_check_fifodbg(dev_priv);
 }
@@ -4117,13 +4125,18 @@ int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
        return ret;
 }
 
+static void vlv_force_wake_reset(struct drm_i915_private *dev_priv)
+{
+       I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_DISABLE(0xffff));
+}
+
 static void vlv_force_wake_get(struct drm_i915_private *dev_priv)
 {
        if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK_VLV) & 1) == 0,
                            FORCEWAKE_ACK_TIMEOUT_MS))
                DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
 
-       I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_ENABLE(1));
+       I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
 
        if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK_VLV) & 1),
                            FORCEWAKE_ACK_TIMEOUT_MS))
@@ -4134,49 +4147,87 @@ static void vlv_force_wake_get(struct drm_i915_private *dev_priv)
 
 static void vlv_force_wake_put(struct drm_i915_private *dev_priv)
 {
-       I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_DISABLE(1));
+       I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
        /* The below doubles as a POSTING_READ */
        gen6_gt_check_fifodbg(dev_priv);
 }
 
+void intel_gt_reset(struct drm_device *dev)
+{
+       struct drm_i915_private *dev_priv = dev->dev_private;
+
+       if (IS_VALLEYVIEW(dev)) {
+               vlv_force_wake_reset(dev_priv);
+       } else if (INTEL_INFO(dev)->gen >= 6) {
+               __gen6_gt_force_wake_reset(dev_priv);
+               if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
+                       __gen6_gt_force_wake_mt_reset(dev_priv);
+       }
+}
+
 void intel_gt_init(struct drm_device *dev)
 {
        struct drm_i915_private *dev_priv = dev->dev_private;
 
        spin_lock_init(&dev_priv->gt_lock);
 
+       intel_gt_reset(dev);
+
        if (IS_VALLEYVIEW(dev)) {
                dev_priv->gt.force_wake_get = vlv_force_wake_get;
                dev_priv->gt.force_wake_put = vlv_force_wake_put;
-       } else if (INTEL_INFO(dev)->gen >= 6) {
+       } else if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) {
+               dev_priv->gt.force_wake_get = __gen6_gt_force_wake_mt_get;
+               dev_priv->gt.force_wake_put = __gen6_gt_force_wake_mt_put;
+       } else if (IS_GEN6(dev)) {
                dev_priv->gt.force_wake_get = __gen6_gt_force_wake_get;
                dev_priv->gt.force_wake_put = __gen6_gt_force_wake_put;
+       }
+}
 
-               /* IVB configs may use multi-threaded forcewake */
-               if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) {
-                       u32 ecobus;
-
-                       /* A small trick here - if the bios hasn't configured
-                        * MT forcewake, and if the device is in RC6, then
-                        * force_wake_mt_get will not wake the device and the
-                        * ECOBUS read will return zero. Which will be
-                        * (correctly) interpreted by the test below as MT
-                        * forcewake being disabled.
-                        */
-                       mutex_lock(&dev->struct_mutex);
-                       __gen6_gt_force_wake_mt_get(dev_priv);
-                       ecobus = I915_READ_NOTRACE(ECOBUS);
-                       __gen6_gt_force_wake_mt_put(dev_priv);
-                       mutex_unlock(&dev->struct_mutex);
-
-                       if (ecobus & FORCEWAKE_MT_ENABLE) {
-                               DRM_DEBUG_KMS("Using MT version of forcewake\n");
-                               dev_priv->gt.force_wake_get =
-                                       __gen6_gt_force_wake_mt_get;
-                               dev_priv->gt.force_wake_put =
-                                       __gen6_gt_force_wake_mt_put;
-                       }
-               }
+int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u8 mbox, u32 *val)
+{
+       WARN_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex));
+
+       if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
+               DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
+               return -EAGAIN;
        }
+
+       I915_WRITE(GEN6_PCODE_DATA, *val);
+       I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
+
+       if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
+                    500)) {
+               DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
+               return -ETIMEDOUT;
+       }
+
+       *val = I915_READ(GEN6_PCODE_DATA);
+       I915_WRITE(GEN6_PCODE_DATA, 0);
+
+       return 0;
 }
 
+int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u8 mbox, u32 val)
+{
+       WARN_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex));
+
+       if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
+               DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
+               return -EAGAIN;
+       }
+
+       I915_WRITE(GEN6_PCODE_DATA, val);
+       I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
+
+       if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
+                    500)) {
+               DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
+               return -ETIMEDOUT;
+       }
+
+       I915_WRITE(GEN6_PCODE_DATA, 0);
+
+       return 0;
+}