drivers/gpu/drm/amd/amdgpu/cik.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include "drmP.h"
  28 #include "amdgpu.h"
  29 #include "amdgpu_atombios.h"
  30 #include "amdgpu_ih.h"
  31 #include "amdgpu_uvd.h"
  32 #include "amdgpu_vce.h"
  33 #include "cikd.h"
  34 #include "atom.h"
  35 #include "amd_pcie.h"
  36
  37 #include "cik.h"
  38 #include "gmc_v7_0.h"
  39 #include "cik_ih.h"
  40 #include "dce_v8_0.h"
  41 #include "gfx_v7_0.h"
  42 #include "cik_sdma.h"
  43 #include "uvd_v4_2.h"
  44 #include "vce_v2_0.h"
  45 #include "cik_dpm.h"
  46
  47 #include "uvd/uvd_4_2_d.h"
  48
  49 #include "smu/smu_7_0_1_d.h"
  50 #include "smu/smu_7_0_1_sh_mask.h"
  51
  52 #include "dce/dce_8_0_d.h"
  53 #include "dce/dce_8_0_sh_mask.h"
  54
  55 #include "bif/bif_4_1_d.h"
  56 #include "bif/bif_4_1_sh_mask.h"
  57
  58 #include "gca/gfx_7_2_d.h"
  59 #include "gca/gfx_7_2_enum.h"
  60 #include "gca/gfx_7_2_sh_mask.h"
  61
  62 #include "gmc/gmc_7_1_d.h"
  63 #include "gmc/gmc_7_1_sh_mask.h"
  64
  65 #include "oss/oss_2_0_d.h"
  66 #include "oss/oss_2_0_sh_mask.h"
  67
  68 #include "amdgpu_amdkfd.h"
  69 #include "amdgpu_powerplay.h"
  70
  71 /*
  72  * Indirect registers accessor
  73  */
  74 static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
  75 {
  76         unsigned long flags;
  77         u32 r;
  78
  79         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
  80         WREG32(mmPCIE_INDEX, reg);
  81         (void)RREG32(mmPCIE_INDEX);
  82         r = RREG32(mmPCIE_DATA);
  83         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
  84         return r;
  85 }
  86
  87 static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
  88 {
  89         unsigned long flags;
  90
  91         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
  92         WREG32(mmPCIE_INDEX, reg);
  93         (void)RREG32(mmPCIE_INDEX);
  94         WREG32(mmPCIE_DATA, v);
  95         (void)RREG32(mmPCIE_DATA);
  96         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
  97 }
  98
  99 static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
 100 {
 101         unsigned long flags;
 102         u32 r;
 103
 104         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 105         WREG32(mmSMC_IND_INDEX_0, (reg));
 106         r = RREG32(mmSMC_IND_DATA_0);
 107         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 108         return r;
 109 }
 110
 111 static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 112 {
 113         unsigned long flags;
 114
 115         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 116         WREG32(mmSMC_IND_INDEX_0, (reg));
 117         WREG32(mmSMC_IND_DATA_0, (v));
 118         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 119 }
 120
 121 static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
 122 {
 123         unsigned long flags;
 124         u32 r;
 125
 126         spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 127         WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 128         r = RREG32(mmUVD_CTX_DATA);
 129         spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 130         return r;
 131 }
 132
 133 static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 134 {
 135         unsigned long flags;
 136
 137         spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 138         WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 139         WREG32(mmUVD_CTX_DATA, (v));
 140         spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 141 }
 142
 143 static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
 144 {
 145         unsigned long flags;
 146         u32 r;
 147
 148         spin_lock_irqsave(&adev->didt_idx_lock, flags);
 149         WREG32(mmDIDT_IND_INDEX, (reg));
 150         r = RREG32(mmDIDT_IND_DATA);
 151         spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 152         return r;
 153 }
 154
 155 static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 156 {
 157         unsigned long flags;
 158
 159         spin_lock_irqsave(&adev->didt_idx_lock, flags);
 160         WREG32(mmDIDT_IND_INDEX, (reg));
 161         WREG32(mmDIDT_IND_DATA, (v));
 162         spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 163 }
 164
 165 static const u32 bonaire_golden_spm_registers[] =
 166 {
 167         0xc200, 0xe0ffffff, 0xe0000000
 168 };
 169
 170 static const u32 bonaire_golden_common_registers[] =
 171 {
 172         0x31dc, 0xffffffff, 0x00000800,
 173         0x31dd, 0xffffffff, 0x00000800,
 174         0x31e6, 0xffffffff, 0x00007fbf,
 175         0x31e7, 0xffffffff, 0x00007faf
 176 };
 177
 178 static const u32 bonaire_golden_registers[] =
 179 {
 180         0xcd5, 0x00000333, 0x00000333,
 181         0xcd4, 0x000c0fc0, 0x00040200,
 182         0x2684, 0x00010000, 0x00058208,
 183         0xf000, 0xffff1fff, 0x00140000,
 184         0xf080, 0xfdfc0fff, 0x00000100,
 185         0xf08d, 0x40000000, 0x40000200,
 186         0x260c, 0xffffffff, 0x00000000,
 187         0x260d, 0xf00fffff, 0x00000400,
 188         0x260e, 0x0002021c, 0x00020200,
 189         0x31e, 0x00000080, 0x00000000,
 190         0x16ec, 0x000000f0, 0x00000070,
 191         0x16f0, 0xf0311fff, 0x80300000,
 192         0x263e, 0x73773777, 0x12010001,
 193         0xd43, 0x00810000, 0x408af000,
 194         0x1c0c, 0x31000111, 0x00000011,
 195         0xbd2, 0x73773777, 0x12010001,
 196         0x883, 0x00007fb6, 0x0021a1b1,
 197         0x884, 0x00007fb6, 0x002021b1,
 198         0x860, 0x00007fb6, 0x00002191,
 199         0x886, 0x00007fb6, 0x002121b1,
 200         0x887, 0x00007fb6, 0x002021b1,
 201         0x877, 0x00007fb6, 0x00002191,
 202         0x878, 0x00007fb6, 0x00002191,
 203         0xd8a, 0x0000003f, 0x0000000a,
 204         0xd8b, 0x0000003f, 0x0000000a,
 205         0xab9, 0x00073ffe, 0x000022a2,
 206         0x903, 0x000007ff, 0x00000000,
 207         0x2285, 0xf000003f, 0x00000007,
 208         0x22fc, 0x00002001, 0x00000001,
 209         0x22c9, 0xffffffff, 0x00ffffff,
 210         0xc281, 0x0000ff0f, 0x00000000,
 211         0xa293, 0x07ffffff, 0x06000000,
 212         0x136, 0x00000fff, 0x00000100,
 213         0xf9e, 0x00000001, 0x00000002,
 214         0x2440, 0x03000000, 0x0362c688,
 215         0x2300, 0x000000ff, 0x00000001,
 216         0x390, 0x00001fff, 0x00001fff,
 217         0x2418, 0x0000007f, 0x00000020,
 218         0x2542, 0x00010000, 0x00010000,
 219         0x2b05, 0x000003ff, 0x000000f3,
 220         0x2b03, 0xffffffff, 0x00001032
 221 };
 222
 223 static const u32 bonaire_mgcg_cgcg_init[] =
 224 {
 225         0x3108, 0xffffffff, 0xfffffffc,
 226         0xc200, 0xffffffff, 0xe0000000,
 227         0xf0a8, 0xffffffff, 0x00000100,
 228         0xf082, 0xffffffff, 0x00000100,
 229         0xf0b0, 0xffffffff, 0xc0000100,
 230         0xf0b2, 0xffffffff, 0xc0000100,
 231         0xf0b1, 0xffffffff, 0xc0000100,
 232         0x1579, 0xffffffff, 0x00600100,
 233         0xf0a0, 0xffffffff, 0x00000100,
 234         0xf085, 0xffffffff, 0x06000100,
 235         0xf088, 0xffffffff, 0x00000100,
 236         0xf086, 0xffffffff, 0x06000100,
 237         0xf081, 0xffffffff, 0x00000100,
 238         0xf0b8, 0xffffffff, 0x00000100,
 239         0xf089, 0xffffffff, 0x00000100,
 240         0xf080, 0xffffffff, 0x00000100,
 241         0xf08c, 0xffffffff, 0x00000100,
 242         0xf08d, 0xffffffff, 0x00000100,
 243         0xf094, 0xffffffff, 0x00000100,
 244         0xf095, 0xffffffff, 0x00000100,
 245         0xf096, 0xffffffff, 0x00000100,
 246         0xf097, 0xffffffff, 0x00000100,
 247         0xf098, 0xffffffff, 0x00000100,
 248         0xf09f, 0xffffffff, 0x00000100,
 249         0xf09e, 0xffffffff, 0x00000100,
 250         0xf084, 0xffffffff, 0x06000100,
 251         0xf0a4, 0xffffffff, 0x00000100,
 252         0xf09d, 0xffffffff, 0x00000100,
 253         0xf0ad, 0xffffffff, 0x00000100,
 254         0xf0ac, 0xffffffff, 0x00000100,
 255         0xf09c, 0xffffffff, 0x00000100,
 256         0xc200, 0xffffffff, 0xe0000000,
 257         0xf008, 0xffffffff, 0x00010000,
 258         0xf009, 0xffffffff, 0x00030002,
 259         0xf00a, 0xffffffff, 0x00040007,
 260         0xf00b, 0xffffffff, 0x00060005,
 261         0xf00c, 0xffffffff, 0x00090008,
 262         0xf00d, 0xffffffff, 0x00010000,
 263         0xf00e, 0xffffffff, 0x00030002,
 264         0xf00f, 0xffffffff, 0x00040007,
 265         0xf010, 0xffffffff, 0x00060005,
 266         0xf011, 0xffffffff, 0x00090008,
 267         0xf012, 0xffffffff, 0x00010000,
 268         0xf013, 0xffffffff, 0x00030002,
 269         0xf014, 0xffffffff, 0x00040007,
 270         0xf015, 0xffffffff, 0x00060005,
 271         0xf016, 0xffffffff, 0x00090008,
 272         0xf017, 0xffffffff, 0x00010000,
 273         0xf018, 0xffffffff, 0x00030002,
 274         0xf019, 0xffffffff, 0x00040007,
 275         0xf01a, 0xffffffff, 0x00060005,
 276         0xf01b, 0xffffffff, 0x00090008,
 277         0xf01c, 0xffffffff, 0x00010000,
 278         0xf01d, 0xffffffff, 0x00030002,
 279         0xf01e, 0xffffffff, 0x00040007,
 280         0xf01f, 0xffffffff, 0x00060005,
 281         0xf020, 0xffffffff, 0x00090008,
 282         0xf021, 0xffffffff, 0x00010000,
 283         0xf022, 0xffffffff, 0x00030002,
 284         0xf023, 0xffffffff, 0x00040007,
 285         0xf024, 0xffffffff, 0x00060005,
 286         0xf025, 0xffffffff, 0x00090008,
 287         0xf026, 0xffffffff, 0x00010000,
 288         0xf027, 0xffffffff, 0x00030002,
 289         0xf028, 0xffffffff, 0x00040007,
 290         0xf029, 0xffffffff, 0x00060005,
 291         0xf02a, 0xffffffff, 0x00090008,
 292         0xf000, 0xffffffff, 0x96e00200,
 293         0x21c2, 0xffffffff, 0x00900100,
 294         0x3109, 0xffffffff, 0x0020003f,
 295         0xe, 0xffffffff, 0x0140001c,
 296         0xf, 0x000f0000, 0x000f0000,
 297         0x88, 0xffffffff, 0xc060000c,
 298         0x89, 0xc0000fff, 0x00000100,
 299         0x3e4, 0xffffffff, 0x00000100,
 300         0x3e6, 0x00000101, 0x00000000,
 301         0x82a, 0xffffffff, 0x00000104,
 302         0x1579, 0xff000fff, 0x00000100,
 303         0xc33, 0xc0000fff, 0x00000104,
 304         0x3079, 0x00000001, 0x00000001,
 305         0x3403, 0xff000ff0, 0x00000100,
 306         0x3603, 0xff000ff0, 0x00000100
 307 };
 308
 309 static const u32 spectre_golden_spm_registers[] =
 310 {
 311         0xc200, 0xe0ffffff, 0xe0000000
 312 };
 313
 314 static const u32 spectre_golden_common_registers[] =
 315 {
 316         0x31dc, 0xffffffff, 0x00000800,
 317         0x31dd, 0xffffffff, 0x00000800,
 318         0x31e6, 0xffffffff, 0x00007fbf,
 319         0x31e7, 0xffffffff, 0x00007faf
 320 };
 321
 322 static const u32 spectre_golden_registers[] =
 323 {
 324         0xf000, 0xffff1fff, 0x96940200,
 325         0xf003, 0xffff0001, 0xff000000,
 326         0xf080, 0xfffc0fff, 0x00000100,
 327         0x1bb6, 0x00010101, 0x00010000,
 328         0x260d, 0xf00fffff, 0x00000400,
 329         0x260e, 0xfffffffc, 0x00020200,
 330         0x16ec, 0x000000f0, 0x00000070,
 331         0x16f0, 0xf0311fff, 0x80300000,
 332         0x263e, 0x73773777, 0x12010001,
 333         0x26df, 0x00ff0000, 0x00fc0000,
 334         0xbd2, 0x73773777, 0x12010001,
 335         0x2285, 0xf000003f, 0x00000007,
 336         0x22c9, 0xffffffff, 0x00ffffff,
 337         0xa0d4, 0x3f3f3fff, 0x00000082,
 338         0xa0d5, 0x0000003f, 0x00000000,
 339         0xf9e, 0x00000001, 0x00000002,
 340         0x244f, 0xffff03df, 0x00000004,
 341         0x31da, 0x00000008, 0x00000008,
 342         0x2300, 0x000008ff, 0x00000800,
 343         0x2542, 0x00010000, 0x00010000,
 344         0x2b03, 0xffffffff, 0x54763210,
 345         0x853e, 0x01ff01ff, 0x00000002,
 346         0x8526, 0x007ff800, 0x00200000,
 347         0x8057, 0xffffffff, 0x00000f40,
 348         0xc24d, 0xffffffff, 0x00000001
 349 };
 350
 351 static const u32 spectre_mgcg_cgcg_init[] =
 352 {
 353         0x3108, 0xffffffff, 0xfffffffc,
 354         0xc200, 0xffffffff, 0xe0000000,
 355         0xf0a8, 0xffffffff, 0x00000100,
 356         0xf082, 0xffffffff, 0x00000100,
 357         0xf0b0, 0xffffffff, 0x00000100,
 358         0xf0b2, 0xffffffff, 0x00000100,
 359         0xf0b1, 0xffffffff, 0x00000100,
 360         0x1579, 0xffffffff, 0x00600100,
 361         0xf0a0, 0xffffffff, 0x00000100,
 362         0xf085, 0xffffffff, 0x06000100,
 363         0xf088, 0xffffffff, 0x00000100,
 364         0xf086, 0xffffffff, 0x06000100,
 365         0xf081, 0xffffffff, 0x00000100,
 366         0xf0b8, 0xffffffff, 0x00000100,
 367         0xf089, 0xffffffff, 0x00000100,
 368         0xf080, 0xffffffff, 0x00000100,
 369         0xf08c, 0xffffffff, 0x00000100,
 370         0xf08d, 0xffffffff, 0x00000100,
 371         0xf094, 0xffffffff, 0x00000100,
 372         0xf095, 0xffffffff, 0x00000100,
 373         0xf096, 0xffffffff, 0x00000100,
 374         0xf097, 0xffffffff, 0x00000100,
 375         0xf098, 0xffffffff, 0x00000100,
 376         0xf09f, 0xffffffff, 0x00000100,
 377         0xf09e, 0xffffffff, 0x00000100,
 378         0xf084, 0xffffffff, 0x06000100,
 379         0xf0a4, 0xffffffff, 0x00000100,
 380         0xf09d, 0xffffffff, 0x00000100,
 381         0xf0ad, 0xffffffff, 0x00000100,
 382         0xf0ac, 0xffffffff, 0x00000100,
 383         0xf09c, 0xffffffff, 0x00000100,
 384         0xc200, 0xffffffff, 0xe0000000,
 385         0xf008, 0xffffffff, 0x00010000,
 386         0xf009, 0xffffffff, 0x00030002,
 387         0xf00a, 0xffffffff, 0x00040007,
 388         0xf00b, 0xffffffff, 0x00060005,
 389         0xf00c, 0xffffffff, 0x00090008,
 390         0xf00d, 0xffffffff, 0x00010000,
 391         0xf00e, 0xffffffff, 0x00030002,
 392         0xf00f, 0xffffffff, 0x00040007,
 393         0xf010, 0xffffffff, 0x00060005,
 394         0xf011, 0xffffffff, 0x00090008,
 395         0xf012, 0xffffffff, 0x00010000,
 396         0xf013, 0xffffffff, 0x00030002,
 397         0xf014, 0xffffffff, 0x00040007,
 398         0xf015, 0xffffffff, 0x00060005,
 399         0xf016, 0xffffffff, 0x00090008,
 400         0xf017, 0xffffffff, 0x00010000,
 401         0xf018, 0xffffffff, 0x00030002,
 402         0xf019, 0xffffffff, 0x00040007,
 403         0xf01a, 0xffffffff, 0x00060005,
 404         0xf01b, 0xffffffff, 0x00090008,
 405         0xf01c, 0xffffffff, 0x00010000,
 406         0xf01d, 0xffffffff, 0x00030002,
 407         0xf01e, 0xffffffff, 0x00040007,
 408         0xf01f, 0xffffffff, 0x00060005,
 409         0xf020, 0xffffffff, 0x00090008,
 410         0xf021, 0xffffffff, 0x00010000,
 411         0xf022, 0xffffffff, 0x00030002,
 412         0xf023, 0xffffffff, 0x00040007,
 413         0xf024, 0xffffffff, 0x00060005,
 414         0xf025, 0xffffffff, 0x00090008,
 415         0xf026, 0xffffffff, 0x00010000,
 416         0xf027, 0xffffffff, 0x00030002,
 417         0xf028, 0xffffffff, 0x00040007,
 418         0xf029, 0xffffffff, 0x00060005,
 419         0xf02a, 0xffffffff, 0x00090008,
 420         0xf02b, 0xffffffff, 0x00010000,
 421         0xf02c, 0xffffffff, 0x00030002,
 422         0xf02d, 0xffffffff, 0x00040007,
 423         0xf02e, 0xffffffff, 0x00060005,
 424         0xf02f, 0xffffffff, 0x00090008,
 425         0xf000, 0xffffffff, 0x96e00200,
 426         0x21c2, 0xffffffff, 0x00900100,
 427         0x3109, 0xffffffff, 0x0020003f,
 428         0xe, 0xffffffff, 0x0140001c,
 429         0xf, 0x000f0000, 0x000f0000,
 430         0x88, 0xffffffff, 0xc060000c,
 431         0x89, 0xc0000fff, 0x00000100,
 432         0x3e4, 0xffffffff, 0x00000100,
 433         0x3e6, 0x00000101, 0x00000000,
 434         0x82a, 0xffffffff, 0x00000104,
 435         0x1579, 0xff000fff, 0x00000100,
 436         0xc33, 0xc0000fff, 0x00000104,
 437         0x3079, 0x00000001, 0x00000001,
 438         0x3403, 0xff000ff0, 0x00000100,
 439         0x3603, 0xff000ff0, 0x00000100
 440 };
 441
 442 static const u32 kalindi_golden_spm_registers[] =
 443 {
 444         0xc200, 0xe0ffffff, 0xe0000000
 445 };
 446
 447 static const u32 kalindi_golden_common_registers[] =
 448 {
 449         0x31dc, 0xffffffff, 0x00000800,
 450         0x31dd, 0xffffffff, 0x00000800,
 451         0x31e6, 0xffffffff, 0x00007fbf,
 452         0x31e7, 0xffffffff, 0x00007faf
 453 };
 454
 455 static const u32 kalindi_golden_registers[] =
 456 {
 457         0xf000, 0xffffdfff, 0x6e944040,
 458         0x1579, 0xff607fff, 0xfc000100,
 459         0xf088, 0xff000fff, 0x00000100,
 460         0xf089, 0xff000fff, 0x00000100,
 461         0xf080, 0xfffc0fff, 0x00000100,
 462         0x1bb6, 0x00010101, 0x00010000,
 463         0x260c, 0xffffffff, 0x00000000,
 464         0x260d, 0xf00fffff, 0x00000400,
 465         0x16ec, 0x000000f0, 0x00000070,
 466         0x16f0, 0xf0311fff, 0x80300000,
 467         0x263e, 0x73773777, 0x12010001,
 468         0x263f, 0xffffffff, 0x00000010,
 469         0x26df, 0x00ff0000, 0x00fc0000,
 470         0x200c, 0x00001f0f, 0x0000100a,
 471         0xbd2, 0x73773777, 0x12010001,
 472         0x902, 0x000fffff, 0x000c007f,
 473         0x2285, 0xf000003f, 0x00000007,
 474         0x22c9, 0x3fff3fff, 0x00ffcfff,
 475         0xc281, 0x0000ff0f, 0x00000000,
 476         0xa293, 0x07ffffff, 0x06000000,
 477         0x136, 0x00000fff, 0x00000100,
 478         0xf9e, 0x00000001, 0x00000002,
 479         0x31da, 0x00000008, 0x00000008,
 480         0x2300, 0x000000ff, 0x00000003,
 481         0x853e, 0x01ff01ff, 0x00000002,
 482         0x8526, 0x007ff800, 0x00200000,
 483         0x8057, 0xffffffff, 0x00000f40,
 484         0x2231, 0x001f3ae3, 0x00000082,
 485         0x2235, 0x0000001f, 0x00000010,
 486         0xc24d, 0xffffffff, 0x00000000
 487 };
 488
 489 static const u32 kalindi_mgcg_cgcg_init[] =
 490 {
 491         0x3108, 0xffffffff, 0xfffffffc,
 492         0xc200, 0xffffffff, 0xe0000000,
 493         0xf0a8, 0xffffffff, 0x00000100,
 494         0xf082, 0xffffffff, 0x00000100,
 495         0xf0b0, 0xffffffff, 0x00000100,
 496         0xf0b2, 0xffffffff, 0x00000100,
 497         0xf0b1, 0xffffffff, 0x00000100,
 498         0x1579, 0xffffffff, 0x00600100,
 499         0xf0a0, 0xffffffff, 0x00000100,
 500         0xf085, 0xffffffff, 0x06000100,
 501         0xf088, 0xffffffff, 0x00000100,
 502         0xf086, 0xffffffff, 0x06000100,
 503         0xf081, 0xffffffff, 0x00000100,
 504         0xf0b8, 0xffffffff, 0x00000100,
 505         0xf089, 0xffffffff, 0x00000100,
 506         0xf080, 0xffffffff, 0x00000100,
 507         0xf08c, 0xffffffff, 0x00000100,
 508         0xf08d, 0xffffffff, 0x00000100,
 509         0xf094, 0xffffffff, 0x00000100,
 510         0xf095, 0xffffffff, 0x00000100,
 511         0xf096, 0xffffffff, 0x00000100,
 512         0xf097, 0xffffffff, 0x00000100,
 513         0xf098, 0xffffffff, 0x00000100,
 514         0xf09f, 0xffffffff, 0x00000100,
 515         0xf09e, 0xffffffff, 0x00000100,
 516         0xf084, 0xffffffff, 0x06000100,
 517         0xf0a4, 0xffffffff, 0x00000100,
 518         0xf09d, 0xffffffff, 0x00000100,
 519         0xf0ad, 0xffffffff, 0x00000100,
 520         0xf0ac, 0xffffffff, 0x00000100,
 521         0xf09c, 0xffffffff, 0x00000100,
 522         0xc200, 0xffffffff, 0xe0000000,
 523         0xf008, 0xffffffff, 0x00010000,
 524         0xf009, 0xffffffff, 0x00030002,
 525         0xf00a, 0xffffffff, 0x00040007,
 526         0xf00b, 0xffffffff, 0x00060005,
 527         0xf00c, 0xffffffff, 0x00090008,
 528         0xf00d, 0xffffffff, 0x00010000,
 529         0xf00e, 0xffffffff, 0x00030002,
 530         0xf00f, 0xffffffff, 0x00040007,
 531         0xf010, 0xffffffff, 0x00060005,
 532         0xf011, 0xffffffff, 0x00090008,
 533         0xf000, 0xffffffff, 0x96e00200,
 534         0x21c2, 0xffffffff, 0x00900100,
 535         0x3109, 0xffffffff, 0x0020003f,
 536         0xe, 0xffffffff, 0x0140001c,
 537         0xf, 0x000f0000, 0x000f0000,
 538         0x88, 0xffffffff, 0xc060000c,
 539         0x89, 0xc0000fff, 0x00000100,
 540         0x82a, 0xffffffff, 0x00000104,
 541         0x1579, 0xff000fff, 0x00000100,
 542         0xc33, 0xc0000fff, 0x00000104,
 543         0x3079, 0x00000001, 0x00000001,
 544         0x3403, 0xff000ff0, 0x00000100,
 545         0x3603, 0xff000ff0, 0x00000100
 546 };
 547
 548 static const u32 hawaii_golden_spm_registers[] =
 549 {
 550         0xc200, 0xe0ffffff, 0xe0000000
 551 };
 552
 553 static const u32 hawaii_golden_common_registers[] =
 554 {
 555         0xc200, 0xffffffff, 0xe0000000,
 556         0xa0d4, 0xffffffff, 0x3a00161a,
 557         0xa0d5, 0xffffffff, 0x0000002e,
 558         0x2684, 0xffffffff, 0x00018208,
 559         0x263e, 0xffffffff, 0x12011003
 560 };
 561
 562 static const u32 hawaii_golden_registers[] =
 563 {
 564         0xcd5, 0x00000333, 0x00000333,
 565         0x2684, 0x00010000, 0x00058208,
 566         0x260c, 0xffffffff, 0x00000000,
 567         0x260d, 0xf00fffff, 0x00000400,
 568         0x260e, 0x0002021c, 0x00020200,
 569         0x31e, 0x00000080, 0x00000000,
 570         0x16ec, 0x000000f0, 0x00000070,
 571         0x16f0, 0xf0311fff, 0x80300000,
 572         0xd43, 0x00810000, 0x408af000,
 573         0x1c0c, 0x31000111, 0x00000011,
 574         0xbd2, 0x73773777, 0x12010001,
 575         0x848, 0x0000007f, 0x0000001b,
 576         0x877, 0x00007fb6, 0x00002191,
 577         0xd8a, 0x0000003f, 0x0000000a,
 578         0xd8b, 0x0000003f, 0x0000000a,
 579         0xab9, 0x00073ffe, 0x000022a2,
 580         0x903, 0x000007ff, 0x00000000,
 581         0x22fc, 0x00002001, 0x00000001,
 582         0x22c9, 0xffffffff, 0x00ffffff,
 583         0xc281, 0x0000ff0f, 0x00000000,
 584         0xa293, 0x07ffffff, 0x06000000,
 585         0xf9e, 0x00000001, 0x00000002,
 586         0x31da, 0x00000008, 0x00000008,
 587         0x31dc, 0x00000f00, 0x00000800,
 588         0x31dd, 0x00000f00, 0x00000800,
 589         0x31e6, 0x00ffffff, 0x00ff7fbf,
 590         0x31e7, 0x00ffffff, 0x00ff7faf,
 591         0x2300, 0x000000ff, 0x00000800,
 592         0x390, 0x00001fff, 0x00001fff,
 593         0x2418, 0x0000007f, 0x00000020,
 594         0x2542, 0x00010000, 0x00010000,
 595         0x2b80, 0x00100000, 0x000ff07c,
 596         0x2b05, 0x000003ff, 0x0000000f,
 597         0x2b04, 0xffffffff, 0x7564fdec,
 598         0x2b03, 0xffffffff, 0x3120b9a8,
 599         0x2b02, 0x20000000, 0x0f9c0000
 600 };
 601
 602 static const u32 hawaii_mgcg_cgcg_init[] =
 603 {
 604         0x3108, 0xffffffff, 0xfffffffd,
 605         0xc200, 0xffffffff, 0xe0000000,
 606         0xf0a8, 0xffffffff, 0x00000100,
 607         0xf082, 0xffffffff, 0x00000100,
 608         0xf0b0, 0xffffffff, 0x00000100,
 609         0xf0b2, 0xffffffff, 0x00000100,
 610         0xf0b1, 0xffffffff, 0x00000100,
 611         0x1579, 0xffffffff, 0x00200100,
 612         0xf0a0, 0xffffffff, 0x00000100,
 613         0xf085, 0xffffffff, 0x06000100,
 614         0xf088, 0xffffffff, 0x00000100,
 615         0xf086, 0xffffffff, 0x06000100,
 616         0xf081, 0xffffffff, 0x00000100,
 617         0xf0b8, 0xffffffff, 0x00000100,
 618         0xf089, 0xffffffff, 0x00000100,
 619         0xf080, 0xffffffff, 0x00000100,
 620         0xf08c, 0xffffffff, 0x00000100,
 621         0xf08d, 0xffffffff, 0x00000100,
 622         0xf094, 0xffffffff, 0x00000100,
 623         0xf095, 0xffffffff, 0x00000100,
 624         0xf096, 0xffffffff, 0x00000100,
 625         0xf097, 0xffffffff, 0x00000100,
 626         0xf098, 0xffffffff, 0x00000100,
 627         0xf09f, 0xffffffff, 0x00000100,
 628         0xf09e, 0xffffffff, 0x00000100,
 629         0xf084, 0xffffffff, 0x06000100,
 630         0xf0a4, 0xffffffff, 0x00000100,
 631         0xf09d, 0xffffffff, 0x00000100,
 632         0xf0ad, 0xffffffff, 0x00000100,
 633         0xf0ac, 0xffffffff, 0x00000100,
 634         0xf09c, 0xffffffff, 0x00000100,
 635         0xc200, 0xffffffff, 0xe0000000,
 636         0xf008, 0xffffffff, 0x00010000,
 637         0xf009, 0xffffffff, 0x00030002,
 638         0xf00a, 0xffffffff, 0x00040007,
 639         0xf00b, 0xffffffff, 0x00060005,
 640         0xf00c, 0xffffffff, 0x00090008,
 641         0xf00d, 0xffffffff, 0x00010000,
 642         0xf00e, 0xffffffff, 0x00030002,
 643         0xf00f, 0xffffffff, 0x00040007,
 644         0xf010, 0xffffffff, 0x00060005,
 645         0xf011, 0xffffffff, 0x00090008,
 646         0xf012, 0xffffffff, 0x00010000,
 647         0xf013, 0xffffffff, 0x00030002,
 648         0xf014, 0xffffffff, 0x00040007,
 649         0xf015, 0xffffffff, 0x00060005,
 650         0xf016, 0xffffffff, 0x00090008,
 651         0xf017, 0xffffffff, 0x00010000,
 652         0xf018, 0xffffffff, 0x00030002,
 653         0xf019, 0xffffffff, 0x00040007,
 654         0xf01a, 0xffffffff, 0x00060005,
 655         0xf01b, 0xffffffff, 0x00090008,
 656         0xf01c, 0xffffffff, 0x00010000,
 657         0xf01d, 0xffffffff, 0x00030002,
 658         0xf01e, 0xffffffff, 0x00040007,
 659         0xf01f, 0xffffffff, 0x00060005,
 660         0xf020, 0xffffffff, 0x00090008,
 661         0xf021, 0xffffffff, 0x00010000,
 662         0xf022, 0xffffffff, 0x00030002,
 663         0xf023, 0xffffffff, 0x00040007,
 664         0xf024, 0xffffffff, 0x00060005,
 665         0xf025, 0xffffffff, 0x00090008,
 666         0xf026, 0xffffffff, 0x00010000,
 667         0xf027, 0xffffffff, 0x00030002,
 668         0xf028, 0xffffffff, 0x00040007,
 669         0xf029, 0xffffffff, 0x00060005,
 670         0xf02a, 0xffffffff, 0x00090008,
 671         0xf02b, 0xffffffff, 0x00010000,
 672         0xf02c, 0xffffffff, 0x00030002,
 673         0xf02d, 0xffffffff, 0x00040007,
 674         0xf02e, 0xffffffff, 0x00060005,
 675         0xf02f, 0xffffffff, 0x00090008,
 676         0xf030, 0xffffffff, 0x00010000,
 677         0xf031, 0xffffffff, 0x00030002,
 678         0xf032, 0xffffffff, 0x00040007,
 679         0xf033, 0xffffffff, 0x00060005,
 680         0xf034, 0xffffffff, 0x00090008,
 681         0xf035, 0xffffffff, 0x00010000,
 682         0xf036, 0xffffffff, 0x00030002,
 683         0xf037, 0xffffffff, 0x00040007,
 684         0xf038, 0xffffffff, 0x00060005,
 685         0xf039, 0xffffffff, 0x00090008,
 686         0xf03a, 0xffffffff, 0x00010000,
 687         0xf03b, 0xffffffff, 0x00030002,
 688         0xf03c, 0xffffffff, 0x00040007,
 689         0xf03d, 0xffffffff, 0x00060005,
 690         0xf03e, 0xffffffff, 0x00090008,
 691         0x30c6, 0xffffffff, 0x00020200,
 692         0xcd4, 0xffffffff, 0x00000200,
 693         0x570, 0xffffffff, 0x00000400,
 694         0x157a, 0xffffffff, 0x00000000,
 695         0xbd4, 0xffffffff, 0x00000902,
 696         0xf000, 0xffffffff, 0x96940200,
 697         0x21c2, 0xffffffff, 0x00900100,
 698         0x3109, 0xffffffff, 0x0020003f,
 699         0xe, 0xffffffff, 0x0140001c,
 700         0xf, 0x000f0000, 0x000f0000,
 701         0x88, 0xffffffff, 0xc060000c,
 702         0x89, 0xc0000fff, 0x00000100,
 703         0x3e4, 0xffffffff, 0x00000100,
 704         0x3e6, 0x00000101, 0x00000000,
 705         0x82a, 0xffffffff, 0x00000104,
 706         0x1579, 0xff000fff, 0x00000100,
 707         0xc33, 0xc0000fff, 0x00000104,
 708         0x3079, 0x00000001, 0x00000001,
 709         0x3403, 0xff000ff0, 0x00000100,
 710         0x3603, 0xff000ff0, 0x00000100
 711 };
 712
 713 static const u32 godavari_golden_registers[] =
 714 {
 715         0x1579, 0xff607fff, 0xfc000100,
 716         0x1bb6, 0x00010101, 0x00010000,
 717         0x260c, 0xffffffff, 0x00000000,
 718         0x260c0, 0xf00fffff, 0x00000400,
 719         0x184c, 0xffffffff, 0x00010000,
 720         0x16ec, 0x000000f0, 0x00000070,
 721         0x16f0, 0xf0311fff, 0x80300000,
 722         0x263e, 0x73773777, 0x12010001,
 723         0x263f, 0xffffffff, 0x00000010,
 724         0x200c, 0x00001f0f, 0x0000100a,
 725         0xbd2, 0x73773777, 0x12010001,
 726         0x902, 0x000fffff, 0x000c007f,
 727         0x2285, 0xf000003f, 0x00000007,
 728         0x22c9, 0xffffffff, 0x00ff0fff,
 729         0xc281, 0x0000ff0f, 0x00000000,
 730         0xa293, 0x07ffffff, 0x06000000,
 731         0x136, 0x00000fff, 0x00000100,
 732         0x3405, 0x00010000, 0x00810001,
 733         0x3605, 0x00010000, 0x00810001,
 734         0xf9e, 0x00000001, 0x00000002,
 735         0x31da, 0x00000008, 0x00000008,
 736         0x31dc, 0x00000f00, 0x00000800,
 737         0x31dd, 0x00000f00, 0x00000800,
 738         0x31e6, 0x00ffffff, 0x00ff7fbf,
 739         0x31e7, 0x00ffffff, 0x00ff7faf,
 740         0x2300, 0x000000ff, 0x00000001,
 741         0x853e, 0x01ff01ff, 0x00000002,
 742         0x8526, 0x007ff800, 0x00200000,
 743         0x8057, 0xffffffff, 0x00000f40,
 744         0x2231, 0x001f3ae3, 0x00000082,
 745         0x2235, 0x0000001f, 0x00000010,
 746         0xc24d, 0xffffffff, 0x00000000
 747 };
 748
 749 static void cik_init_golden_registers(struct amdgpu_device *adev)
 750 {
 751         /* Some of the registers might be dependent on GRBM_GFX_INDEX */
 752         mutex_lock(&adev->grbm_idx_mutex);
 753
 754         switch (adev->asic_type) {
 755         case CHIP_BONAIRE:
 756                 amdgpu_program_register_sequence(adev,
 757                                                  bonaire_mgcg_cgcg_init,
 758                                                  (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
 759                 amdgpu_program_register_sequence(adev,
 760                                                  bonaire_golden_registers,
 761                                                  (const u32)ARRAY_SIZE(bonaire_golden_registers));
 762                 amdgpu_program_register_sequence(adev,
 763                                                  bonaire_golden_common_registers,
 764                                                  (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
 765                 amdgpu_program_register_sequence(adev,
 766                                                  bonaire_golden_spm_registers,
 767                                                  (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
 768                 break;
 769         case CHIP_KABINI:
 770                 amdgpu_program_register_sequence(adev,
 771                                                  kalindi_mgcg_cgcg_init,
 772                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 773                 amdgpu_program_register_sequence(adev,
 774                                                  kalindi_golden_registers,
 775                                                  (const u32)ARRAY_SIZE(kalindi_golden_registers));
 776                 amdgpu_program_register_sequence(adev,
 777                                                  kalindi_golden_common_registers,
 778                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
 779                 amdgpu_program_register_sequence(adev,
 780                                                  kalindi_golden_spm_registers,
 781                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
 782                 break;
 783         case CHIP_MULLINS:
 784                 amdgpu_program_register_sequence(adev,
 785                                                  kalindi_mgcg_cgcg_init,
 786                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 787                 amdgpu_program_register_sequence(adev,
 788                                                  godavari_golden_registers,
 789                                                  (const u32)ARRAY_SIZE(godavari_golden_registers));
 790                 amdgpu_program_register_sequence(adev,
 791                                                  kalindi_golden_common_registers,
 792                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
 793                 amdgpu_program_register_sequence(adev,
 794                                                  kalindi_golden_spm_registers,
 795                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
 796                 break;
 797         case CHIP_KAVERI:
 798                 amdgpu_program_register_sequence(adev,
 799                                                  spectre_mgcg_cgcg_init,
 800                                                  (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
 801                 amdgpu_program_register_sequence(adev,
 802                                                  spectre_golden_registers,
 803                                                  (const u32)ARRAY_SIZE(spectre_golden_registers));
 804                 amdgpu_program_register_sequence(adev,
 805                                                  spectre_golden_common_registers,
 806                                                  (const u32)ARRAY_SIZE(spectre_golden_common_registers));
 807                 amdgpu_program_register_sequence(adev,
 808                                                  spectre_golden_spm_registers,
 809                                                  (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
 810                 break;
 811         case CHIP_HAWAII:
 812                 amdgpu_program_register_sequence(adev,
 813                                                  hawaii_mgcg_cgcg_init,
 814                                                  (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
 815                 amdgpu_program_register_sequence(adev,
 816                                                  hawaii_golden_registers,
 817                                                  (const u32)ARRAY_SIZE(hawaii_golden_registers));
 818                 amdgpu_program_register_sequence(adev,
 819                                                  hawaii_golden_common_registers,
 820                                                  (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
 821                 amdgpu_program_register_sequence(adev,
 822                                                  hawaii_golden_spm_registers,
 823                                                  (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
 824                 break;
 825         default:
 826                 break;
 827         }
 828         mutex_unlock(&adev->grbm_idx_mutex);
 829 }
 830
 831 /**
 832  * cik_get_xclk - get the xclk
 833  *
 834  * @adev: amdgpu_device pointer
 835  *
 836  * Returns the reference clock used by the gfx engine
 837  * (CIK).
 838  */
 839 static u32 cik_get_xclk(struct amdgpu_device *adev)
 840 {
 841         u32 reference_clock = adev->clock.spll.reference_freq;
 842
 843         if (adev->flags & AMD_IS_APU) {
 844                 if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
 845                         return reference_clock / 2;
 846         } else {
 847                 if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
 848                         return reference_clock / 4;
 849         }
 850         return reference_clock;
 851 }
 852
 853 /**
 854  * cik_srbm_select - select specific register instances
 855  *
 856  * @adev: amdgpu_device pointer
 857  * @me: selected ME (micro engine)
 858  * @pipe: pipe
 859  * @queue: queue
 860  * @vmid: VMID
 861  *
 862  * Switches the currently active registers instances.  Some
 863  * registers are instanced per VMID, others are instanced per
 864  * me/pipe/queue combination.
 865  */
 866 void cik_srbm_select(struct amdgpu_device *adev,
 867                      u32 me, u32 pipe, u32 queue, u32 vmid)
 868 {
 869         u32 srbm_gfx_cntl =
 870                 (((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
 871                 ((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
 872                 ((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
 873                 ((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
 874         WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
 875 }
 876
 877 static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
 878 {
 879         uint32_t tmp;
 880
 881         tmp = RREG32(mmCONFIG_CNTL);
 882         if (state == false)
 883                 tmp |= CONFIG_CNTL__VGA_DIS_MASK;
 884         else
 885                 tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
 886         WREG32(mmCONFIG_CNTL, tmp);
 887 }
 888
 889 static bool cik_read_disabled_bios(struct amdgpu_device *adev)
 890 {
 891         u32 bus_cntl;
 892         u32 d1vga_control = 0;
 893         u32 d2vga_control = 0;
 894         u32 vga_render_control = 0;
 895         u32 rom_cntl;
 896         bool r;
 897
 898         bus_cntl = RREG32(mmBUS_CNTL);
 899         if (adev->mode_info.num_crtc) {
 900                 d1vga_control = RREG32(mmD1VGA_CONTROL);
 901                 d2vga_control = RREG32(mmD2VGA_CONTROL);
 902                 vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
 903         }
 904         rom_cntl = RREG32_SMC(ixROM_CNTL);
 905
 906         /* enable the rom */
 907         WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
 908         if (adev->mode_info.num_crtc) {
 909                 /* Disable VGA mode */
 910                 WREG32(mmD1VGA_CONTROL,
 911                        (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 912                                           D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 913                 WREG32(mmD2VGA_CONTROL,
 914                        (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 915                                           D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 916                 WREG32(mmVGA_RENDER_CONTROL,
 917                        (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
 918         }
 919         WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
 920
 921         r = amdgpu_read_bios(adev);
 922
 923         /* restore regs */
 924         WREG32(mmBUS_CNTL, bus_cntl);
 925         if (adev->mode_info.num_crtc) {
 926                 WREG32(mmD1VGA_CONTROL, d1vga_control);
 927                 WREG32(mmD2VGA_CONTROL, d2vga_control);
 928                 WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
 929         }
 930         WREG32_SMC(ixROM_CNTL, rom_cntl);
 931         return r;
 932 }
 933
 934 static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
 935                                    u8 *bios, u32 length_bytes)
 936 {
 937         u32 *dw_ptr;
 938         unsigned long flags;
 939         u32 i, length_dw;
 940
 941         if (bios == NULL)
 942                 return false;
 943         if (length_bytes == 0)
 944                 return false;
 945         /* APU vbios image is part of sbios image */
 946         if (adev->flags & AMD_IS_APU)
 947                 return false;
 948
 949         dw_ptr = (u32 *)bios;
 950         length_dw = ALIGN(length_bytes, 4) / 4;
 951         /* take the smc lock since we are using the smc index */
 952         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 953         /* set rom index to 0 */
 954         WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
 955         WREG32(mmSMC_IND_DATA_0, 0);
 956         /* set index to data for continous read */
 957         WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
 958         for (i = 0; i < length_dw; i++)
 959                 dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
 960         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 961
 962         return true;
 963 }
 964
 965 static u32 cik_get_virtual_caps(struct amdgpu_device *adev)
 966 {
 967         /* CIK does not support SR-IOV */
 968         return 0;
 969 }
 970
 971 static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
 972         {mmGRBM_STATUS, false},
 973         {mmGB_ADDR_CONFIG, false},
 974         {mmMC_ARB_RAMCFG, false},
 975         {mmGB_TILE_MODE0, false},
 976         {mmGB_TILE_MODE1, false},
 977         {mmGB_TILE_MODE2, false},
 978         {mmGB_TILE_MODE3, false},
 979         {mmGB_TILE_MODE4, false},
 980         {mmGB_TILE_MODE5, false},
 981         {mmGB_TILE_MODE6, false},
 982         {mmGB_TILE_MODE7, false},
 983         {mmGB_TILE_MODE8, false},
 984         {mmGB_TILE_MODE9, false},
 985         {mmGB_TILE_MODE10, false},
 986         {mmGB_TILE_MODE11, false},
 987         {mmGB_TILE_MODE12, false},
 988         {mmGB_TILE_MODE13, false},
 989         {mmGB_TILE_MODE14, false},
 990         {mmGB_TILE_MODE15, false},
 991         {mmGB_TILE_MODE16, false},
 992         {mmGB_TILE_MODE17, false},
 993         {mmGB_TILE_MODE18, false},
 994         {mmGB_TILE_MODE19, false},
 995         {mmGB_TILE_MODE20, false},
 996         {mmGB_TILE_MODE21, false},
 997         {mmGB_TILE_MODE22, false},
 998         {mmGB_TILE_MODE23, false},
 999         {mmGB_TILE_MODE24, false},
1000         {mmGB_TILE_MODE25, false},
1001         {mmGB_TILE_MODE26, false},
1002         {mmGB_TILE_MODE27, false},
1003         {mmGB_TILE_MODE28, false},
1004         {mmGB_TILE_MODE29, false},
1005         {mmGB_TILE_MODE30, false},
1006         {mmGB_TILE_MODE31, false},
1007         {mmGB_MACROTILE_MODE0, false},
1008         {mmGB_MACROTILE_MODE1, false},
1009         {mmGB_MACROTILE_MODE2, false},
1010         {mmGB_MACROTILE_MODE3, false},
1011         {mmGB_MACROTILE_MODE4, false},
1012         {mmGB_MACROTILE_MODE5, false},
1013         {mmGB_MACROTILE_MODE6, false},
1014         {mmGB_MACROTILE_MODE7, false},
1015         {mmGB_MACROTILE_MODE8, false},
1016         {mmGB_MACROTILE_MODE9, false},
1017         {mmGB_MACROTILE_MODE10, false},
1018         {mmGB_MACROTILE_MODE11, false},
1019         {mmGB_MACROTILE_MODE12, false},
1020         {mmGB_MACROTILE_MODE13, false},
1021         {mmGB_MACROTILE_MODE14, false},
1022         {mmGB_MACROTILE_MODE15, false},
1023         {mmCC_RB_BACKEND_DISABLE, false, true},
1024         {mmGC_USER_RB_BACKEND_DISABLE, false, true},
1025         {mmGB_BACKEND_MAP, false, false},
1026         {mmPA_SC_RASTER_CONFIG, false, true},
1027         {mmPA_SC_RASTER_CONFIG_1, false, true},
1028 };
1029
1030 static uint32_t cik_read_indexed_register(struct amdgpu_device *adev,
1031                                           u32 se_num, u32 sh_num,
1032                                           u32 reg_offset)
1033 {
1034         uint32_t val;
1035
1036         mutex_lock(&adev->grbm_idx_mutex);
1037         if (se_num != 0xffffffff || sh_num != 0xffffffff)
1038                 gfx_v7_0_select_se_sh(adev, se_num, sh_num);
1039
1040         val = RREG32(reg_offset);
1041
1042         if (se_num != 0xffffffff || sh_num != 0xffffffff)
1043                 gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff);
1044         mutex_unlock(&adev->grbm_idx_mutex);
1045         return val;
1046 }
1047
1048 static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1049                              u32 sh_num, u32 reg_offset, u32 *value)
1050 {
1051         uint32_t i;
1052
1053         *value = 0;
1054         for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1055                 if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1056                         continue;
1057
1058                 if (!cik_allowed_read_registers[i].untouched)
1059                         *value = cik_allowed_read_registers[i].grbm_indexed ?
1060                                  cik_read_indexed_register(adev, se_num,
1061                                                            sh_num, reg_offset) :
1062                                  RREG32(reg_offset);
1063                 return 0;
1064         }
1065         return -EINVAL;
1066 }
1067
1068 struct kv_reset_save_regs {
1069         u32 gmcon_reng_execute;
1070         u32 gmcon_misc;
1071         u32 gmcon_misc3;
1072 };
1073
1074 static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1075                                    struct kv_reset_save_regs *save)
1076 {
1077         save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1078         save->gmcon_misc = RREG32(mmGMCON_MISC);
1079         save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1080
1081         WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1082                 ~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1083         WREG32(mmGMCON_MISC, save->gmcon_misc &
1084                 ~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1085                         GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1086 }
1087
1088 static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1089                                       struct kv_reset_save_regs *save)
1090 {
1091         int i;
1092
1093         WREG32(mmGMCON_PGFSM_WRITE, 0);
1094         WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1095
1096         for (i = 0; i < 5; i++)
1097                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1098
1099         WREG32(mmGMCON_PGFSM_WRITE, 0);
1100         WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1101
1102         for (i = 0; i < 5; i++)
1103                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1104
1105         WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1106         WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1107
1108         for (i = 0; i < 5; i++)
1109                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1110
1111         WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1112         WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1113
1114         for (i = 0; i < 5; i++)
1115                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1116
1117         WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1118         WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1119
1120         for (i = 0; i < 5; i++)
1121                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1122
1123         WREG32(mmGMCON_PGFSM_WRITE, 0);
1124         WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1125
1126         for (i = 0; i < 5; i++)
1127                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1128
1129         WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1130         WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1131
1132         for (i = 0; i < 5; i++)
1133                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1134
1135         WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1136         WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1137
1138         for (i = 0; i < 5; i++)
1139                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1140
1141         WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1142         WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1143
1144         for (i = 0; i < 5; i++)
1145                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1146
1147         WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1148         WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1149
1150         for (i = 0; i < 5; i++)
1151                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1152
1153         WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1154         WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1155
1156         WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1157         WREG32(mmGMCON_MISC, save->gmcon_misc);
1158         WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1159 }
1160
1161 static int cik_gpu_pci_config_reset(struct amdgpu_device *adev)
1162 {
1163         struct kv_reset_save_regs kv_save = { 0 };
1164         u32 i;
1165         int r = -EINVAL;
1166
1167         dev_info(adev->dev, "GPU pci config reset\n");
1168
1169         if (adev->flags & AMD_IS_APU)
1170                 kv_save_regs_for_reset(adev, &kv_save);
1171
1172         /* disable BM */
1173         pci_clear_master(adev->pdev);
1174         /* reset */
1175         amdgpu_pci_config_reset(adev);
1176
1177         udelay(100);
1178
1179         /* wait for asic to come out of reset */
1180         for (i = 0; i < adev->usec_timeout; i++) {
1181                 if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1182                         r = 0;
1183                         break;
1184                 }
1185                 udelay(1);
1186         }
1187
1188         /* does asic init need to be run first??? */
1189         if (adev->flags & AMD_IS_APU)
1190                 kv_restore_regs_for_reset(adev, &kv_save);
1191
1192         return r;
1193 }
1194
1195 static void cik_set_bios_scratch_engine_hung(struct amdgpu_device *adev, bool hung)
1196 {
1197         u32 tmp = RREG32(mmBIOS_SCRATCH_3);
1198
1199         if (hung)
1200                 tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1201         else
1202                 tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1203
1204         WREG32(mmBIOS_SCRATCH_3, tmp);
1205 }
1206
1207 /**
1208  * cik_asic_reset - soft reset GPU
1209  *
1210  * @adev: amdgpu_device pointer
1211  *
1212  * Look up which blocks are hung and attempt
1213  * to reset them.
1214  * Returns 0 for success.
1215  */
1216 static int cik_asic_reset(struct amdgpu_device *adev)
1217 {
1218         int r;
1219         cik_set_bios_scratch_engine_hung(adev, true);
1220
1221         r = cik_gpu_pci_config_reset(adev);
1222
1223         cik_set_bios_scratch_engine_hung(adev, false);
1224
1225         return r;
1226 }
1227
1228 static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1229                               u32 cntl_reg, u32 status_reg)
1230 {
1231         int r, i;
1232         struct atom_clock_dividers dividers;
1233         uint32_t tmp;
1234
1235         r = amdgpu_atombios_get_clock_dividers(adev,
1236                                                COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1237                                                clock, false, &dividers);
1238         if (r)
1239                 return r;
1240
1241         tmp = RREG32_SMC(cntl_reg);
1242         tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1243                 CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1244         tmp |= dividers.post_divider;
1245         WREG32_SMC(cntl_reg, tmp);
1246
1247         for (i = 0; i < 100; i++) {
1248                 if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1249                         break;
1250                 mdelay(10);
1251         }
1252         if (i == 100)
1253                 return -ETIMEDOUT;
1254
1255         return 0;
1256 }
1257
1258 static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1259 {
1260         int r = 0;
1261
1262         r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1263         if (r)
1264                 return r;
1265
1266         r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1267         return r;
1268 }
1269
1270 static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1271 {
1272         int r, i;
1273         struct atom_clock_dividers dividers;
1274         u32 tmp;
1275
1276         r = amdgpu_atombios_get_clock_dividers(adev,
1277                                                COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1278                                                ecclk, false, &dividers);
1279         if (r)
1280                 return r;
1281
1282         for (i = 0; i < 100; i++) {
1283                 if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1284                         break;
1285                 mdelay(10);
1286         }
1287         if (i == 100)
1288                 return -ETIMEDOUT;
1289
1290         tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1291         tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1292                 CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1293         tmp |= dividers.post_divider;
1294         WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1295
1296         for (i = 0; i < 100; i++) {
1297                 if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1298                         break;
1299                 mdelay(10);
1300         }
1301         if (i == 100)
1302                 return -ETIMEDOUT;
1303
1304         return 0;
1305 }
1306
1307 static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1308 {
1309         struct pci_dev *root = adev->pdev->bus->self;
1310         int bridge_pos, gpu_pos;
1311         u32 speed_cntl, current_data_rate;
1312         int i;
1313         u16 tmp16;
1314
1315         if (pci_is_root_bus(adev->pdev->bus))
1316                 return;
1317
1318         if (amdgpu_pcie_gen2 == 0)
1319                 return;
1320
1321         if (adev->flags & AMD_IS_APU)
1322                 return;
1323
1324         if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1325                                         CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1326                 return;
1327
1328         speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1329         current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1330                 PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1331         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1332                 if (current_data_rate == 2) {
1333                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1334                         return;
1335                 }
1336                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1337         } else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1338                 if (current_data_rate == 1) {
1339                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1340                         return;
1341                 }
1342                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1343         }
1344
1345         bridge_pos = pci_pcie_cap(root);
1346         if (!bridge_pos)
1347                 return;
1348
1349         gpu_pos = pci_pcie_cap(adev->pdev);
1350         if (!gpu_pos)
1351                 return;
1352
1353         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1354                 /* re-try equalization if gen3 is not already enabled */
1355                 if (current_data_rate != 2) {
1356                         u16 bridge_cfg, gpu_cfg;
1357                         u16 bridge_cfg2, gpu_cfg2;
1358                         u32 max_lw, current_lw, tmp;
1359
1360                         pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1361                         pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1362
1363                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1364                         pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1365
1366                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1367                         pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1368
1369                         tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1370                         max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1371                                 PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1372                         current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1373                                 >> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1374
1375                         if (current_lw < max_lw) {
1376                                 tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1377                                 if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1378                                         tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1379                                                 PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1380                                         tmp |= (max_lw <<
1381                                                 PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1382                                         tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1383                                         PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1384                                         PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1385                                         WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1386                                 }
1387                         }
1388
1389                         for (i = 0; i < 10; i++) {
1390                                 /* check status */
1391                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
1392                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1393                                         break;
1394
1395                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1396                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1397
1398                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
1399                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
1400
1401                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1402                                 tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1403                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1404
1405                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1406                                 tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1407                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1408
1409                                 mdelay(100);
1410
1411                                 /* linkctl */
1412                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
1413                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1414                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1415                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1416
1417                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
1418                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1419                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1420                                 pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1421
1422                                 /* linkctl2 */
1423                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
1424                                 tmp16 &= ~((1 << 4) | (7 << 9));
1425                                 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
1426                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
1427
1428                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1429                                 tmp16 &= ~((1 << 4) | (7 << 9));
1430                                 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
1431                                 pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1432
1433                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1434                                 tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1435                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1436                         }
1437                 }
1438         }
1439
1440         /* set the link speed */
1441         speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1442                 PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1443         speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1444         WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1445
1446         pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1447         tmp16 &= ~0xf;
1448         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1449                 tmp16 |= 3; /* gen3 */
1450         else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1451                 tmp16 |= 2; /* gen2 */
1452         else
1453                 tmp16 |= 1; /* gen1 */
1454         pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1455
1456         speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1457         speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1458         WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1459
1460         for (i = 0; i < adev->usec_timeout; i++) {
1461                 speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1462                 if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1463                         break;
1464                 udelay(1);
1465         }
1466 }
1467
1468 static void cik_program_aspm(struct amdgpu_device *adev)
1469 {
1470         u32 data, orig;
1471         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1472         bool disable_clkreq = false;
1473
1474         if (amdgpu_aspm == 0)
1475                 return;
1476
1477         if (pci_is_root_bus(adev->pdev->bus))
1478                 return;
1479
1480         /* XXX double check APUs */
1481         if (adev->flags & AMD_IS_APU)
1482                 return;
1483
1484         orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1485         data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1486         data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1487                 PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1488         if (orig != data)
1489                 WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1490
1491         orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1492         data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1493         if (orig != data)
1494                 WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1495
1496         orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1497         data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1498         if (orig != data)
1499                 WREG32_PCIE(ixPCIE_P_CNTL, data);
1500
1501         orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1502         data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1503                 PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1504         data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1505         if (!disable_l0s)
1506                 data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1507
1508         if (!disable_l1) {
1509                 data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1510                 data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1511                 if (orig != data)
1512                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1513
1514                 if (!disable_plloff_in_l1) {
1515                         bool clk_req_support;
1516
1517                         orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1518                         data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1519                                 PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1520                         data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1521                                 (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1522                         if (orig != data)
1523                                 WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1524
1525                         orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1526                         data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1527                                 PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1528                         data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1529                                 (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1530                         if (orig != data)
1531                                 WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1532
1533                         orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1534                         data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1535                                 PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1536                         data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1537                                 (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1538                         if (orig != data)
1539                                 WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1540
1541                         orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1542                         data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1543                                 PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1544                         data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1545                                 (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1546                         if (orig != data)
1547                                 WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1548
1549                         orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1550                         data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1551                         data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1552                         if (orig != data)
1553                                 WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1554
1555                         if (!disable_clkreq) {
1556                                 struct pci_dev *root = adev->pdev->bus->self;
1557                                 u32 lnkcap;
1558
1559                                 clk_req_support = false;
1560                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1561                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1562                                         clk_req_support = true;
1563                         } else {
1564                                 clk_req_support = false;
1565                         }
1566
1567                         if (clk_req_support) {
1568                                 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1569                                 data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1570                                         PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1571                                 if (orig != data)
1572                                         WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1573
1574                                 orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1575                                 data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1576                                         THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1577                                 data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1578                                         (1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1579                                 if (orig != data)
1580                                         WREG32_SMC(ixTHM_CLK_CNTL, data);
1581
1582                                 orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1583                                 data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1584                                         MISC_CLK_CTRL__ZCLK_SEL_MASK);
1585                                 data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1586                                         (1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1587                                 if (orig != data)
1588                                         WREG32_SMC(ixMISC_CLK_CTRL, data);
1589
1590                                 orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1591                                 data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1592                                 if (orig != data)
1593                                         WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1594
1595                                 orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1596                                 data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1597                                 if (orig != data)
1598                                         WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1599
1600                                 orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1601                                 data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1602                                 data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1603                                 if (orig != data)
1604                                         WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1605                         }
1606                 }
1607         } else {
1608                 if (orig != data)
1609                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1610         }
1611
1612         orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1613         data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1614                 PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1615                 PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1616         if (orig != data)
1617                 WREG32_PCIE(ixPCIE_CNTL2, data);
1618
1619         if (!disable_l0s) {
1620                 data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1621                 if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1622                                 PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1623                         data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1624                         if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1625                         (data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1626                                 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1627                                 data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1628                                 if (orig != data)
1629                                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1630                         }
1631                 }
1632         }
1633 }
1634
1635 static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1636 {
1637         return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1638                 >> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1639 }
1640
1641 static const struct amdgpu_ip_block_version bonaire_ip_blocks[] =
1642 {
1643         /* ORDER MATTERS! */
1644         {
1645                 .type = AMD_IP_BLOCK_TYPE_COMMON,
1646                 .major = 1,
1647                 .minor = 0,
1648                 .rev = 0,
1649                 .funcs = &cik_common_ip_funcs,
1650         },
1651         {
1652                 .type = AMD_IP_BLOCK_TYPE_GMC,
1653                 .major = 7,
1654                 .minor = 0,
1655                 .rev = 0,
1656                 .funcs = &gmc_v7_0_ip_funcs,
1657         },
1658         {
1659                 .type = AMD_IP_BLOCK_TYPE_IH,
1660                 .major = 2,
1661                 .minor = 0,
1662                 .rev = 0,
1663                 .funcs = &cik_ih_ip_funcs,
1664         },
1665         {
1666                 .type = AMD_IP_BLOCK_TYPE_SMC,
1667                 .major = 7,
1668                 .minor = 0,
1669                 .rev = 0,
1670                 .funcs = &amdgpu_pp_ip_funcs,
1671         },
1672         {
1673                 .type = AMD_IP_BLOCK_TYPE_DCE,
1674                 .major = 8,
1675                 .minor = 2,
1676                 .rev = 0,
1677                 .funcs = &dce_v8_0_ip_funcs,
1678         },
1679         {
1680                 .type = AMD_IP_BLOCK_TYPE_GFX,
1681                 .major = 7,
1682                 .minor = 2,
1683                 .rev = 0,
1684                 .funcs = &gfx_v7_0_ip_funcs,
1685         },
1686         {
1687                 .type = AMD_IP_BLOCK_TYPE_SDMA,
1688                 .major = 2,
1689                 .minor = 0,
1690                 .rev = 0,
1691                 .funcs = &cik_sdma_ip_funcs,
1692         },
1693         {
1694                 .type = AMD_IP_BLOCK_TYPE_UVD,
1695                 .major = 4,
1696                 .minor = 2,
1697                 .rev = 0,
1698                 .funcs = &uvd_v4_2_ip_funcs,
1699         },
1700         {
1701                 .type = AMD_IP_BLOCK_TYPE_VCE,
1702                 .major = 2,
1703                 .minor = 0,
1704                 .rev = 0,
1705                 .funcs = &vce_v2_0_ip_funcs,
1706         },
1707 };
1708
1709 static const struct amdgpu_ip_block_version hawaii_ip_blocks[] =
1710 {
1711         /* ORDER MATTERS! */
1712         {
1713                 .type = AMD_IP_BLOCK_TYPE_COMMON,
1714                 .major = 1,
1715                 .minor = 0,
1716                 .rev = 0,
1717                 .funcs = &cik_common_ip_funcs,
1718         },
1719         {
1720                 .type = AMD_IP_BLOCK_TYPE_GMC,
1721                 .major = 7,
1722                 .minor = 0,
1723                 .rev = 0,
1724                 .funcs = &gmc_v7_0_ip_funcs,
1725         },
1726         {
1727                 .type = AMD_IP_BLOCK_TYPE_IH,
1728                 .major = 2,
1729                 .minor = 0,
1730                 .rev = 0,
1731                 .funcs = &cik_ih_ip_funcs,
1732         },
1733         {
1734                 .type = AMD_IP_BLOCK_TYPE_SMC,
1735                 .major = 7,
1736                 .minor = 0,
1737                 .rev = 0,
1738                 .funcs = &amdgpu_pp_ip_funcs,
1739         },
1740         {
1741                 .type = AMD_IP_BLOCK_TYPE_DCE,
1742                 .major = 8,
1743                 .minor = 5,
1744                 .rev = 0,
1745                 .funcs = &dce_v8_0_ip_funcs,
1746         },
1747         {
1748                 .type = AMD_IP_BLOCK_TYPE_GFX,
1749                 .major = 7,
1750                 .minor = 3,
1751                 .rev = 0,
1752                 .funcs = &gfx_v7_0_ip_funcs,
1753         },
1754         {
1755                 .type = AMD_IP_BLOCK_TYPE_SDMA,
1756                 .major = 2,
1757                 .minor = 0,
1758                 .rev = 0,
1759                 .funcs = &cik_sdma_ip_funcs,
1760         },
1761         {
1762                 .type = AMD_IP_BLOCK_TYPE_UVD,
1763                 .major = 4,
1764                 .minor = 2,
1765                 .rev = 0,
1766                 .funcs = &uvd_v4_2_ip_funcs,
1767         },
1768         {
1769                 .type = AMD_IP_BLOCK_TYPE_VCE,
1770                 .major = 2,
1771                 .minor = 0,
1772                 .rev = 0,
1773                 .funcs = &vce_v2_0_ip_funcs,
1774         },
1775 };
1776
1777 static const struct amdgpu_ip_block_version kabini_ip_blocks[] =
1778 {
1779         /* ORDER MATTERS! */
1780         {
1781                 .type = AMD_IP_BLOCK_TYPE_COMMON,
1782                 .major = 1,
1783                 .minor = 0,
1784                 .rev = 0,
1785                 .funcs = &cik_common_ip_funcs,
1786         },
1787         {
1788                 .type = AMD_IP_BLOCK_TYPE_GMC,
1789                 .major = 7,
1790                 .minor = 0,
1791                 .rev = 0,
1792                 .funcs = &gmc_v7_0_ip_funcs,
1793         },
1794         {
1795                 .type = AMD_IP_BLOCK_TYPE_IH,
1796                 .major = 2,
1797                 .minor = 0,
1798                 .rev = 0,
1799                 .funcs = &cik_ih_ip_funcs,
1800         },
1801         {
1802                 .type = AMD_IP_BLOCK_TYPE_SMC,
1803                 .major = 7,
1804                 .minor = 0,
1805                 .rev = 0,
1806                 .funcs = &amdgpu_pp_ip_funcs,
1807         },
1808         {
1809                 .type = AMD_IP_BLOCK_TYPE_DCE,
1810                 .major = 8,
1811                 .minor = 3,
1812                 .rev = 0,
1813                 .funcs = &dce_v8_0_ip_funcs,
1814         },
1815         {
1816                 .type = AMD_IP_BLOCK_TYPE_GFX,
1817                 .major = 7,
1818                 .minor = 2,
1819                 .rev = 0,
1820                 .funcs = &gfx_v7_0_ip_funcs,
1821         },
1822         {
1823                 .type = AMD_IP_BLOCK_TYPE_SDMA,
1824                 .major = 2,
1825                 .minor = 0,
1826                 .rev = 0,
1827                 .funcs = &cik_sdma_ip_funcs,
1828         },
1829         {
1830                 .type = AMD_IP_BLOCK_TYPE_UVD,
1831                 .major = 4,
1832                 .minor = 2,
1833                 .rev = 0,
1834                 .funcs = &uvd_v4_2_ip_funcs,
1835         },
1836         {
1837                 .type = AMD_IP_BLOCK_TYPE_VCE,
1838                 .major = 2,
1839                 .minor = 0,
1840                 .rev = 0,
1841                 .funcs = &vce_v2_0_ip_funcs,
1842         },
1843 };
1844
1845 static const struct amdgpu_ip_block_version mullins_ip_blocks[] =
1846 {
1847         /* ORDER MATTERS! */
1848         {
1849                 .type = AMD_IP_BLOCK_TYPE_COMMON,
1850                 .major = 1,
1851                 .minor = 0,
1852                 .rev = 0,
1853                 .funcs = &cik_common_ip_funcs,
1854         },
1855         {
1856                 .type = AMD_IP_BLOCK_TYPE_GMC,
1857                 .major = 7,
1858                 .minor = 0,
1859                 .rev = 0,
1860                 .funcs = &gmc_v7_0_ip_funcs,
1861         },
1862         {
1863                 .type = AMD_IP_BLOCK_TYPE_IH,
1864                 .major = 2,
1865                 .minor = 0,
1866                 .rev = 0,
1867                 .funcs = &cik_ih_ip_funcs,
1868         },
1869         {
1870                 .type = AMD_IP_BLOCK_TYPE_SMC,
1871                 .major = 7,
1872                 .minor = 0,
1873                 .rev = 0,
1874                 .funcs = &amdgpu_pp_ip_funcs,
1875         },
1876         {
1877                 .type = AMD_IP_BLOCK_TYPE_DCE,
1878                 .major = 8,
1879                 .minor = 3,
1880                 .rev = 0,
1881                 .funcs = &dce_v8_0_ip_funcs,
1882         },
1883         {
1884                 .type = AMD_IP_BLOCK_TYPE_GFX,
1885                 .major = 7,
1886                 .minor = 2,
1887                 .rev = 0,
1888                 .funcs = &gfx_v7_0_ip_funcs,
1889         },
1890         {
1891                 .type = AMD_IP_BLOCK_TYPE_SDMA,
1892                 .major = 2,
1893                 .minor = 0,
1894                 .rev = 0,
1895                 .funcs = &cik_sdma_ip_funcs,
1896         },
1897         {
1898                 .type = AMD_IP_BLOCK_TYPE_UVD,
1899                 .major = 4,
1900                 .minor = 2,
1901                 .rev = 0,
1902                 .funcs = &uvd_v4_2_ip_funcs,
1903         },
1904         {
1905                 .type = AMD_IP_BLOCK_TYPE_VCE,
1906                 .major = 2,
1907                 .minor = 0,
1908                 .rev = 0,
1909                 .funcs = &vce_v2_0_ip_funcs,
1910         },
1911 };
1912
1913 static const struct amdgpu_ip_block_version kaveri_ip_blocks[] =
1914 {
1915         /* ORDER MATTERS! */
1916         {
1917                 .type = AMD_IP_BLOCK_TYPE_COMMON,
1918                 .major = 1,
1919                 .minor = 0,
1920                 .rev = 0,
1921                 .funcs = &cik_common_ip_funcs,
1922         },
1923         {
1924                 .type = AMD_IP_BLOCK_TYPE_GMC,
1925                 .major = 7,
1926                 .minor = 0,
1927                 .rev = 0,
1928                 .funcs = &gmc_v7_0_ip_funcs,
1929         },
1930         {
1931                 .type = AMD_IP_BLOCK_TYPE_IH,
1932                 .major = 2,
1933                 .minor = 0,
1934                 .rev = 0,
1935                 .funcs = &cik_ih_ip_funcs,
1936         },
1937         {
1938                 .type = AMD_IP_BLOCK_TYPE_SMC,
1939                 .major = 7,
1940                 .minor = 0,
1941                 .rev = 0,
1942                 .funcs = &amdgpu_pp_ip_funcs,
1943         },
1944         {
1945                 .type = AMD_IP_BLOCK_TYPE_DCE,
1946                 .major = 8,
1947                 .minor = 1,
1948                 .rev = 0,
1949                 .funcs = &dce_v8_0_ip_funcs,
1950         },
1951         {
1952                 .type = AMD_IP_BLOCK_TYPE_GFX,
1953                 .major = 7,
1954                 .minor = 1,
1955                 .rev = 0,
1956                 .funcs = &gfx_v7_0_ip_funcs,
1957         },
1958         {
1959                 .type = AMD_IP_BLOCK_TYPE_SDMA,
1960                 .major = 2,
1961                 .minor = 0,
1962                 .rev = 0,
1963                 .funcs = &cik_sdma_ip_funcs,
1964         },
1965         {
1966                 .type = AMD_IP_BLOCK_TYPE_UVD,
1967                 .major = 4,
1968                 .minor = 2,
1969                 .rev = 0,
1970                 .funcs = &uvd_v4_2_ip_funcs,
1971         },
1972         {
1973                 .type = AMD_IP_BLOCK_TYPE_VCE,
1974                 .major = 2,
1975                 .minor = 0,
1976                 .rev = 0,
1977                 .funcs = &vce_v2_0_ip_funcs,
1978         },
1979 };
1980
1981 int cik_set_ip_blocks(struct amdgpu_device *adev)
1982 {
1983         switch (adev->asic_type) {
1984         case CHIP_BONAIRE:
1985                 adev->ip_blocks = bonaire_ip_blocks;
1986                 adev->num_ip_blocks = ARRAY_SIZE(bonaire_ip_blocks);
1987                 break;
1988         case CHIP_HAWAII:
1989                 adev->ip_blocks = hawaii_ip_blocks;
1990                 adev->num_ip_blocks = ARRAY_SIZE(hawaii_ip_blocks);
1991                 break;
1992         case CHIP_KAVERI:
1993                 adev->ip_blocks = kaveri_ip_blocks;
1994                 adev->num_ip_blocks = ARRAY_SIZE(kaveri_ip_blocks);
1995                 break;
1996         case CHIP_KABINI:
1997                 adev->ip_blocks = kabini_ip_blocks;
1998                 adev->num_ip_blocks = ARRAY_SIZE(kabini_ip_blocks);
1999                 break;
2000         case CHIP_MULLINS:
2001                 adev->ip_blocks = mullins_ip_blocks;
2002                 adev->num_ip_blocks = ARRAY_SIZE(mullins_ip_blocks);
2003                 break;
2004         default:
2005                 /* FIXME: not supported yet */
2006                 return -EINVAL;
2007         }
2008
2009         return 0;
2010 }
2011
2012 static const struct amdgpu_asic_funcs cik_asic_funcs =
2013 {
2014         .read_disabled_bios = &cik_read_disabled_bios,
2015         .read_bios_from_rom = &cik_read_bios_from_rom,
2016         .read_register = &cik_read_register,
2017         .reset = &cik_asic_reset,
2018         .set_vga_state = &cik_vga_set_state,
2019         .get_xclk = &cik_get_xclk,
2020         .set_uvd_clocks = &cik_set_uvd_clocks,
2021         .set_vce_clocks = &cik_set_vce_clocks,
2022         .get_virtual_caps = &cik_get_virtual_caps,
2023         /* these should be moved to their own ip modules */
2024         .get_gpu_clock_counter = &gfx_v7_0_get_gpu_clock_counter,
2025         .wait_for_mc_idle = &gmc_v7_0_mc_wait_for_idle,
2026 };
2027
2028 static int cik_common_early_init(void *handle)
2029 {
2030         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2031
2032         adev->smc_rreg = &cik_smc_rreg;
2033         adev->smc_wreg = &cik_smc_wreg;
2034         adev->pcie_rreg = &cik_pcie_rreg;
2035         adev->pcie_wreg = &cik_pcie_wreg;
2036         adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
2037         adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
2038         adev->didt_rreg = &cik_didt_rreg;
2039         adev->didt_wreg = &cik_didt_wreg;
2040
2041         adev->asic_funcs = &cik_asic_funcs;
2042
2043         adev->rev_id = cik_get_rev_id(adev);
2044         adev->external_rev_id = 0xFF;
2045         switch (adev->asic_type) {
2046         case CHIP_BONAIRE:
2047                 adev->cg_flags =
2048                         AMD_CG_SUPPORT_GFX_MGCG |
2049                         AMD_CG_SUPPORT_GFX_MGLS |
2050                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
2051                         AMD_CG_SUPPORT_GFX_CGLS |
2052                         AMD_CG_SUPPORT_GFX_CGTS |
2053                         AMD_CG_SUPPORT_GFX_CGTS_LS |
2054                         AMD_CG_SUPPORT_GFX_CP_LS |
2055                         AMD_CG_SUPPORT_MC_LS |
2056                         AMD_CG_SUPPORT_MC_MGCG |
2057                         AMD_CG_SUPPORT_SDMA_MGCG |
2058                         AMD_CG_SUPPORT_SDMA_LS |
2059                         AMD_CG_SUPPORT_BIF_LS |
2060                         AMD_CG_SUPPORT_VCE_MGCG |
2061                         AMD_CG_SUPPORT_UVD_MGCG |
2062                         AMD_CG_SUPPORT_HDP_LS |
2063                         AMD_CG_SUPPORT_HDP_MGCG;
2064                 adev->pg_flags = 0;
2065                 adev->external_rev_id = adev->rev_id + 0x14;
2066                 break;
2067         case CHIP_HAWAII:
2068                 adev->cg_flags =
2069                         AMD_CG_SUPPORT_GFX_MGCG |
2070                         AMD_CG_SUPPORT_GFX_MGLS |
2071                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
2072                         AMD_CG_SUPPORT_GFX_CGLS |
2073                         AMD_CG_SUPPORT_GFX_CGTS |
2074                         AMD_CG_SUPPORT_GFX_CP_LS |
2075                         AMD_CG_SUPPORT_MC_LS |
2076                         AMD_CG_SUPPORT_MC_MGCG |
2077                         AMD_CG_SUPPORT_SDMA_MGCG |
2078                         AMD_CG_SUPPORT_SDMA_LS |
2079                         AMD_CG_SUPPORT_BIF_LS |
2080                         AMD_CG_SUPPORT_VCE_MGCG |
2081                         AMD_CG_SUPPORT_UVD_MGCG |
2082                         AMD_CG_SUPPORT_HDP_LS |
2083                         AMD_CG_SUPPORT_HDP_MGCG;
2084                 adev->pg_flags = 0;
2085                 adev->external_rev_id = 0x28;
2086                 break;
2087         case CHIP_KAVERI:
2088                 adev->cg_flags =
2089                         AMD_CG_SUPPORT_GFX_MGCG |
2090                         AMD_CG_SUPPORT_GFX_MGLS |
2091                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
2092                         AMD_CG_SUPPORT_GFX_CGLS |
2093                         AMD_CG_SUPPORT_GFX_CGTS |
2094                         AMD_CG_SUPPORT_GFX_CGTS_LS |
2095                         AMD_CG_SUPPORT_GFX_CP_LS |
2096                         AMD_CG_SUPPORT_SDMA_MGCG |
2097                         AMD_CG_SUPPORT_SDMA_LS |
2098                         AMD_CG_SUPPORT_BIF_LS |
2099                         AMD_CG_SUPPORT_VCE_MGCG |
2100                         AMD_CG_SUPPORT_UVD_MGCG |
2101                         AMD_CG_SUPPORT_HDP_LS |
2102                         AMD_CG_SUPPORT_HDP_MGCG;
2103                 adev->pg_flags =
2104                         /*AMD_PG_SUPPORT_GFX_PG |
2105                           AMD_PG_SUPPORT_GFX_SMG |
2106                           AMD_PG_SUPPORT_GFX_DMG |*/
2107                         AMD_PG_SUPPORT_UVD |
2108                         /*AMD_PG_SUPPORT_VCE |
2109                           AMD_PG_SUPPORT_CP |
2110                           AMD_PG_SUPPORT_GDS |
2111                           AMD_PG_SUPPORT_RLC_SMU_HS |
2112                           AMD_PG_SUPPORT_ACP |
2113                           AMD_PG_SUPPORT_SAMU |*/
2114                         0;
2115                 if (adev->pdev->device == 0x1312 ||
2116                         adev->pdev->device == 0x1316 ||
2117                         adev->pdev->device == 0x1317)
2118                         adev->external_rev_id = 0x41;
2119                 else
2120                         adev->external_rev_id = 0x1;
2121                 break;
2122         case CHIP_KABINI:
2123         case CHIP_MULLINS:
2124                 adev->cg_flags =
2125                         AMD_CG_SUPPORT_GFX_MGCG |
2126                         AMD_CG_SUPPORT_GFX_MGLS |
2127                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
2128                         AMD_CG_SUPPORT_GFX_CGLS |
2129                         AMD_CG_SUPPORT_GFX_CGTS |
2130                         AMD_CG_SUPPORT_GFX_CGTS_LS |
2131                         AMD_CG_SUPPORT_GFX_CP_LS |
2132                         AMD_CG_SUPPORT_SDMA_MGCG |
2133                         AMD_CG_SUPPORT_SDMA_LS |
2134                         AMD_CG_SUPPORT_BIF_LS |
2135                         AMD_CG_SUPPORT_VCE_MGCG |
2136                         AMD_CG_SUPPORT_UVD_MGCG |
2137                         AMD_CG_SUPPORT_HDP_LS |
2138                         AMD_CG_SUPPORT_HDP_MGCG;
2139                 adev->pg_flags =
2140                         /*AMD_PG_SUPPORT_GFX_PG |
2141                           AMD_PG_SUPPORT_GFX_SMG | */
2142                         AMD_PG_SUPPORT_UVD |
2143                         /*AMD_PG_SUPPORT_VCE |
2144                           AMD_PG_SUPPORT_CP |
2145                           AMD_PG_SUPPORT_GDS |
2146                           AMD_PG_SUPPORT_RLC_SMU_HS |
2147                           AMD_PG_SUPPORT_SAMU |*/
2148                         0;
2149                 if (adev->asic_type == CHIP_KABINI) {
2150                         if (adev->rev_id == 0)
2151                                 adev->external_rev_id = 0x81;
2152                         else if (adev->rev_id == 1)
2153                                 adev->external_rev_id = 0x82;
2154                         else if (adev->rev_id == 2)
2155                                 adev->external_rev_id = 0x85;
2156                 } else
2157                         adev->external_rev_id = adev->rev_id + 0xa1;
2158                 break;
2159         default:
2160                 /* FIXME: not supported yet */
2161                 return -EINVAL;
2162         }
2163
2164         amdgpu_get_pcie_info(adev);
2165
2166         return 0;
2167 }
2168
2169 static int cik_common_sw_init(void *handle)
2170 {
2171         return 0;
2172 }
2173
2174 static int cik_common_sw_fini(void *handle)
2175 {
2176         return 0;
2177 }
2178
2179 static int cik_common_hw_init(void *handle)
2180 {
2181         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2182
2183         /* move the golden regs per IP block */
2184         cik_init_golden_registers(adev);
2185         /* enable pcie gen2/3 link */
2186         cik_pcie_gen3_enable(adev);
2187         /* enable aspm */
2188         cik_program_aspm(adev);
2189
2190         return 0;
2191 }
2192
2193 static int cik_common_hw_fini(void *handle)
2194 {
2195         return 0;
2196 }
2197
2198 static int cik_common_suspend(void *handle)
2199 {
2200         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2201
2202         amdgpu_amdkfd_suspend(adev);
2203
2204         return cik_common_hw_fini(adev);
2205 }
2206
2207 static int cik_common_resume(void *handle)
2208 {
2209         int r;
2210         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2211
2212         r = cik_common_hw_init(adev);
2213         if (r)
2214                 return r;
2215
2216         return amdgpu_amdkfd_resume(adev);
2217 }
2218
2219 static bool cik_common_is_idle(void *handle)
2220 {
2221         return true;
2222 }
2223
2224 static int cik_common_wait_for_idle(void *handle)
2225 {
2226         return 0;
2227 }
2228
2229 static int cik_common_soft_reset(void *handle)
2230 {
2231         /* XXX hard reset?? */
2232         return 0;
2233 }
2234
2235 static int cik_common_set_clockgating_state(void *handle,
2236                                             enum amd_clockgating_state state)
2237 {
2238         return 0;
2239 }
2240
2241 static int cik_common_set_powergating_state(void *handle,
2242                                             enum amd_powergating_state state)
2243 {
2244         return 0;
2245 }
2246
2247 const struct amd_ip_funcs cik_common_ip_funcs = {
2248         .name = "cik_common",
2249         .early_init = cik_common_early_init,
2250         .late_init = NULL,
2251         .sw_init = cik_common_sw_init,
2252         .sw_fini = cik_common_sw_fini,
2253         .hw_init = cik_common_hw_init,
2254         .hw_fini = cik_common_hw_fini,
2255         .suspend = cik_common_suspend,
2256         .resume = cik_common_resume,
2257         .is_idle = cik_common_is_idle,
2258         .wait_for_idle = cik_common_wait_for_idle,
2259         .soft_reset = cik_common_soft_reset,
2260         .set_clockgating_state = cik_common_set_clockgating_state,
2261         .set_powergating_state = cik_common_set_powergating_state,
2262 };