drivers/gpu/drm/radeon/cik.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include "drmP.h"
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include "cikd.h"
  31 #include "atom.h"
  32 #include "cik_blit_shaders.h"
  33 #include "radeon_ucode.h"
  34 #include "clearstate_ci.h"
  35
  36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
  37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
  38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
  39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
  40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
  41 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
  42 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
  43 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
  44 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
  45 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
  46 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
  47 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
  48 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
  49 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
  50 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
  51 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
  52 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
  53 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
  54 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
  55 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
  56 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
  57 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
  58 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
  59 MODULE_FIRMWARE("radeon/KABINI_me.bin");
  60 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
  61 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
  62 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
  63 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
  64
  65 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
  66 extern void r600_ih_ring_fini(struct radeon_device *rdev);
  67 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
  68 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
  69 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
  70 extern void sumo_rlc_fini(struct radeon_device *rdev);
  71 extern int sumo_rlc_init(struct radeon_device *rdev);
  72 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
  73 extern void si_rlc_reset(struct radeon_device *rdev);
  74 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
  75 extern int cik_sdma_resume(struct radeon_device *rdev);
  76 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
  77 extern void cik_sdma_fini(struct radeon_device *rdev);
  78 static void cik_rlc_stop(struct radeon_device *rdev);
  79 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
  80 static void cik_program_aspm(struct radeon_device *rdev);
  81 static void cik_init_pg(struct radeon_device *rdev);
  82 static void cik_init_cg(struct radeon_device *rdev);
  83 static void cik_fini_pg(struct radeon_device *rdev);
  84 static void cik_fini_cg(struct radeon_device *rdev);
  85 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
  86                                           bool enable);
  87
  88 /* get temperature in millidegrees */
  89 int ci_get_temp(struct radeon_device *rdev)
  90 {
  91         u32 temp;
  92         int actual_temp = 0;
  93
  94         temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
  95                 CTF_TEMP_SHIFT;
  96
  97         if (temp & 0x200)
  98                 actual_temp = 255;
  99         else
 100                 actual_temp = temp & 0x1ff;
 101
 102         actual_temp = actual_temp * 1000;
 103
 104         return actual_temp;
 105 }
 106
 107 /* get temperature in millidegrees */
 108 int kv_get_temp(struct radeon_device *rdev)
 109 {
 110         u32 temp;
 111         int actual_temp = 0;
 112
 113         temp = RREG32_SMC(0xC0300E0C);
 114
 115         if (temp)
 116                 actual_temp = (temp / 8) - 49;
 117         else
 118                 actual_temp = 0;
 119
 120         actual_temp = actual_temp * 1000;
 121
 122         return actual_temp;
 123 }
 124
 125 /*
 126  * Indirect registers accessor
 127  */
 128 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
 129 {
 130         unsigned long flags;
 131         u32 r;
 132
 133         spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
 134         WREG32(PCIE_INDEX, reg);
 135         (void)RREG32(PCIE_INDEX);
 136         r = RREG32(PCIE_DATA);
 137         spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
 138         return r;
 139 }
 140
 141 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 142 {
 143         unsigned long flags;
 144
 145         spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
 146         WREG32(PCIE_INDEX, reg);
 147         (void)RREG32(PCIE_INDEX);
 148         WREG32(PCIE_DATA, v);
 149         (void)RREG32(PCIE_DATA);
 150         spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
 151 }
 152
 153 static const u32 spectre_rlc_save_restore_register_list[] =
 154 {
 155         (0x0e00 << 16) | (0xc12c >> 2),
 156         0x00000000,
 157         (0x0e00 << 16) | (0xc140 >> 2),
 158         0x00000000,
 159         (0x0e00 << 16) | (0xc150 >> 2),
 160         0x00000000,
 161         (0x0e00 << 16) | (0xc15c >> 2),
 162         0x00000000,
 163         (0x0e00 << 16) | (0xc168 >> 2),
 164         0x00000000,
 165         (0x0e00 << 16) | (0xc170 >> 2),
 166         0x00000000,
 167         (0x0e00 << 16) | (0xc178 >> 2),
 168         0x00000000,
 169         (0x0e00 << 16) | (0xc204 >> 2),
 170         0x00000000,
 171         (0x0e00 << 16) | (0xc2b4 >> 2),
 172         0x00000000,
 173         (0x0e00 << 16) | (0xc2b8 >> 2),
 174         0x00000000,
 175         (0x0e00 << 16) | (0xc2bc >> 2),
 176         0x00000000,
 177         (0x0e00 << 16) | (0xc2c0 >> 2),
 178         0x00000000,
 179         (0x0e00 << 16) | (0x8228 >> 2),
 180         0x00000000,
 181         (0x0e00 << 16) | (0x829c >> 2),
 182         0x00000000,
 183         (0x0e00 << 16) | (0x869c >> 2),
 184         0x00000000,
 185         (0x0600 << 16) | (0x98f4 >> 2),
 186         0x00000000,
 187         (0x0e00 << 16) | (0x98f8 >> 2),
 188         0x00000000,
 189         (0x0e00 << 16) | (0x9900 >> 2),
 190         0x00000000,
 191         (0x0e00 << 16) | (0xc260 >> 2),
 192         0x00000000,
 193         (0x0e00 << 16) | (0x90e8 >> 2),
 194         0x00000000,
 195         (0x0e00 << 16) | (0x3c000 >> 2),
 196         0x00000000,
 197         (0x0e00 << 16) | (0x3c00c >> 2),
 198         0x00000000,
 199         (0x0e00 << 16) | (0x8c1c >> 2),
 200         0x00000000,
 201         (0x0e00 << 16) | (0x9700 >> 2),
 202         0x00000000,
 203         (0x0e00 << 16) | (0xcd20 >> 2),
 204         0x00000000,
 205         (0x4e00 << 16) | (0xcd20 >> 2),
 206         0x00000000,
 207         (0x5e00 << 16) | (0xcd20 >> 2),
 208         0x00000000,
 209         (0x6e00 << 16) | (0xcd20 >> 2),
 210         0x00000000,
 211         (0x7e00 << 16) | (0xcd20 >> 2),
 212         0x00000000,
 213         (0x8e00 << 16) | (0xcd20 >> 2),
 214         0x00000000,
 215         (0x9e00 << 16) | (0xcd20 >> 2),
 216         0x00000000,
 217         (0xae00 << 16) | (0xcd20 >> 2),
 218         0x00000000,
 219         (0xbe00 << 16) | (0xcd20 >> 2),
 220         0x00000000,
 221         (0x0e00 << 16) | (0x89bc >> 2),
 222         0x00000000,
 223         (0x0e00 << 16) | (0x8900 >> 2),
 224         0x00000000,
 225         0x3,
 226         (0x0e00 << 16) | (0xc130 >> 2),
 227         0x00000000,
 228         (0x0e00 << 16) | (0xc134 >> 2),
 229         0x00000000,
 230         (0x0e00 << 16) | (0xc1fc >> 2),
 231         0x00000000,
 232         (0x0e00 << 16) | (0xc208 >> 2),
 233         0x00000000,
 234         (0x0e00 << 16) | (0xc264 >> 2),
 235         0x00000000,
 236         (0x0e00 << 16) | (0xc268 >> 2),
 237         0x00000000,
 238         (0x0e00 << 16) | (0xc26c >> 2),
 239         0x00000000,
 240         (0x0e00 << 16) | (0xc270 >> 2),
 241         0x00000000,
 242         (0x0e00 << 16) | (0xc274 >> 2),
 243         0x00000000,
 244         (0x0e00 << 16) | (0xc278 >> 2),
 245         0x00000000,
 246         (0x0e00 << 16) | (0xc27c >> 2),
 247         0x00000000,
 248         (0x0e00 << 16) | (0xc280 >> 2),
 249         0x00000000,
 250         (0x0e00 << 16) | (0xc284 >> 2),
 251         0x00000000,
 252         (0x0e00 << 16) | (0xc288 >> 2),
 253         0x00000000,
 254         (0x0e00 << 16) | (0xc28c >> 2),
 255         0x00000000,
 256         (0x0e00 << 16) | (0xc290 >> 2),
 257         0x00000000,
 258         (0x0e00 << 16) | (0xc294 >> 2),
 259         0x00000000,
 260         (0x0e00 << 16) | (0xc298 >> 2),
 261         0x00000000,
 262         (0x0e00 << 16) | (0xc29c >> 2),
 263         0x00000000,
 264         (0x0e00 << 16) | (0xc2a0 >> 2),
 265         0x00000000,
 266         (0x0e00 << 16) | (0xc2a4 >> 2),
 267         0x00000000,
 268         (0x0e00 << 16) | (0xc2a8 >> 2),
 269         0x00000000,
 270         (0x0e00 << 16) | (0xc2ac  >> 2),
 271         0x00000000,
 272         (0x0e00 << 16) | (0xc2b0 >> 2),
 273         0x00000000,
 274         (0x0e00 << 16) | (0x301d0 >> 2),
 275         0x00000000,
 276         (0x0e00 << 16) | (0x30238 >> 2),
 277         0x00000000,
 278         (0x0e00 << 16) | (0x30250 >> 2),
 279         0x00000000,
 280         (0x0e00 << 16) | (0x30254 >> 2),
 281         0x00000000,
 282         (0x0e00 << 16) | (0x30258 >> 2),
 283         0x00000000,
 284         (0x0e00 << 16) | (0x3025c >> 2),
 285         0x00000000,
 286         (0x4e00 << 16) | (0xc900 >> 2),
 287         0x00000000,
 288         (0x5e00 << 16) | (0xc900 >> 2),
 289         0x00000000,
 290         (0x6e00 << 16) | (0xc900 >> 2),
 291         0x00000000,
 292         (0x7e00 << 16) | (0xc900 >> 2),
 293         0x00000000,
 294         (0x8e00 << 16) | (0xc900 >> 2),
 295         0x00000000,
 296         (0x9e00 << 16) | (0xc900 >> 2),
 297         0x00000000,
 298         (0xae00 << 16) | (0xc900 >> 2),
 299         0x00000000,
 300         (0xbe00 << 16) | (0xc900 >> 2),
 301         0x00000000,
 302         (0x4e00 << 16) | (0xc904 >> 2),
 303         0x00000000,
 304         (0x5e00 << 16) | (0xc904 >> 2),
 305         0x00000000,
 306         (0x6e00 << 16) | (0xc904 >> 2),
 307         0x00000000,
 308         (0x7e00 << 16) | (0xc904 >> 2),
 309         0x00000000,
 310         (0x8e00 << 16) | (0xc904 >> 2),
 311         0x00000000,
 312         (0x9e00 << 16) | (0xc904 >> 2),
 313         0x00000000,
 314         (0xae00 << 16) | (0xc904 >> 2),
 315         0x00000000,
 316         (0xbe00 << 16) | (0xc904 >> 2),
 317         0x00000000,
 318         (0x4e00 << 16) | (0xc908 >> 2),
 319         0x00000000,
 320         (0x5e00 << 16) | (0xc908 >> 2),
 321         0x00000000,
 322         (0x6e00 << 16) | (0xc908 >> 2),
 323         0x00000000,
 324         (0x7e00 << 16) | (0xc908 >> 2),
 325         0x00000000,
 326         (0x8e00 << 16) | (0xc908 >> 2),
 327         0x00000000,
 328         (0x9e00 << 16) | (0xc908 >> 2),
 329         0x00000000,
 330         (0xae00 << 16) | (0xc908 >> 2),
 331         0x00000000,
 332         (0xbe00 << 16) | (0xc908 >> 2),
 333         0x00000000,
 334         (0x4e00 << 16) | (0xc90c >> 2),
 335         0x00000000,
 336         (0x5e00 << 16) | (0xc90c >> 2),
 337         0x00000000,
 338         (0x6e00 << 16) | (0xc90c >> 2),
 339         0x00000000,
 340         (0x7e00 << 16) | (0xc90c >> 2),
 341         0x00000000,
 342         (0x8e00 << 16) | (0xc90c >> 2),
 343         0x00000000,
 344         (0x9e00 << 16) | (0xc90c >> 2),
 345         0x00000000,
 346         (0xae00 << 16) | (0xc90c >> 2),
 347         0x00000000,
 348         (0xbe00 << 16) | (0xc90c >> 2),
 349         0x00000000,
 350         (0x4e00 << 16) | (0xc910 >> 2),
 351         0x00000000,
 352         (0x5e00 << 16) | (0xc910 >> 2),
 353         0x00000000,
 354         (0x6e00 << 16) | (0xc910 >> 2),
 355         0x00000000,
 356         (0x7e00 << 16) | (0xc910 >> 2),
 357         0x00000000,
 358         (0x8e00 << 16) | (0xc910 >> 2),
 359         0x00000000,
 360         (0x9e00 << 16) | (0xc910 >> 2),
 361         0x00000000,
 362         (0xae00 << 16) | (0xc910 >> 2),
 363         0x00000000,
 364         (0xbe00 << 16) | (0xc910 >> 2),
 365         0x00000000,
 366         (0x0e00 << 16) | (0xc99c >> 2),
 367         0x00000000,
 368         (0x0e00 << 16) | (0x9834 >> 2),
 369         0x00000000,
 370         (0x0000 << 16) | (0x30f00 >> 2),
 371         0x00000000,
 372         (0x0001 << 16) | (0x30f00 >> 2),
 373         0x00000000,
 374         (0x0000 << 16) | (0x30f04 >> 2),
 375         0x00000000,
 376         (0x0001 << 16) | (0x30f04 >> 2),
 377         0x00000000,
 378         (0x0000 << 16) | (0x30f08 >> 2),
 379         0x00000000,
 380         (0x0001 << 16) | (0x30f08 >> 2),
 381         0x00000000,
 382         (0x0000 << 16) | (0x30f0c >> 2),
 383         0x00000000,
 384         (0x0001 << 16) | (0x30f0c >> 2),
 385         0x00000000,
 386         (0x0600 << 16) | (0x9b7c >> 2),
 387         0x00000000,
 388         (0x0e00 << 16) | (0x8a14 >> 2),
 389         0x00000000,
 390         (0x0e00 << 16) | (0x8a18 >> 2),
 391         0x00000000,
 392         (0x0600 << 16) | (0x30a00 >> 2),
 393         0x00000000,
 394         (0x0e00 << 16) | (0x8bf0 >> 2),
 395         0x00000000,
 396         (0x0e00 << 16) | (0x8bcc >> 2),
 397         0x00000000,
 398         (0x0e00 << 16) | (0x8b24 >> 2),
 399         0x00000000,
 400         (0x0e00 << 16) | (0x30a04 >> 2),
 401         0x00000000,
 402         (0x0600 << 16) | (0x30a10 >> 2),
 403         0x00000000,
 404         (0x0600 << 16) | (0x30a14 >> 2),
 405         0x00000000,
 406         (0x0600 << 16) | (0x30a18 >> 2),
 407         0x00000000,
 408         (0x0600 << 16) | (0x30a2c >> 2),
 409         0x00000000,
 410         (0x0e00 << 16) | (0xc700 >> 2),
 411         0x00000000,
 412         (0x0e00 << 16) | (0xc704 >> 2),
 413         0x00000000,
 414         (0x0e00 << 16) | (0xc708 >> 2),
 415         0x00000000,
 416         (0x0e00 << 16) | (0xc768 >> 2),
 417         0x00000000,
 418         (0x0400 << 16) | (0xc770 >> 2),
 419         0x00000000,
 420         (0x0400 << 16) | (0xc774 >> 2),
 421         0x00000000,
 422         (0x0400 << 16) | (0xc778 >> 2),
 423         0x00000000,
 424         (0x0400 << 16) | (0xc77c >> 2),
 425         0x00000000,
 426         (0x0400 << 16) | (0xc780 >> 2),
 427         0x00000000,
 428         (0x0400 << 16) | (0xc784 >> 2),
 429         0x00000000,
 430         (0x0400 << 16) | (0xc788 >> 2),
 431         0x00000000,
 432         (0x0400 << 16) | (0xc78c >> 2),
 433         0x00000000,
 434         (0x0400 << 16) | (0xc798 >> 2),
 435         0x00000000,
 436         (0x0400 << 16) | (0xc79c >> 2),
 437         0x00000000,
 438         (0x0400 << 16) | (0xc7a0 >> 2),
 439         0x00000000,
 440         (0x0400 << 16) | (0xc7a4 >> 2),
 441         0x00000000,
 442         (0x0400 << 16) | (0xc7a8 >> 2),
 443         0x00000000,
 444         (0x0400 << 16) | (0xc7ac >> 2),
 445         0x00000000,
 446         (0x0400 << 16) | (0xc7b0 >> 2),
 447         0x00000000,
 448         (0x0400 << 16) | (0xc7b4 >> 2),
 449         0x00000000,
 450         (0x0e00 << 16) | (0x9100 >> 2),
 451         0x00000000,
 452         (0x0e00 << 16) | (0x3c010 >> 2),
 453         0x00000000,
 454         (0x0e00 << 16) | (0x92a8 >> 2),
 455         0x00000000,
 456         (0x0e00 << 16) | (0x92ac >> 2),
 457         0x00000000,
 458         (0x0e00 << 16) | (0x92b4 >> 2),
 459         0x00000000,
 460         (0x0e00 << 16) | (0x92b8 >> 2),
 461         0x00000000,
 462         (0x0e00 << 16) | (0x92bc >> 2),
 463         0x00000000,
 464         (0x0e00 << 16) | (0x92c0 >> 2),
 465         0x00000000,
 466         (0x0e00 << 16) | (0x92c4 >> 2),
 467         0x00000000,
 468         (0x0e00 << 16) | (0x92c8 >> 2),
 469         0x00000000,
 470         (0x0e00 << 16) | (0x92cc >> 2),
 471         0x00000000,
 472         (0x0e00 << 16) | (0x92d0 >> 2),
 473         0x00000000,
 474         (0x0e00 << 16) | (0x8c00 >> 2),
 475         0x00000000,
 476         (0x0e00 << 16) | (0x8c04 >> 2),
 477         0x00000000,
 478         (0x0e00 << 16) | (0x8c20 >> 2),
 479         0x00000000,
 480         (0x0e00 << 16) | (0x8c38 >> 2),
 481         0x00000000,
 482         (0x0e00 << 16) | (0x8c3c >> 2),
 483         0x00000000,
 484         (0x0e00 << 16) | (0xae00 >> 2),
 485         0x00000000,
 486         (0x0e00 << 16) | (0x9604 >> 2),
 487         0x00000000,
 488         (0x0e00 << 16) | (0xac08 >> 2),
 489         0x00000000,
 490         (0x0e00 << 16) | (0xac0c >> 2),
 491         0x00000000,
 492         (0x0e00 << 16) | (0xac10 >> 2),
 493         0x00000000,
 494         (0x0e00 << 16) | (0xac14 >> 2),
 495         0x00000000,
 496         (0x0e00 << 16) | (0xac58 >> 2),
 497         0x00000000,
 498         (0x0e00 << 16) | (0xac68 >> 2),
 499         0x00000000,
 500         (0x0e00 << 16) | (0xac6c >> 2),
 501         0x00000000,
 502         (0x0e00 << 16) | (0xac70 >> 2),
 503         0x00000000,
 504         (0x0e00 << 16) | (0xac74 >> 2),
 505         0x00000000,
 506         (0x0e00 << 16) | (0xac78 >> 2),
 507         0x00000000,
 508         (0x0e00 << 16) | (0xac7c >> 2),
 509         0x00000000,
 510         (0x0e00 << 16) | (0xac80 >> 2),
 511         0x00000000,
 512         (0x0e00 << 16) | (0xac84 >> 2),
 513         0x00000000,
 514         (0x0e00 << 16) | (0xac88 >> 2),
 515         0x00000000,
 516         (0x0e00 << 16) | (0xac8c >> 2),
 517         0x00000000,
 518         (0x0e00 << 16) | (0x970c >> 2),
 519         0x00000000,
 520         (0x0e00 << 16) | (0x9714 >> 2),
 521         0x00000000,
 522         (0x0e00 << 16) | (0x9718 >> 2),
 523         0x00000000,
 524         (0x0e00 << 16) | (0x971c >> 2),
 525         0x00000000,
 526         (0x0e00 << 16) | (0x31068 >> 2),
 527         0x00000000,
 528         (0x4e00 << 16) | (0x31068 >> 2),
 529         0x00000000,
 530         (0x5e00 << 16) | (0x31068 >> 2),
 531         0x00000000,
 532         (0x6e00 << 16) | (0x31068 >> 2),
 533         0x00000000,
 534         (0x7e00 << 16) | (0x31068 >> 2),
 535         0x00000000,
 536         (0x8e00 << 16) | (0x31068 >> 2),
 537         0x00000000,
 538         (0x9e00 << 16) | (0x31068 >> 2),
 539         0x00000000,
 540         (0xae00 << 16) | (0x31068 >> 2),
 541         0x00000000,
 542         (0xbe00 << 16) | (0x31068 >> 2),
 543         0x00000000,
 544         (0x0e00 << 16) | (0xcd10 >> 2),
 545         0x00000000,
 546         (0x0e00 << 16) | (0xcd14 >> 2),
 547         0x00000000,
 548         (0x0e00 << 16) | (0x88b0 >> 2),
 549         0x00000000,
 550         (0x0e00 << 16) | (0x88b4 >> 2),
 551         0x00000000,
 552         (0x0e00 << 16) | (0x88b8 >> 2),
 553         0x00000000,
 554         (0x0e00 << 16) | (0x88bc >> 2),
 555         0x00000000,
 556         (0x0400 << 16) | (0x89c0 >> 2),
 557         0x00000000,
 558         (0x0e00 << 16) | (0x88c4 >> 2),
 559         0x00000000,
 560         (0x0e00 << 16) | (0x88c8 >> 2),
 561         0x00000000,
 562         (0x0e00 << 16) | (0x88d0 >> 2),
 563         0x00000000,
 564         (0x0e00 << 16) | (0x88d4 >> 2),
 565         0x00000000,
 566         (0x0e00 << 16) | (0x88d8 >> 2),
 567         0x00000000,
 568         (0x0e00 << 16) | (0x8980 >> 2),
 569         0x00000000,
 570         (0x0e00 << 16) | (0x30938 >> 2),
 571         0x00000000,
 572         (0x0e00 << 16) | (0x3093c >> 2),
 573         0x00000000,
 574         (0x0e00 << 16) | (0x30940 >> 2),
 575         0x00000000,
 576         (0x0e00 << 16) | (0x89a0 >> 2),
 577         0x00000000,
 578         (0x0e00 << 16) | (0x30900 >> 2),
 579         0x00000000,
 580         (0x0e00 << 16) | (0x30904 >> 2),
 581         0x00000000,
 582         (0x0e00 << 16) | (0x89b4 >> 2),
 583         0x00000000,
 584         (0x0e00 << 16) | (0x3c210 >> 2),
 585         0x00000000,
 586         (0x0e00 << 16) | (0x3c214 >> 2),
 587         0x00000000,
 588         (0x0e00 << 16) | (0x3c218 >> 2),
 589         0x00000000,
 590         (0x0e00 << 16) | (0x8904 >> 2),
 591         0x00000000,
 592         0x5,
 593         (0x0e00 << 16) | (0x8c28 >> 2),
 594         (0x0e00 << 16) | (0x8c2c >> 2),
 595         (0x0e00 << 16) | (0x8c30 >> 2),
 596         (0x0e00 << 16) | (0x8c34 >> 2),
 597         (0x0e00 << 16) | (0x9600 >> 2),
 598 };
 599
 600 static const u32 kalindi_rlc_save_restore_register_list[] =
 601 {
 602         (0x0e00 << 16) | (0xc12c >> 2),
 603         0x00000000,
 604         (0x0e00 << 16) | (0xc140 >> 2),
 605         0x00000000,
 606         (0x0e00 << 16) | (0xc150 >> 2),
 607         0x00000000,
 608         (0x0e00 << 16) | (0xc15c >> 2),
 609         0x00000000,
 610         (0x0e00 << 16) | (0xc168 >> 2),
 611         0x00000000,
 612         (0x0e00 << 16) | (0xc170 >> 2),
 613         0x00000000,
 614         (0x0e00 << 16) | (0xc204 >> 2),
 615         0x00000000,
 616         (0x0e00 << 16) | (0xc2b4 >> 2),
 617         0x00000000,
 618         (0x0e00 << 16) | (0xc2b8 >> 2),
 619         0x00000000,
 620         (0x0e00 << 16) | (0xc2bc >> 2),
 621         0x00000000,
 622         (0x0e00 << 16) | (0xc2c0 >> 2),
 623         0x00000000,
 624         (0x0e00 << 16) | (0x8228 >> 2),
 625         0x00000000,
 626         (0x0e00 << 16) | (0x829c >> 2),
 627         0x00000000,
 628         (0x0e00 << 16) | (0x869c >> 2),
 629         0x00000000,
 630         (0x0600 << 16) | (0x98f4 >> 2),
 631         0x00000000,
 632         (0x0e00 << 16) | (0x98f8 >> 2),
 633         0x00000000,
 634         (0x0e00 << 16) | (0x9900 >> 2),
 635         0x00000000,
 636         (0x0e00 << 16) | (0xc260 >> 2),
 637         0x00000000,
 638         (0x0e00 << 16) | (0x90e8 >> 2),
 639         0x00000000,
 640         (0x0e00 << 16) | (0x3c000 >> 2),
 641         0x00000000,
 642         (0x0e00 << 16) | (0x3c00c >> 2),
 643         0x00000000,
 644         (0x0e00 << 16) | (0x8c1c >> 2),
 645         0x00000000,
 646         (0x0e00 << 16) | (0x9700 >> 2),
 647         0x00000000,
 648         (0x0e00 << 16) | (0xcd20 >> 2),
 649         0x00000000,
 650         (0x4e00 << 16) | (0xcd20 >> 2),
 651         0x00000000,
 652         (0x5e00 << 16) | (0xcd20 >> 2),
 653         0x00000000,
 654         (0x6e00 << 16) | (0xcd20 >> 2),
 655         0x00000000,
 656         (0x7e00 << 16) | (0xcd20 >> 2),
 657         0x00000000,
 658         (0x0e00 << 16) | (0x89bc >> 2),
 659         0x00000000,
 660         (0x0e00 << 16) | (0x8900 >> 2),
 661         0x00000000,
 662         0x3,
 663         (0x0e00 << 16) | (0xc130 >> 2),
 664         0x00000000,
 665         (0x0e00 << 16) | (0xc134 >> 2),
 666         0x00000000,
 667         (0x0e00 << 16) | (0xc1fc >> 2),
 668         0x00000000,
 669         (0x0e00 << 16) | (0xc208 >> 2),
 670         0x00000000,
 671         (0x0e00 << 16) | (0xc264 >> 2),
 672         0x00000000,
 673         (0x0e00 << 16) | (0xc268 >> 2),
 674         0x00000000,
 675         (0x0e00 << 16) | (0xc26c >> 2),
 676         0x00000000,
 677         (0x0e00 << 16) | (0xc270 >> 2),
 678         0x00000000,
 679         (0x0e00 << 16) | (0xc274 >> 2),
 680         0x00000000,
 681         (0x0e00 << 16) | (0xc28c >> 2),
 682         0x00000000,
 683         (0x0e00 << 16) | (0xc290 >> 2),
 684         0x00000000,
 685         (0x0e00 << 16) | (0xc294 >> 2),
 686         0x00000000,
 687         (0x0e00 << 16) | (0xc298 >> 2),
 688         0x00000000,
 689         (0x0e00 << 16) | (0xc2a0 >> 2),
 690         0x00000000,
 691         (0x0e00 << 16) | (0xc2a4 >> 2),
 692         0x00000000,
 693         (0x0e00 << 16) | (0xc2a8 >> 2),
 694         0x00000000,
 695         (0x0e00 << 16) | (0xc2ac >> 2),
 696         0x00000000,
 697         (0x0e00 << 16) | (0x301d0 >> 2),
 698         0x00000000,
 699         (0x0e00 << 16) | (0x30238 >> 2),
 700         0x00000000,
 701         (0x0e00 << 16) | (0x30250 >> 2),
 702         0x00000000,
 703         (0x0e00 << 16) | (0x30254 >> 2),
 704         0x00000000,
 705         (0x0e00 << 16) | (0x30258 >> 2),
 706         0x00000000,
 707         (0x0e00 << 16) | (0x3025c >> 2),
 708         0x00000000,
 709         (0x4e00 << 16) | (0xc900 >> 2),
 710         0x00000000,
 711         (0x5e00 << 16) | (0xc900 >> 2),
 712         0x00000000,
 713         (0x6e00 << 16) | (0xc900 >> 2),
 714         0x00000000,
 715         (0x7e00 << 16) | (0xc900 >> 2),
 716         0x00000000,
 717         (0x4e00 << 16) | (0xc904 >> 2),
 718         0x00000000,
 719         (0x5e00 << 16) | (0xc904 >> 2),
 720         0x00000000,
 721         (0x6e00 << 16) | (0xc904 >> 2),
 722         0x00000000,
 723         (0x7e00 << 16) | (0xc904 >> 2),
 724         0x00000000,
 725         (0x4e00 << 16) | (0xc908 >> 2),
 726         0x00000000,
 727         (0x5e00 << 16) | (0xc908 >> 2),
 728         0x00000000,
 729         (0x6e00 << 16) | (0xc908 >> 2),
 730         0x00000000,
 731         (0x7e00 << 16) | (0xc908 >> 2),
 732         0x00000000,
 733         (0x4e00 << 16) | (0xc90c >> 2),
 734         0x00000000,
 735         (0x5e00 << 16) | (0xc90c >> 2),
 736         0x00000000,
 737         (0x6e00 << 16) | (0xc90c >> 2),
 738         0x00000000,
 739         (0x7e00 << 16) | (0xc90c >> 2),
 740         0x00000000,
 741         (0x4e00 << 16) | (0xc910 >> 2),
 742         0x00000000,
 743         (0x5e00 << 16) | (0xc910 >> 2),
 744         0x00000000,
 745         (0x6e00 << 16) | (0xc910 >> 2),
 746         0x00000000,
 747         (0x7e00 << 16) | (0xc910 >> 2),
 748         0x00000000,
 749         (0x0e00 << 16) | (0xc99c >> 2),
 750         0x00000000,
 751         (0x0e00 << 16) | (0x9834 >> 2),
 752         0x00000000,
 753         (0x0000 << 16) | (0x30f00 >> 2),
 754         0x00000000,
 755         (0x0000 << 16) | (0x30f04 >> 2),
 756         0x00000000,
 757         (0x0000 << 16) | (0x30f08 >> 2),
 758         0x00000000,
 759         (0x0000 << 16) | (0x30f0c >> 2),
 760         0x00000000,
 761         (0x0600 << 16) | (0x9b7c >> 2),
 762         0x00000000,
 763         (0x0e00 << 16) | (0x8a14 >> 2),
 764         0x00000000,
 765         (0x0e00 << 16) | (0x8a18 >> 2),
 766         0x00000000,
 767         (0x0600 << 16) | (0x30a00 >> 2),
 768         0x00000000,
 769         (0x0e00 << 16) | (0x8bf0 >> 2),
 770         0x00000000,
 771         (0x0e00 << 16) | (0x8bcc >> 2),
 772         0x00000000,
 773         (0x0e00 << 16) | (0x8b24 >> 2),
 774         0x00000000,
 775         (0x0e00 << 16) | (0x30a04 >> 2),
 776         0x00000000,
 777         (0x0600 << 16) | (0x30a10 >> 2),
 778         0x00000000,
 779         (0x0600 << 16) | (0x30a14 >> 2),
 780         0x00000000,
 781         (0x0600 << 16) | (0x30a18 >> 2),
 782         0x00000000,
 783         (0x0600 << 16) | (0x30a2c >> 2),
 784         0x00000000,
 785         (0x0e00 << 16) | (0xc700 >> 2),
 786         0x00000000,
 787         (0x0e00 << 16) | (0xc704 >> 2),
 788         0x00000000,
 789         (0x0e00 << 16) | (0xc708 >> 2),
 790         0x00000000,
 791         (0x0e00 << 16) | (0xc768 >> 2),
 792         0x00000000,
 793         (0x0400 << 16) | (0xc770 >> 2),
 794         0x00000000,
 795         (0x0400 << 16) | (0xc774 >> 2),
 796         0x00000000,
 797         (0x0400 << 16) | (0xc798 >> 2),
 798         0x00000000,
 799         (0x0400 << 16) | (0xc79c >> 2),
 800         0x00000000,
 801         (0x0e00 << 16) | (0x9100 >> 2),
 802         0x00000000,
 803         (0x0e00 << 16) | (0x3c010 >> 2),
 804         0x00000000,
 805         (0x0e00 << 16) | (0x8c00 >> 2),
 806         0x00000000,
 807         (0x0e00 << 16) | (0x8c04 >> 2),
 808         0x00000000,
 809         (0x0e00 << 16) | (0x8c20 >> 2),
 810         0x00000000,
 811         (0x0e00 << 16) | (0x8c38 >> 2),
 812         0x00000000,
 813         (0x0e00 << 16) | (0x8c3c >> 2),
 814         0x00000000,
 815         (0x0e00 << 16) | (0xae00 >> 2),
 816         0x00000000,
 817         (0x0e00 << 16) | (0x9604 >> 2),
 818         0x00000000,
 819         (0x0e00 << 16) | (0xac08 >> 2),
 820         0x00000000,
 821         (0x0e00 << 16) | (0xac0c >> 2),
 822         0x00000000,
 823         (0x0e00 << 16) | (0xac10 >> 2),
 824         0x00000000,
 825         (0x0e00 << 16) | (0xac14 >> 2),
 826         0x00000000,
 827         (0x0e00 << 16) | (0xac58 >> 2),
 828         0x00000000,
 829         (0x0e00 << 16) | (0xac68 >> 2),
 830         0x00000000,
 831         (0x0e00 << 16) | (0xac6c >> 2),
 832         0x00000000,
 833         (0x0e00 << 16) | (0xac70 >> 2),
 834         0x00000000,
 835         (0x0e00 << 16) | (0xac74 >> 2),
 836         0x00000000,
 837         (0x0e00 << 16) | (0xac78 >> 2),
 838         0x00000000,
 839         (0x0e00 << 16) | (0xac7c >> 2),
 840         0x00000000,
 841         (0x0e00 << 16) | (0xac80 >> 2),
 842         0x00000000,
 843         (0x0e00 << 16) | (0xac84 >> 2),
 844         0x00000000,
 845         (0x0e00 << 16) | (0xac88 >> 2),
 846         0x00000000,
 847         (0x0e00 << 16) | (0xac8c >> 2),
 848         0x00000000,
 849         (0x0e00 << 16) | (0x970c >> 2),
 850         0x00000000,
 851         (0x0e00 << 16) | (0x9714 >> 2),
 852         0x00000000,
 853         (0x0e00 << 16) | (0x9718 >> 2),
 854         0x00000000,
 855         (0x0e00 << 16) | (0x971c >> 2),
 856         0x00000000,
 857         (0x0e00 << 16) | (0x31068 >> 2),
 858         0x00000000,
 859         (0x4e00 << 16) | (0x31068 >> 2),
 860         0x00000000,
 861         (0x5e00 << 16) | (0x31068 >> 2),
 862         0x00000000,
 863         (0x6e00 << 16) | (0x31068 >> 2),
 864         0x00000000,
 865         (0x7e00 << 16) | (0x31068 >> 2),
 866         0x00000000,
 867         (0x0e00 << 16) | (0xcd10 >> 2),
 868         0x00000000,
 869         (0x0e00 << 16) | (0xcd14 >> 2),
 870         0x00000000,
 871         (0x0e00 << 16) | (0x88b0 >> 2),
 872         0x00000000,
 873         (0x0e00 << 16) | (0x88b4 >> 2),
 874         0x00000000,
 875         (0x0e00 << 16) | (0x88b8 >> 2),
 876         0x00000000,
 877         (0x0e00 << 16) | (0x88bc >> 2),
 878         0x00000000,
 879         (0x0400 << 16) | (0x89c0 >> 2),
 880         0x00000000,
 881         (0x0e00 << 16) | (0x88c4 >> 2),
 882         0x00000000,
 883         (0x0e00 << 16) | (0x88c8 >> 2),
 884         0x00000000,
 885         (0x0e00 << 16) | (0x88d0 >> 2),
 886         0x00000000,
 887         (0x0e00 << 16) | (0x88d4 >> 2),
 888         0x00000000,
 889         (0x0e00 << 16) | (0x88d8 >> 2),
 890         0x00000000,
 891         (0x0e00 << 16) | (0x8980 >> 2),
 892         0x00000000,
 893         (0x0e00 << 16) | (0x30938 >> 2),
 894         0x00000000,
 895         (0x0e00 << 16) | (0x3093c >> 2),
 896         0x00000000,
 897         (0x0e00 << 16) | (0x30940 >> 2),
 898         0x00000000,
 899         (0x0e00 << 16) | (0x89a0 >> 2),
 900         0x00000000,
 901         (0x0e00 << 16) | (0x30900 >> 2),
 902         0x00000000,
 903         (0x0e00 << 16) | (0x30904 >> 2),
 904         0x00000000,
 905         (0x0e00 << 16) | (0x89b4 >> 2),
 906         0x00000000,
 907         (0x0e00 << 16) | (0x3e1fc >> 2),
 908         0x00000000,
 909         (0x0e00 << 16) | (0x3c210 >> 2),
 910         0x00000000,
 911         (0x0e00 << 16) | (0x3c214 >> 2),
 912         0x00000000,
 913         (0x0e00 << 16) | (0x3c218 >> 2),
 914         0x00000000,
 915         (0x0e00 << 16) | (0x8904 >> 2),
 916         0x00000000,
 917         0x5,
 918         (0x0e00 << 16) | (0x8c28 >> 2),
 919         (0x0e00 << 16) | (0x8c2c >> 2),
 920         (0x0e00 << 16) | (0x8c30 >> 2),
 921         (0x0e00 << 16) | (0x8c34 >> 2),
 922         (0x0e00 << 16) | (0x9600 >> 2),
 923 };
 924
 925 static const u32 bonaire_golden_spm_registers[] =
 926 {
 927         0x30800, 0xe0ffffff, 0xe0000000
 928 };
 929
 930 static const u32 bonaire_golden_common_registers[] =
 931 {
 932         0xc770, 0xffffffff, 0x00000800,
 933         0xc774, 0xffffffff, 0x00000800,
 934         0xc798, 0xffffffff, 0x00007fbf,
 935         0xc79c, 0xffffffff, 0x00007faf
 936 };
 937
 938 static const u32 bonaire_golden_registers[] =
 939 {
 940         0x3354, 0x00000333, 0x00000333,
 941         0x3350, 0x000c0fc0, 0x00040200,
 942         0x9a10, 0x00010000, 0x00058208,
 943         0x3c000, 0xffff1fff, 0x00140000,
 944         0x3c200, 0xfdfc0fff, 0x00000100,
 945         0x3c234, 0x40000000, 0x40000200,
 946         0x9830, 0xffffffff, 0x00000000,
 947         0x9834, 0xf00fffff, 0x00000400,
 948         0x9838, 0x0002021c, 0x00020200,
 949         0xc78, 0x00000080, 0x00000000,
 950         0x5bb0, 0x000000f0, 0x00000070,
 951         0x5bc0, 0xf0311fff, 0x80300000,
 952         0x98f8, 0x73773777, 0x12010001,
 953         0x350c, 0x00810000, 0x408af000,
 954         0x7030, 0x31000111, 0x00000011,
 955         0x2f48, 0x73773777, 0x12010001,
 956         0x220c, 0x00007fb6, 0x0021a1b1,
 957         0x2210, 0x00007fb6, 0x002021b1,
 958         0x2180, 0x00007fb6, 0x00002191,
 959         0x2218, 0x00007fb6, 0x002121b1,
 960         0x221c, 0x00007fb6, 0x002021b1,
 961         0x21dc, 0x00007fb6, 0x00002191,
 962         0x21e0, 0x00007fb6, 0x00002191,
 963         0x3628, 0x0000003f, 0x0000000a,
 964         0x362c, 0x0000003f, 0x0000000a,
 965         0x2ae4, 0x00073ffe, 0x000022a2,
 966         0x240c, 0x000007ff, 0x00000000,
 967         0x8a14, 0xf000003f, 0x00000007,
 968         0x8bf0, 0x00002001, 0x00000001,
 969         0x8b24, 0xffffffff, 0x00ffffff,
 970         0x30a04, 0x0000ff0f, 0x00000000,
 971         0x28a4c, 0x07ffffff, 0x06000000,
 972         0x4d8, 0x00000fff, 0x00000100,
 973         0x3e78, 0x00000001, 0x00000002,
 974         0x9100, 0x03000000, 0x0362c688,
 975         0x8c00, 0x000000ff, 0x00000001,
 976         0xe40, 0x00001fff, 0x00001fff,
 977         0x9060, 0x0000007f, 0x00000020,
 978         0x9508, 0x00010000, 0x00010000,
 979         0xac14, 0x000003ff, 0x000000f3,
 980         0xac0c, 0xffffffff, 0x00001032
 981 };
 982
 983 static const u32 bonaire_mgcg_cgcg_init[] =
 984 {
 985         0xc420, 0xffffffff, 0xfffffffc,
 986         0x30800, 0xffffffff, 0xe0000000,
 987         0x3c2a0, 0xffffffff, 0x00000100,
 988         0x3c208, 0xffffffff, 0x00000100,
 989         0x3c2c0, 0xffffffff, 0xc0000100,
 990         0x3c2c8, 0xffffffff, 0xc0000100,
 991         0x3c2c4, 0xffffffff, 0xc0000100,
 992         0x55e4, 0xffffffff, 0x00600100,
 993         0x3c280, 0xffffffff, 0x00000100,
 994         0x3c214, 0xffffffff, 0x06000100,
 995         0x3c220, 0xffffffff, 0x00000100,
 996         0x3c218, 0xffffffff, 0x06000100,
 997         0x3c204, 0xffffffff, 0x00000100,
 998         0x3c2e0, 0xffffffff, 0x00000100,
 999         0x3c224, 0xffffffff, 0x00000100,
1000         0x3c200, 0xffffffff, 0x00000100,
1001         0x3c230, 0xffffffff, 0x00000100,
1002         0x3c234, 0xffffffff, 0x00000100,
1003         0x3c250, 0xffffffff, 0x00000100,
1004         0x3c254, 0xffffffff, 0x00000100,
1005         0x3c258, 0xffffffff, 0x00000100,
1006         0x3c25c, 0xffffffff, 0x00000100,
1007         0x3c260, 0xffffffff, 0x00000100,
1008         0x3c27c, 0xffffffff, 0x00000100,
1009         0x3c278, 0xffffffff, 0x00000100,
1010         0x3c210, 0xffffffff, 0x06000100,
1011         0x3c290, 0xffffffff, 0x00000100,
1012         0x3c274, 0xffffffff, 0x00000100,
1013         0x3c2b4, 0xffffffff, 0x00000100,
1014         0x3c2b0, 0xffffffff, 0x00000100,
1015         0x3c270, 0xffffffff, 0x00000100,
1016         0x30800, 0xffffffff, 0xe0000000,
1017         0x3c020, 0xffffffff, 0x00010000,
1018         0x3c024, 0xffffffff, 0x00030002,
1019         0x3c028, 0xffffffff, 0x00040007,
1020         0x3c02c, 0xffffffff, 0x00060005,
1021         0x3c030, 0xffffffff, 0x00090008,
1022         0x3c034, 0xffffffff, 0x00010000,
1023         0x3c038, 0xffffffff, 0x00030002,
1024         0x3c03c, 0xffffffff, 0x00040007,
1025         0x3c040, 0xffffffff, 0x00060005,
1026         0x3c044, 0xffffffff, 0x00090008,
1027         0x3c048, 0xffffffff, 0x00010000,
1028         0x3c04c, 0xffffffff, 0x00030002,
1029         0x3c050, 0xffffffff, 0x00040007,
1030         0x3c054, 0xffffffff, 0x00060005,
1031         0x3c058, 0xffffffff, 0x00090008,
1032         0x3c05c, 0xffffffff, 0x00010000,
1033         0x3c060, 0xffffffff, 0x00030002,
1034         0x3c064, 0xffffffff, 0x00040007,
1035         0x3c068, 0xffffffff, 0x00060005,
1036         0x3c06c, 0xffffffff, 0x00090008,
1037         0x3c070, 0xffffffff, 0x00010000,
1038         0x3c074, 0xffffffff, 0x00030002,
1039         0x3c078, 0xffffffff, 0x00040007,
1040         0x3c07c, 0xffffffff, 0x00060005,
1041         0x3c080, 0xffffffff, 0x00090008,
1042         0x3c084, 0xffffffff, 0x00010000,
1043         0x3c088, 0xffffffff, 0x00030002,
1044         0x3c08c, 0xffffffff, 0x00040007,
1045         0x3c090, 0xffffffff, 0x00060005,
1046         0x3c094, 0xffffffff, 0x00090008,
1047         0x3c098, 0xffffffff, 0x00010000,
1048         0x3c09c, 0xffffffff, 0x00030002,
1049         0x3c0a0, 0xffffffff, 0x00040007,
1050         0x3c0a4, 0xffffffff, 0x00060005,
1051         0x3c0a8, 0xffffffff, 0x00090008,
1052         0x3c000, 0xffffffff, 0x96e00200,
1053         0x8708, 0xffffffff, 0x00900100,
1054         0xc424, 0xffffffff, 0x0020003f,
1055         0x38, 0xffffffff, 0x0140001c,
1056         0x3c, 0x000f0000, 0x000f0000,
1057         0x220, 0xffffffff, 0xC060000C,
1058         0x224, 0xc0000fff, 0x00000100,
1059         0xf90, 0xffffffff, 0x00000100,
1060         0xf98, 0x00000101, 0x00000000,
1061         0x20a8, 0xffffffff, 0x00000104,
1062         0x55e4, 0xff000fff, 0x00000100,
1063         0x30cc, 0xc0000fff, 0x00000104,
1064         0xc1e4, 0x00000001, 0x00000001,
1065         0xd00c, 0xff000ff0, 0x00000100,
1066         0xd80c, 0xff000ff0, 0x00000100
1067 };
1068
1069 static const u32 spectre_golden_spm_registers[] =
1070 {
1071         0x30800, 0xe0ffffff, 0xe0000000
1072 };
1073
1074 static const u32 spectre_golden_common_registers[] =
1075 {
1076         0xc770, 0xffffffff, 0x00000800,
1077         0xc774, 0xffffffff, 0x00000800,
1078         0xc798, 0xffffffff, 0x00007fbf,
1079         0xc79c, 0xffffffff, 0x00007faf
1080 };
1081
1082 static const u32 spectre_golden_registers[] =
1083 {
1084         0x3c000, 0xffff1fff, 0x96940200,
1085         0x3c00c, 0xffff0001, 0xff000000,
1086         0x3c200, 0xfffc0fff, 0x00000100,
1087         0x6ed8, 0x00010101, 0x00010000,
1088         0x9834, 0xf00fffff, 0x00000400,
1089         0x9838, 0xfffffffc, 0x00020200,
1090         0x5bb0, 0x000000f0, 0x00000070,
1091         0x5bc0, 0xf0311fff, 0x80300000,
1092         0x98f8, 0x73773777, 0x12010001,
1093         0x9b7c, 0x00ff0000, 0x00fc0000,
1094         0x2f48, 0x73773777, 0x12010001,
1095         0x8a14, 0xf000003f, 0x00000007,
1096         0x8b24, 0xffffffff, 0x00ffffff,
1097         0x28350, 0x3f3f3fff, 0x00000082,
1098         0x28355, 0x0000003f, 0x00000000,
1099         0x3e78, 0x00000001, 0x00000002,
1100         0x913c, 0xffff03df, 0x00000004,
1101         0xc768, 0x00000008, 0x00000008,
1102         0x8c00, 0x000008ff, 0x00000800,
1103         0x9508, 0x00010000, 0x00010000,
1104         0xac0c, 0xffffffff, 0x54763210,
1105         0x214f8, 0x01ff01ff, 0x00000002,
1106         0x21498, 0x007ff800, 0x00200000,
1107         0x2015c, 0xffffffff, 0x00000f40,
1108         0x30934, 0xffffffff, 0x00000001
1109 };
1110
1111 static const u32 spectre_mgcg_cgcg_init[] =
1112 {
1113         0xc420, 0xffffffff, 0xfffffffc,
1114         0x30800, 0xffffffff, 0xe0000000,
1115         0x3c2a0, 0xffffffff, 0x00000100,
1116         0x3c208, 0xffffffff, 0x00000100,
1117         0x3c2c0, 0xffffffff, 0x00000100,
1118         0x3c2c8, 0xffffffff, 0x00000100,
1119         0x3c2c4, 0xffffffff, 0x00000100,
1120         0x55e4, 0xffffffff, 0x00600100,
1121         0x3c280, 0xffffffff, 0x00000100,
1122         0x3c214, 0xffffffff, 0x06000100,
1123         0x3c220, 0xffffffff, 0x00000100,
1124         0x3c218, 0xffffffff, 0x06000100,
1125         0x3c204, 0xffffffff, 0x00000100,
1126         0x3c2e0, 0xffffffff, 0x00000100,
1127         0x3c224, 0xffffffff, 0x00000100,
1128         0x3c200, 0xffffffff, 0x00000100,
1129         0x3c230, 0xffffffff, 0x00000100,
1130         0x3c234, 0xffffffff, 0x00000100,
1131         0x3c250, 0xffffffff, 0x00000100,
1132         0x3c254, 0xffffffff, 0x00000100,
1133         0x3c258, 0xffffffff, 0x00000100,
1134         0x3c25c, 0xffffffff, 0x00000100,
1135         0x3c260, 0xffffffff, 0x00000100,
1136         0x3c27c, 0xffffffff, 0x00000100,
1137         0x3c278, 0xffffffff, 0x00000100,
1138         0x3c210, 0xffffffff, 0x06000100,
1139         0x3c290, 0xffffffff, 0x00000100,
1140         0x3c274, 0xffffffff, 0x00000100,
1141         0x3c2b4, 0xffffffff, 0x00000100,
1142         0x3c2b0, 0xffffffff, 0x00000100,
1143         0x3c270, 0xffffffff, 0x00000100,
1144         0x30800, 0xffffffff, 0xe0000000,
1145         0x3c020, 0xffffffff, 0x00010000,
1146         0x3c024, 0xffffffff, 0x00030002,
1147         0x3c028, 0xffffffff, 0x00040007,
1148         0x3c02c, 0xffffffff, 0x00060005,
1149         0x3c030, 0xffffffff, 0x00090008,
1150         0x3c034, 0xffffffff, 0x00010000,
1151         0x3c038, 0xffffffff, 0x00030002,
1152         0x3c03c, 0xffffffff, 0x00040007,
1153         0x3c040, 0xffffffff, 0x00060005,
1154         0x3c044, 0xffffffff, 0x00090008,
1155         0x3c048, 0xffffffff, 0x00010000,
1156         0x3c04c, 0xffffffff, 0x00030002,
1157         0x3c050, 0xffffffff, 0x00040007,
1158         0x3c054, 0xffffffff, 0x00060005,
1159         0x3c058, 0xffffffff, 0x00090008,
1160         0x3c05c, 0xffffffff, 0x00010000,
1161         0x3c060, 0xffffffff, 0x00030002,
1162         0x3c064, 0xffffffff, 0x00040007,
1163         0x3c068, 0xffffffff, 0x00060005,
1164         0x3c06c, 0xffffffff, 0x00090008,
1165         0x3c070, 0xffffffff, 0x00010000,
1166         0x3c074, 0xffffffff, 0x00030002,
1167         0x3c078, 0xffffffff, 0x00040007,
1168         0x3c07c, 0xffffffff, 0x00060005,
1169         0x3c080, 0xffffffff, 0x00090008,
1170         0x3c084, 0xffffffff, 0x00010000,
1171         0x3c088, 0xffffffff, 0x00030002,
1172         0x3c08c, 0xffffffff, 0x00040007,
1173         0x3c090, 0xffffffff, 0x00060005,
1174         0x3c094, 0xffffffff, 0x00090008,
1175         0x3c098, 0xffffffff, 0x00010000,
1176         0x3c09c, 0xffffffff, 0x00030002,
1177         0x3c0a0, 0xffffffff, 0x00040007,
1178         0x3c0a4, 0xffffffff, 0x00060005,
1179         0x3c0a8, 0xffffffff, 0x00090008,
1180         0x3c0ac, 0xffffffff, 0x00010000,
1181         0x3c0b0, 0xffffffff, 0x00030002,
1182         0x3c0b4, 0xffffffff, 0x00040007,
1183         0x3c0b8, 0xffffffff, 0x00060005,
1184         0x3c0bc, 0xffffffff, 0x00090008,
1185         0x3c000, 0xffffffff, 0x96e00200,
1186         0x8708, 0xffffffff, 0x00900100,
1187         0xc424, 0xffffffff, 0x0020003f,
1188         0x38, 0xffffffff, 0x0140001c,
1189         0x3c, 0x000f0000, 0x000f0000,
1190         0x220, 0xffffffff, 0xC060000C,
1191         0x224, 0xc0000fff, 0x00000100,
1192         0xf90, 0xffffffff, 0x00000100,
1193         0xf98, 0x00000101, 0x00000000,
1194         0x20a8, 0xffffffff, 0x00000104,
1195         0x55e4, 0xff000fff, 0x00000100,
1196         0x30cc, 0xc0000fff, 0x00000104,
1197         0xc1e4, 0x00000001, 0x00000001,
1198         0xd00c, 0xff000ff0, 0x00000100,
1199         0xd80c, 0xff000ff0, 0x00000100
1200 };
1201
1202 static const u32 kalindi_golden_spm_registers[] =
1203 {
1204         0x30800, 0xe0ffffff, 0xe0000000
1205 };
1206
1207 static const u32 kalindi_golden_common_registers[] =
1208 {
1209         0xc770, 0xffffffff, 0x00000800,
1210         0xc774, 0xffffffff, 0x00000800,
1211         0xc798, 0xffffffff, 0x00007fbf,
1212         0xc79c, 0xffffffff, 0x00007faf
1213 };
1214
1215 static const u32 kalindi_golden_registers[] =
1216 {
1217         0x3c000, 0xffffdfff, 0x6e944040,
1218         0x55e4, 0xff607fff, 0xfc000100,
1219         0x3c220, 0xff000fff, 0x00000100,
1220         0x3c224, 0xff000fff, 0x00000100,
1221         0x3c200, 0xfffc0fff, 0x00000100,
1222         0x6ed8, 0x00010101, 0x00010000,
1223         0x9830, 0xffffffff, 0x00000000,
1224         0x9834, 0xf00fffff, 0x00000400,
1225         0x5bb0, 0x000000f0, 0x00000070,
1226         0x5bc0, 0xf0311fff, 0x80300000,
1227         0x98f8, 0x73773777, 0x12010001,
1228         0x98fc, 0xffffffff, 0x00000010,
1229         0x9b7c, 0x00ff0000, 0x00fc0000,
1230         0x8030, 0x00001f0f, 0x0000100a,
1231         0x2f48, 0x73773777, 0x12010001,
1232         0x2408, 0x000fffff, 0x000c007f,
1233         0x8a14, 0xf000003f, 0x00000007,
1234         0x8b24, 0x3fff3fff, 0x00ffcfff,
1235         0x30a04, 0x0000ff0f, 0x00000000,
1236         0x28a4c, 0x07ffffff, 0x06000000,
1237         0x4d8, 0x00000fff, 0x00000100,
1238         0x3e78, 0x00000001, 0x00000002,
1239         0xc768, 0x00000008, 0x00000008,
1240         0x8c00, 0x000000ff, 0x00000003,
1241         0x214f8, 0x01ff01ff, 0x00000002,
1242         0x21498, 0x007ff800, 0x00200000,
1243         0x2015c, 0xffffffff, 0x00000f40,
1244         0x88c4, 0x001f3ae3, 0x00000082,
1245         0x88d4, 0x0000001f, 0x00000010,
1246         0x30934, 0xffffffff, 0x00000000
1247 };
1248
1249 static const u32 kalindi_mgcg_cgcg_init[] =
1250 {
1251         0xc420, 0xffffffff, 0xfffffffc,
1252         0x30800, 0xffffffff, 0xe0000000,
1253         0x3c2a0, 0xffffffff, 0x00000100,
1254         0x3c208, 0xffffffff, 0x00000100,
1255         0x3c2c0, 0xffffffff, 0x00000100,
1256         0x3c2c8, 0xffffffff, 0x00000100,
1257         0x3c2c4, 0xffffffff, 0x00000100,
1258         0x55e4, 0xffffffff, 0x00600100,
1259         0x3c280, 0xffffffff, 0x00000100,
1260         0x3c214, 0xffffffff, 0x06000100,
1261         0x3c220, 0xffffffff, 0x00000100,
1262         0x3c218, 0xffffffff, 0x06000100,
1263         0x3c204, 0xffffffff, 0x00000100,
1264         0x3c2e0, 0xffffffff, 0x00000100,
1265         0x3c224, 0xffffffff, 0x00000100,
1266         0x3c200, 0xffffffff, 0x00000100,
1267         0x3c230, 0xffffffff, 0x00000100,
1268         0x3c234, 0xffffffff, 0x00000100,
1269         0x3c250, 0xffffffff, 0x00000100,
1270         0x3c254, 0xffffffff, 0x00000100,
1271         0x3c258, 0xffffffff, 0x00000100,
1272         0x3c25c, 0xffffffff, 0x00000100,
1273         0x3c260, 0xffffffff, 0x00000100,
1274         0x3c27c, 0xffffffff, 0x00000100,
1275         0x3c278, 0xffffffff, 0x00000100,
1276         0x3c210, 0xffffffff, 0x06000100,
1277         0x3c290, 0xffffffff, 0x00000100,
1278         0x3c274, 0xffffffff, 0x00000100,
1279         0x3c2b4, 0xffffffff, 0x00000100,
1280         0x3c2b0, 0xffffffff, 0x00000100,
1281         0x3c270, 0xffffffff, 0x00000100,
1282         0x30800, 0xffffffff, 0xe0000000,
1283         0x3c020, 0xffffffff, 0x00010000,
1284         0x3c024, 0xffffffff, 0x00030002,
1285         0x3c028, 0xffffffff, 0x00040007,
1286         0x3c02c, 0xffffffff, 0x00060005,
1287         0x3c030, 0xffffffff, 0x00090008,
1288         0x3c034, 0xffffffff, 0x00010000,
1289         0x3c038, 0xffffffff, 0x00030002,
1290         0x3c03c, 0xffffffff, 0x00040007,
1291         0x3c040, 0xffffffff, 0x00060005,
1292         0x3c044, 0xffffffff, 0x00090008,
1293         0x3c000, 0xffffffff, 0x96e00200,
1294         0x8708, 0xffffffff, 0x00900100,
1295         0xc424, 0xffffffff, 0x0020003f,
1296         0x38, 0xffffffff, 0x0140001c,
1297         0x3c, 0x000f0000, 0x000f0000,
1298         0x220, 0xffffffff, 0xC060000C,
1299         0x224, 0xc0000fff, 0x00000100,
1300         0x20a8, 0xffffffff, 0x00000104,
1301         0x55e4, 0xff000fff, 0x00000100,
1302         0x30cc, 0xc0000fff, 0x00000104,
1303         0xc1e4, 0x00000001, 0x00000001,
1304         0xd00c, 0xff000ff0, 0x00000100,
1305         0xd80c, 0xff000ff0, 0x00000100
1306 };
1307
1308 static const u32 hawaii_golden_spm_registers[] =
1309 {
1310         0x30800, 0xe0ffffff, 0xe0000000
1311 };
1312
1313 static const u32 hawaii_golden_common_registers[] =
1314 {
1315         0x30800, 0xffffffff, 0xe0000000,
1316         0x28350, 0xffffffff, 0x3a00161a,
1317         0x28354, 0xffffffff, 0x0000002e,
1318         0x9a10, 0xffffffff, 0x00018208,
1319         0x98f8, 0xffffffff, 0x12011003
1320 };
1321
1322 static const u32 hawaii_golden_registers[] =
1323 {
1324         0x3354, 0x00000333, 0x00000333,
1325         0x9a10, 0x00010000, 0x00058208,
1326         0x9830, 0xffffffff, 0x00000000,
1327         0x9834, 0xf00fffff, 0x00000400,
1328         0x9838, 0x0002021c, 0x00020200,
1329         0xc78, 0x00000080, 0x00000000,
1330         0x5bb0, 0x000000f0, 0x00000070,
1331         0x5bc0, 0xf0311fff, 0x80300000,
1332         0x350c, 0x00810000, 0x408af000,
1333         0x7030, 0x31000111, 0x00000011,
1334         0x2f48, 0x73773777, 0x12010001,
1335         0x2120, 0x0000007f, 0x0000001b,
1336         0x21dc, 0x00007fb6, 0x00002191,
1337         0x3628, 0x0000003f, 0x0000000a,
1338         0x362c, 0x0000003f, 0x0000000a,
1339         0x2ae4, 0x00073ffe, 0x000022a2,
1340         0x240c, 0x000007ff, 0x00000000,
1341         0x8bf0, 0x00002001, 0x00000001,
1342         0x8b24, 0xffffffff, 0x00ffffff,
1343         0x30a04, 0x0000ff0f, 0x00000000,
1344         0x28a4c, 0x07ffffff, 0x06000000,
1345         0x3e78, 0x00000001, 0x00000002,
1346         0xc768, 0x00000008, 0x00000008,
1347         0xc770, 0x00000f00, 0x00000800,
1348         0xc774, 0x00000f00, 0x00000800,
1349         0xc798, 0x00ffffff, 0x00ff7fbf,
1350         0xc79c, 0x00ffffff, 0x00ff7faf,
1351         0x8c00, 0x000000ff, 0x00000800,
1352         0xe40, 0x00001fff, 0x00001fff,
1353         0x9060, 0x0000007f, 0x00000020,
1354         0x9508, 0x00010000, 0x00010000,
1355         0xae00, 0x00100000, 0x000ff07c,
1356         0xac14, 0x000003ff, 0x0000000f,
1357         0xac10, 0xffffffff, 0x7564fdec,
1358         0xac0c, 0xffffffff, 0x3120b9a8,
1359         0xac08, 0x20000000, 0x0f9c0000
1360 };
1361
1362 static const u32 hawaii_mgcg_cgcg_init[] =
1363 {
1364         0xc420, 0xffffffff, 0xfffffffd,
1365         0x30800, 0xffffffff, 0xe0000000,
1366         0x3c2a0, 0xffffffff, 0x00000100,
1367         0x3c208, 0xffffffff, 0x00000100,
1368         0x3c2c0, 0xffffffff, 0x00000100,
1369         0x3c2c8, 0xffffffff, 0x00000100,
1370         0x3c2c4, 0xffffffff, 0x00000100,
1371         0x55e4, 0xffffffff, 0x00200100,
1372         0x3c280, 0xffffffff, 0x00000100,
1373         0x3c214, 0xffffffff, 0x06000100,
1374         0x3c220, 0xffffffff, 0x00000100,
1375         0x3c218, 0xffffffff, 0x06000100,
1376         0x3c204, 0xffffffff, 0x00000100,
1377         0x3c2e0, 0xffffffff, 0x00000100,
1378         0x3c224, 0xffffffff, 0x00000100,
1379         0x3c200, 0xffffffff, 0x00000100,
1380         0x3c230, 0xffffffff, 0x00000100,
1381         0x3c234, 0xffffffff, 0x00000100,
1382         0x3c250, 0xffffffff, 0x00000100,
1383         0x3c254, 0xffffffff, 0x00000100,
1384         0x3c258, 0xffffffff, 0x00000100,
1385         0x3c25c, 0xffffffff, 0x00000100,
1386         0x3c260, 0xffffffff, 0x00000100,
1387         0x3c27c, 0xffffffff, 0x00000100,
1388         0x3c278, 0xffffffff, 0x00000100,
1389         0x3c210, 0xffffffff, 0x06000100,
1390         0x3c290, 0xffffffff, 0x00000100,
1391         0x3c274, 0xffffffff, 0x00000100,
1392         0x3c2b4, 0xffffffff, 0x00000100,
1393         0x3c2b0, 0xffffffff, 0x00000100,
1394         0x3c270, 0xffffffff, 0x00000100,
1395         0x30800, 0xffffffff, 0xe0000000,
1396         0x3c020, 0xffffffff, 0x00010000,
1397         0x3c024, 0xffffffff, 0x00030002,
1398         0x3c028, 0xffffffff, 0x00040007,
1399         0x3c02c, 0xffffffff, 0x00060005,
1400         0x3c030, 0xffffffff, 0x00090008,
1401         0x3c034, 0xffffffff, 0x00010000,
1402         0x3c038, 0xffffffff, 0x00030002,
1403         0x3c03c, 0xffffffff, 0x00040007,
1404         0x3c040, 0xffffffff, 0x00060005,
1405         0x3c044, 0xffffffff, 0x00090008,
1406         0x3c048, 0xffffffff, 0x00010000,
1407         0x3c04c, 0xffffffff, 0x00030002,
1408         0x3c050, 0xffffffff, 0x00040007,
1409         0x3c054, 0xffffffff, 0x00060005,
1410         0x3c058, 0xffffffff, 0x00090008,
1411         0x3c05c, 0xffffffff, 0x00010000,
1412         0x3c060, 0xffffffff, 0x00030002,
1413         0x3c064, 0xffffffff, 0x00040007,
1414         0x3c068, 0xffffffff, 0x00060005,
1415         0x3c06c, 0xffffffff, 0x00090008,
1416         0x3c070, 0xffffffff, 0x00010000,
1417         0x3c074, 0xffffffff, 0x00030002,
1418         0x3c078, 0xffffffff, 0x00040007,
1419         0x3c07c, 0xffffffff, 0x00060005,
1420         0x3c080, 0xffffffff, 0x00090008,
1421         0x3c084, 0xffffffff, 0x00010000,
1422         0x3c088, 0xffffffff, 0x00030002,
1423         0x3c08c, 0xffffffff, 0x00040007,
1424         0x3c090, 0xffffffff, 0x00060005,
1425         0x3c094, 0xffffffff, 0x00090008,
1426         0x3c098, 0xffffffff, 0x00010000,
1427         0x3c09c, 0xffffffff, 0x00030002,
1428         0x3c0a0, 0xffffffff, 0x00040007,
1429         0x3c0a4, 0xffffffff, 0x00060005,
1430         0x3c0a8, 0xffffffff, 0x00090008,
1431         0x3c0ac, 0xffffffff, 0x00010000,
1432         0x3c0b0, 0xffffffff, 0x00030002,
1433         0x3c0b4, 0xffffffff, 0x00040007,
1434         0x3c0b8, 0xffffffff, 0x00060005,
1435         0x3c0bc, 0xffffffff, 0x00090008,
1436         0x3c0c0, 0xffffffff, 0x00010000,
1437         0x3c0c4, 0xffffffff, 0x00030002,
1438         0x3c0c8, 0xffffffff, 0x00040007,
1439         0x3c0cc, 0xffffffff, 0x00060005,
1440         0x3c0d0, 0xffffffff, 0x00090008,
1441         0x3c0d4, 0xffffffff, 0x00010000,
1442         0x3c0d8, 0xffffffff, 0x00030002,
1443         0x3c0dc, 0xffffffff, 0x00040007,
1444         0x3c0e0, 0xffffffff, 0x00060005,
1445         0x3c0e4, 0xffffffff, 0x00090008,
1446         0x3c0e8, 0xffffffff, 0x00010000,
1447         0x3c0ec, 0xffffffff, 0x00030002,
1448         0x3c0f0, 0xffffffff, 0x00040007,
1449         0x3c0f4, 0xffffffff, 0x00060005,
1450         0x3c0f8, 0xffffffff, 0x00090008,
1451         0xc318, 0xffffffff, 0x00020200,
1452         0x3350, 0xffffffff, 0x00000200,
1453         0x15c0, 0xffffffff, 0x00000400,
1454         0x55e8, 0xffffffff, 0x00000000,
1455         0x2f50, 0xffffffff, 0x00000902,
1456         0x3c000, 0xffffffff, 0x96940200,
1457         0x8708, 0xffffffff, 0x00900100,
1458         0xc424, 0xffffffff, 0x0020003f,
1459         0x38, 0xffffffff, 0x0140001c,
1460         0x3c, 0x000f0000, 0x000f0000,
1461         0x220, 0xffffffff, 0xc060000c,
1462         0x224, 0xc0000fff, 0x00000100,
1463         0xf90, 0xffffffff, 0x00000100,
1464         0xf98, 0x00000101, 0x00000000,
1465         0x20a8, 0xffffffff, 0x00000104,
1466         0x55e4, 0xff000fff, 0x00000100,
1467         0x30cc, 0xc0000fff, 0x00000104,
1468         0xc1e4, 0x00000001, 0x00000001,
1469         0xd00c, 0xff000ff0, 0x00000100,
1470         0xd80c, 0xff000ff0, 0x00000100
1471 };
1472
1473 static void cik_init_golden_registers(struct radeon_device *rdev)
1474 {
1475         switch (rdev->family) {
1476         case CHIP_BONAIRE:
1477                 radeon_program_register_sequence(rdev,
1478                                                  bonaire_mgcg_cgcg_init,
1479                                                  (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1480                 radeon_program_register_sequence(rdev,
1481                                                  bonaire_golden_registers,
1482                                                  (const u32)ARRAY_SIZE(bonaire_golden_registers));
1483                 radeon_program_register_sequence(rdev,
1484                                                  bonaire_golden_common_registers,
1485                                                  (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1486                 radeon_program_register_sequence(rdev,
1487                                                  bonaire_golden_spm_registers,
1488                                                  (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1489                 break;
1490         case CHIP_KABINI:
1491                 radeon_program_register_sequence(rdev,
1492                                                  kalindi_mgcg_cgcg_init,
1493                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1494                 radeon_program_register_sequence(rdev,
1495                                                  kalindi_golden_registers,
1496                                                  (const u32)ARRAY_SIZE(kalindi_golden_registers));
1497                 radeon_program_register_sequence(rdev,
1498                                                  kalindi_golden_common_registers,
1499                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1500                 radeon_program_register_sequence(rdev,
1501                                                  kalindi_golden_spm_registers,
1502                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1503                 break;
1504         case CHIP_KAVERI:
1505                 radeon_program_register_sequence(rdev,
1506                                                  spectre_mgcg_cgcg_init,
1507                                                  (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1508                 radeon_program_register_sequence(rdev,
1509                                                  spectre_golden_registers,
1510                                                  (const u32)ARRAY_SIZE(spectre_golden_registers));
1511                 radeon_program_register_sequence(rdev,
1512                                                  spectre_golden_common_registers,
1513                                                  (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1514                 radeon_program_register_sequence(rdev,
1515                                                  spectre_golden_spm_registers,
1516                                                  (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1517                 break;
1518         case CHIP_HAWAII:
1519                 radeon_program_register_sequence(rdev,
1520                                                  hawaii_mgcg_cgcg_init,
1521                                                  (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1522                 radeon_program_register_sequence(rdev,
1523                                                  hawaii_golden_registers,
1524                                                  (const u32)ARRAY_SIZE(hawaii_golden_registers));
1525                 radeon_program_register_sequence(rdev,
1526                                                  hawaii_golden_common_registers,
1527                                                  (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1528                 radeon_program_register_sequence(rdev,
1529                                                  hawaii_golden_spm_registers,
1530                                                  (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1531                 break;
1532         default:
1533                 break;
1534         }
1535 }
1536
1537 /**
1538  * cik_get_xclk - get the xclk
1539  *
1540  * @rdev: radeon_device pointer
1541  *
1542  * Returns the reference clock used by the gfx engine
1543  * (CIK).
1544  */
1545 u32 cik_get_xclk(struct radeon_device *rdev)
1546 {
1547         u32 reference_clock = rdev->clock.spll.reference_freq;
1548
1549         if (rdev->flags & RADEON_IS_IGP) {
1550                 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1551                         return reference_clock / 2;
1552         } else {
1553                 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1554                         return reference_clock / 4;
1555         }
1556         return reference_clock;
1557 }
1558
1559 /**
1560  * cik_mm_rdoorbell - read a doorbell dword
1561  *
1562  * @rdev: radeon_device pointer
1563  * @index: doorbell index
1564  *
1565  * Returns the value in the doorbell aperture at the
1566  * requested doorbell index (CIK).
1567  */
1568 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1569 {
1570         if (index < rdev->doorbell.num_doorbells) {
1571                 return readl(rdev->doorbell.ptr + index);
1572         } else {
1573                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1574                 return 0;
1575         }
1576 }
1577
1578 /**
1579  * cik_mm_wdoorbell - write a doorbell dword
1580  *
1581  * @rdev: radeon_device pointer
1582  * @index: doorbell index
1583  * @v: value to write
1584  *
1585  * Writes @v to the doorbell aperture at the
1586  * requested doorbell index (CIK).
1587  */
1588 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1589 {
1590         if (index < rdev->doorbell.num_doorbells) {
1591                 writel(v, rdev->doorbell.ptr + index);
1592         } else {
1593                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1594         }
1595 }
1596
1597 #define BONAIRE_IO_MC_REGS_SIZE 36
1598
1599 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1600 {
1601         {0x00000070, 0x04400000},
1602         {0x00000071, 0x80c01803},
1603         {0x00000072, 0x00004004},
1604         {0x00000073, 0x00000100},
1605         {0x00000074, 0x00ff0000},
1606         {0x00000075, 0x34000000},
1607         {0x00000076, 0x08000014},
1608         {0x00000077, 0x00cc08ec},
1609         {0x00000078, 0x00000400},
1610         {0x00000079, 0x00000000},
1611         {0x0000007a, 0x04090000},
1612         {0x0000007c, 0x00000000},
1613         {0x0000007e, 0x4408a8e8},
1614         {0x0000007f, 0x00000304},
1615         {0x00000080, 0x00000000},
1616         {0x00000082, 0x00000001},
1617         {0x00000083, 0x00000002},
1618         {0x00000084, 0xf3e4f400},
1619         {0x00000085, 0x052024e3},
1620         {0x00000087, 0x00000000},
1621         {0x00000088, 0x01000000},
1622         {0x0000008a, 0x1c0a0000},
1623         {0x0000008b, 0xff010000},
1624         {0x0000008d, 0xffffefff},
1625         {0x0000008e, 0xfff3efff},
1626         {0x0000008f, 0xfff3efbf},
1627         {0x00000092, 0xf7ffffff},
1628         {0x00000093, 0xffffff7f},
1629         {0x00000095, 0x00101101},
1630         {0x00000096, 0x00000fff},
1631         {0x00000097, 0x00116fff},
1632         {0x00000098, 0x60010000},
1633         {0x00000099, 0x10010000},
1634         {0x0000009a, 0x00006000},
1635         {0x0000009b, 0x00001000},
1636         {0x0000009f, 0x00b48000}
1637 };
1638
1639 #define HAWAII_IO_MC_REGS_SIZE 22
1640
1641 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1642 {
1643         {0x0000007d, 0x40000000},
1644         {0x0000007e, 0x40180304},
1645         {0x0000007f, 0x0000ff00},
1646         {0x00000081, 0x00000000},
1647         {0x00000083, 0x00000800},
1648         {0x00000086, 0x00000000},
1649         {0x00000087, 0x00000100},
1650         {0x00000088, 0x00020100},
1651         {0x00000089, 0x00000000},
1652         {0x0000008b, 0x00040000},
1653         {0x0000008c, 0x00000100},
1654         {0x0000008e, 0xff010000},
1655         {0x00000090, 0xffffefff},
1656         {0x00000091, 0xfff3efff},
1657         {0x00000092, 0xfff3efbf},
1658         {0x00000093, 0xf7ffffff},
1659         {0x00000094, 0xffffff7f},
1660         {0x00000095, 0x00000fff},
1661         {0x00000096, 0x00116fff},
1662         {0x00000097, 0x60010000},
1663         {0x00000098, 0x10010000},
1664         {0x0000009f, 0x00c79000}
1665 };
1666
1667
1668 /**
1669  * cik_srbm_select - select specific register instances
1670  *
1671  * @rdev: radeon_device pointer
1672  * @me: selected ME (micro engine)
1673  * @pipe: pipe
1674  * @queue: queue
1675  * @vmid: VMID
1676  *
1677  * Switches the currently active registers instances.  Some
1678  * registers are instanced per VMID, others are instanced per
1679  * me/pipe/queue combination.
1680  */
1681 static void cik_srbm_select(struct radeon_device *rdev,
1682                             u32 me, u32 pipe, u32 queue, u32 vmid)
1683 {
1684         u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1685                              MEID(me & 0x3) |
1686                              VMID(vmid & 0xf) |
1687                              QUEUEID(queue & 0x7));
1688         WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1689 }
1690
1691 /* ucode loading */
1692 /**
1693  * ci_mc_load_microcode - load MC ucode into the hw
1694  *
1695  * @rdev: radeon_device pointer
1696  *
1697  * Load the GDDR MC ucode into the hw (CIK).
1698  * Returns 0 on success, error on failure.
1699  */
1700 static int ci_mc_load_microcode(struct radeon_device *rdev)
1701 {
1702         const __be32 *fw_data;
1703         u32 running, blackout = 0;
1704         u32 *io_mc_regs;
1705         int i, ucode_size, regs_size;
1706
1707         if (!rdev->mc_fw)
1708                 return -EINVAL;
1709
1710         switch (rdev->family) {
1711         case CHIP_BONAIRE:
1712                 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1713                 ucode_size = CIK_MC_UCODE_SIZE;
1714                 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1715                 break;
1716         case CHIP_HAWAII:
1717                 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1718                 ucode_size = HAWAII_MC_UCODE_SIZE;
1719                 regs_size = HAWAII_IO_MC_REGS_SIZE;
1720                 break;
1721         default:
1722                 return -EINVAL;
1723         }
1724
1725         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1726
1727         if (running == 0) {
1728                 if (running) {
1729                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1730                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1731                 }
1732
1733                 /* reset the engine and set to writable */
1734                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1735                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1736
1737                 /* load mc io regs */
1738                 for (i = 0; i < regs_size; i++) {
1739                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1740                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1741                 }
1742                 /* load the MC ucode */
1743                 fw_data = (const __be32 *)rdev->mc_fw->data;
1744                 for (i = 0; i < ucode_size; i++)
1745                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1746
1747                 /* put the engine back into the active state */
1748                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1749                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1750                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1751
1752                 /* wait for training to complete */
1753                 for (i = 0; i < rdev->usec_timeout; i++) {
1754                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1755                                 break;
1756                         udelay(1);
1757                 }
1758                 for (i = 0; i < rdev->usec_timeout; i++) {
1759                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1760                                 break;
1761                         udelay(1);
1762                 }
1763
1764                 if (running)
1765                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1766         }
1767
1768         return 0;
1769 }
1770
1771 /**
1772  * cik_init_microcode - load ucode images from disk
1773  *
1774  * @rdev: radeon_device pointer
1775  *
1776  * Use the firmware interface to load the ucode images into
1777  * the driver (not loaded into hw).
1778  * Returns 0 on success, error on failure.
1779  */
1780 static int cik_init_microcode(struct radeon_device *rdev)
1781 {
1782         const char *chip_name;
1783         size_t pfp_req_size, me_req_size, ce_req_size,
1784                 mec_req_size, rlc_req_size, mc_req_size = 0,
1785                 sdma_req_size, smc_req_size = 0;
1786         char fw_name[30];
1787         int err;
1788
1789         DRM_DEBUG("\n");
1790
1791         switch (rdev->family) {
1792         case CHIP_BONAIRE:
1793                 chip_name = "BONAIRE";
1794                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1795                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1796                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1797                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1798                 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1799                 mc_req_size = CIK_MC_UCODE_SIZE * 4;
1800                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1801                 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1802                 break;
1803         case CHIP_HAWAII:
1804                 chip_name = "HAWAII";
1805                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1806                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1807                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1808                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1809                 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1810                 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1811                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1812                 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1813                 break;
1814         case CHIP_KAVERI:
1815                 chip_name = "KAVERI";
1816                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1817                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1818                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1819                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1820                 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1821                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1822                 break;
1823         case CHIP_KABINI:
1824                 chip_name = "KABINI";
1825                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1826                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1827                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1828                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1829                 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1830                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1831                 break;
1832         default: BUG();
1833         }
1834
1835         DRM_INFO("Loading %s Microcode\n", chip_name);
1836
1837         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1838         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1839         if (err)
1840                 goto out;
1841         if (rdev->pfp_fw->size != pfp_req_size) {
1842                 printk(KERN_ERR
1843                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1844                        rdev->pfp_fw->size, fw_name);
1845                 err = -EINVAL;
1846                 goto out;
1847         }
1848
1849         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1850         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1851         if (err)
1852                 goto out;
1853         if (rdev->me_fw->size != me_req_size) {
1854                 printk(KERN_ERR
1855                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1856                        rdev->me_fw->size, fw_name);
1857                 err = -EINVAL;
1858         }
1859
1860         snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1861         err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1862         if (err)
1863                 goto out;
1864         if (rdev->ce_fw->size != ce_req_size) {
1865                 printk(KERN_ERR
1866                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1867                        rdev->ce_fw->size, fw_name);
1868                 err = -EINVAL;
1869         }
1870
1871         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
1872         err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
1873         if (err)
1874                 goto out;
1875         if (rdev->mec_fw->size != mec_req_size) {
1876                 printk(KERN_ERR
1877                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1878                        rdev->mec_fw->size, fw_name);
1879                 err = -EINVAL;
1880         }
1881
1882         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1883         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1884         if (err)
1885                 goto out;
1886         if (rdev->rlc_fw->size != rlc_req_size) {
1887                 printk(KERN_ERR
1888                        "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1889                        rdev->rlc_fw->size, fw_name);
1890                 err = -EINVAL;
1891         }
1892
1893         snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
1894         err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
1895         if (err)
1896                 goto out;
1897         if (rdev->sdma_fw->size != sdma_req_size) {
1898                 printk(KERN_ERR
1899                        "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1900                        rdev->sdma_fw->size, fw_name);
1901                 err = -EINVAL;
1902         }
1903
1904         /* No SMC, MC ucode on APUs */
1905         if (!(rdev->flags & RADEON_IS_IGP)) {
1906                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1907                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1908                 if (err)
1909                         goto out;
1910                 if (rdev->mc_fw->size != mc_req_size) {
1911                         printk(KERN_ERR
1912                                "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1913                                rdev->mc_fw->size, fw_name);
1914                         err = -EINVAL;
1915                 }
1916
1917                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1918                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1919                 if (err) {
1920                         printk(KERN_ERR
1921                                "smc: error loading firmware \"%s\"\n",
1922                                fw_name);
1923                         release_firmware(rdev->smc_fw);
1924                         rdev->smc_fw = NULL;
1925                         err = 0;
1926                 } else if (rdev->smc_fw->size != smc_req_size) {
1927                         printk(KERN_ERR
1928                                "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1929                                rdev->smc_fw->size, fw_name);
1930                         err = -EINVAL;
1931                 }
1932         }
1933
1934 out:
1935         if (err) {
1936                 if (err != -EINVAL)
1937                         printk(KERN_ERR
1938                                "cik_cp: Failed to load firmware \"%s\"\n",
1939                                fw_name);
1940                 release_firmware(rdev->pfp_fw);
1941                 rdev->pfp_fw = NULL;
1942                 release_firmware(rdev->me_fw);
1943                 rdev->me_fw = NULL;
1944                 release_firmware(rdev->ce_fw);
1945                 rdev->ce_fw = NULL;
1946                 release_firmware(rdev->rlc_fw);
1947                 rdev->rlc_fw = NULL;
1948                 release_firmware(rdev->mc_fw);
1949                 rdev->mc_fw = NULL;
1950                 release_firmware(rdev->smc_fw);
1951                 rdev->smc_fw = NULL;
1952         }
1953         return err;
1954 }
1955
1956 /*
1957  * Core functions
1958  */
1959 /**
1960  * cik_tiling_mode_table_init - init the hw tiling table
1961  *
1962  * @rdev: radeon_device pointer
1963  *
1964  * Starting with SI, the tiling setup is done globally in a
1965  * set of 32 tiling modes.  Rather than selecting each set of
1966  * parameters per surface as on older asics, we just select
1967  * which index in the tiling table we want to use, and the
1968  * surface uses those parameters (CIK).
1969  */
1970 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1971 {
1972         const u32 num_tile_mode_states = 32;
1973         const u32 num_secondary_tile_mode_states = 16;
1974         u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1975         u32 num_pipe_configs;
1976         u32 num_rbs = rdev->config.cik.max_backends_per_se *
1977                 rdev->config.cik.max_shader_engines;
1978
1979         switch (rdev->config.cik.mem_row_size_in_kb) {
1980         case 1:
1981                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1982                 break;
1983         case 2:
1984         default:
1985                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1986                 break;
1987         case 4:
1988                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1989                 break;
1990         }
1991
1992         num_pipe_configs = rdev->config.cik.max_tile_pipes;
1993         if (num_pipe_configs > 8)
1994                 num_pipe_configs = 16;
1995
1996         if (num_pipe_configs == 16) {
1997                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1998                         switch (reg_offset) {
1999                         case 0:
2000                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2001                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2002                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2003                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2004                                 break;
2005                         case 1:
2006                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2007                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2008                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2009                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2010                                 break;
2011                         case 2:
2012                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2013                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2014                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2015                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2016                                 break;
2017                         case 3:
2018                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2019                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2020                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2021                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2022                                 break;
2023                         case 4:
2024                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2025                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2026                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2027                                                  TILE_SPLIT(split_equal_to_row_size));
2028                                 break;
2029                         case 5:
2030                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2031                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2032                                 break;
2033                         case 6:
2034                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2035                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2036                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2037                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2038                                 break;
2039                         case 7:
2040                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2041                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2042                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2043                                                  TILE_SPLIT(split_equal_to_row_size));
2044                                 break;
2045                         case 8:
2046                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2047                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2048                                 break;
2049                         case 9:
2050                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2051                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2052                                 break;
2053                         case 10:
2054                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2055                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2056                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2057                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2058                                 break;
2059                         case 11:
2060                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2061                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2062                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2063                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2064                                 break;
2065                         case 12:
2066                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2067                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2068                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2069                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2070                                 break;
2071                         case 13:
2072                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2073                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2074                                 break;
2075                         case 14:
2076                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2077                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2078                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2079                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2080                                 break;
2081                         case 16:
2082                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2083                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2084                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2085                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2086                                 break;
2087                         case 17:
2088                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2089                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2090                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2091                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2092                                 break;
2093                         case 27:
2094                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2095                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2096                                 break;
2097                         case 28:
2098                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2099                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2100                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2101                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2102                                 break;
2103                         case 29:
2104                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2105                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2106                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2107                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2108                                 break;
2109                         case 30:
2110                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2111                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2112                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2113                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2114                                 break;
2115                         default:
2116                                 gb_tile_moden = 0;
2117                                 break;
2118                         }
2119                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2120                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2121                 }
2122                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2123                         switch (reg_offset) {
2124                         case 0:
2125                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2126                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2127                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2128                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2129                                 break;
2130                         case 1:
2131                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2132                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2133                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2134                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2135                                 break;
2136                         case 2:
2137                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2138                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2139                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2140                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2141                                 break;
2142                         case 3:
2143                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2144                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2145                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2146                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2147                                 break;
2148                         case 4:
2149                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2150                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2151                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2152                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2153                                 break;
2154                         case 5:
2155                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2156                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2157                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2158                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2159                                 break;
2160                         case 6:
2161                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2162                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2163                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2164                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2165                                 break;
2166                         case 8:
2167                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2168                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2169                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2170                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2171                                 break;
2172                         case 9:
2173                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2174                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2175                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2176                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2177                                 break;
2178                         case 10:
2179                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2180                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2181                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2182                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2183                                 break;
2184                         case 11:
2185                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2186                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2187                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2188                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2189                                 break;
2190                         case 12:
2191                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2192                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2193                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2194                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2195                                 break;
2196                         case 13:
2197                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2198                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2199                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2200                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2201                                 break;
2202                         case 14:
2203                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2204                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2205                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2206                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2207                                 break;
2208                         default:
2209                                 gb_tile_moden = 0;
2210                                 break;
2211                         }
2212                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2213                 }
2214         } else if (num_pipe_configs == 8) {
2215                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2216                         switch (reg_offset) {
2217                         case 0:
2218                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2219                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2220                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2221                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2222                                 break;
2223                         case 1:
2224                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2225                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2226                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2227                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2228                                 break;
2229                         case 2:
2230                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2231                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2232                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2233                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2234                                 break;
2235                         case 3:
2236                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2237                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2238                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2239                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2240                                 break;
2241                         case 4:
2242                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2243                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2244                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2245                                                  TILE_SPLIT(split_equal_to_row_size));
2246                                 break;
2247                         case 5:
2248                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2249                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2250                                 break;
2251                         case 6:
2252                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2253                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2254                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2255                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2256                                 break;
2257                         case 7:
2258                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2259                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2260                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2261                                                  TILE_SPLIT(split_equal_to_row_size));
2262                                 break;
2263                         case 8:
2264                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2265                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2266                                 break;
2267                         case 9:
2268                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2269                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2270                                 break;
2271                         case 10:
2272                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2273                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2274                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2275                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2276                                 break;
2277                         case 11:
2278                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2279                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2280                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2281                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2282                                 break;
2283                         case 12:
2284                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2285                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2286                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2287                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2288                                 break;
2289                         case 13:
2290                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2291                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2292                                 break;
2293                         case 14:
2294                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2295                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2296                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2297                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2298                                 break;
2299                         case 16:
2300                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2301                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2302                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2303                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2304                                 break;
2305                         case 17:
2306                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2307                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2308                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2309                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2310                                 break;
2311                         case 27:
2312                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2313                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2314                                 break;
2315                         case 28:
2316                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2317                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2318                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2319                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2320                                 break;
2321                         case 29:
2322                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2323                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2324                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2325                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2326                                 break;
2327                         case 30:
2328                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2329                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2330                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2331                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2332                                 break;
2333                         default:
2334                                 gb_tile_moden = 0;
2335                                 break;
2336                         }
2337                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2338                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2339                 }
2340                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2341                         switch (reg_offset) {
2342                         case 0:
2343                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2344                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2345                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2346                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2347                                 break;
2348                         case 1:
2349                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2350                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2351                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2352                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2353                                 break;
2354                         case 2:
2355                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2356                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2357                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2358                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2359                                 break;
2360                         case 3:
2361                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2362                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2363                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2364                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2365                                 break;
2366                         case 4:
2367                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2368                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2369                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2370                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2371                                 break;
2372                         case 5:
2373                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2374                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2375                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2376                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2377                                 break;
2378                         case 6:
2379                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2380                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2381                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2382                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2383                                 break;
2384                         case 8:
2385                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2386                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2387                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2388                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2389                                 break;
2390                         case 9:
2391                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2392                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2393                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2394                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2395                                 break;
2396                         case 10:
2397                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2398                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2399                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2400                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2401                                 break;
2402                         case 11:
2403                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2404                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2405                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2406                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2407                                 break;
2408                         case 12:
2409                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2410                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2411                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2412                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2413                                 break;
2414                         case 13:
2415                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2416                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2417                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2418                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2419                                 break;
2420                         case 14:
2421                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2422                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2423                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2424                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2425                                 break;
2426                         default:
2427                                 gb_tile_moden = 0;
2428                                 break;
2429                         }
2430                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2431                 }
2432         } else if (num_pipe_configs == 4) {
2433                 if (num_rbs == 4) {
2434                         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2435                                 switch (reg_offset) {
2436                                 case 0:
2437                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2438                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2439                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2440                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2441                                         break;
2442                                 case 1:
2443                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2444                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2445                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2446                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2447                                         break;
2448                                 case 2:
2449                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2450                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2451                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2452                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2453                                         break;
2454                                 case 3:
2455                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2456                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2457                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2458                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2459                                         break;
2460                                 case 4:
2461                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2462                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2463                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2464                                                          TILE_SPLIT(split_equal_to_row_size));
2465                                         break;
2466                                 case 5:
2467                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2468                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2469                                         break;
2470                                 case 6:
2471                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2472                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2473                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2474                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2475                                         break;
2476                                 case 7:
2477                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2478                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2479                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2480                                                          TILE_SPLIT(split_equal_to_row_size));
2481                                         break;
2482                                 case 8:
2483                                         gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2484                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16));
2485                                         break;
2486                                 case 9:
2487                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2488                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2489                                         break;
2490                                 case 10:
2491                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2492                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2493                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2494                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2495                                         break;
2496                                 case 11:
2497                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2498                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2499                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2500                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2501                                         break;
2502                                 case 12:
2503                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2504                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2505                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2506                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2507                                         break;
2508                                 case 13:
2509                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2510                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2511                                         break;
2512                                 case 14:
2513                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2514                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2515                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2516                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2517                                         break;
2518                                 case 16:
2519                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2520                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2521                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2522                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2523                                         break;
2524                                 case 17:
2525                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2526                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2527                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2528                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2529                                         break;
2530                                 case 27:
2531                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2532                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2533                                         break;
2534                                 case 28:
2535                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2536                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2537                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2538                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2539                                         break;
2540                                 case 29:
2541                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2542                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2543                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2544                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2545                                         break;
2546                                 case 30:
2547                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2548                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2549                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2550                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2551                                         break;
2552                                 default:
2553                                         gb_tile_moden = 0;
2554                                         break;
2555                                 }
2556                                 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2557                                 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2558                         }
2559                 } else if (num_rbs < 4) {
2560                         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2561                                 switch (reg_offset) {
2562                                 case 0:
2563                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2564                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2565                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2566                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2567                                         break;
2568                                 case 1:
2569                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2570                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2571                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2572                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2573                                         break;
2574                                 case 2:
2575                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2576                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2577                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2578                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2579                                         break;
2580                                 case 3:
2581                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2582                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2583                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2584                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2585                                         break;
2586                                 case 4:
2587                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2588                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2589                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2590                                                          TILE_SPLIT(split_equal_to_row_size));
2591                                         break;
2592                                 case 5:
2593                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2594                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2595                                         break;
2596                                 case 6:
2597                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2598                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2599                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2600                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2601                                         break;
2602                                 case 7:
2603                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2604                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2605                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2606                                                          TILE_SPLIT(split_equal_to_row_size));
2607                                         break;
2608                                 case 8:
2609                                         gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2610                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16));
2611                                         break;
2612                                 case 9:
2613                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2614                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2615                                         break;
2616                                 case 10:
2617                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2618                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2619                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2620                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2621                                         break;
2622                                 case 11:
2623                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2624                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2625                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2626                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2627                                         break;
2628                                 case 12:
2629                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2630                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2631                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2632                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2633                                         break;
2634                                 case 13:
2635                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2636                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2637                                         break;
2638                                 case 14:
2639                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2640                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2641                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2642                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2643                                         break;
2644                                 case 16:
2645                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2646                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2647                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2648                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2649                                         break;
2650                                 case 17:
2651                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2652                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2653                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2654                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2655                                         break;
2656                                 case 27:
2657                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2658                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2659                                         break;
2660                                 case 28:
2661                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2662                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2663                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2664                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2665                                         break;
2666                                 case 29:
2667                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2668                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2669                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2670                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2671                                         break;
2672                                 case 30:
2673                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2674                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2675                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2676                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2677                                         break;
2678                                 default:
2679                                         gb_tile_moden = 0;
2680                                         break;
2681                                 }
2682                                 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2683                                 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2684                         }
2685                 }
2686                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2687                         switch (reg_offset) {
2688                         case 0:
2689                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2690                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2691                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2692                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2693                                 break;
2694                         case 1:
2695                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2696                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2697                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2698                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2699                                 break;
2700                         case 2:
2701                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2702                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2703                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2704                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2705                                 break;
2706                         case 3:
2707                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2708                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2709                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2710                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2711                                 break;
2712                         case 4:
2713                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2714                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2715                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2716                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2717                                 break;
2718                         case 5:
2719                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2720                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2721                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2722                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2723                                 break;
2724                         case 6:
2725                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2726                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2727                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2728                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2729                                 break;
2730                         case 8:
2731                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2732                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2733                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2734                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2735                                 break;
2736                         case 9:
2737                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2738                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2739                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2740                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2741                                 break;
2742                         case 10:
2743                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2744                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2745                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2746                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2747                                 break;
2748                         case 11:
2749                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2750                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2751                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2752                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2753                                 break;
2754                         case 12:
2755                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2756                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2757                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2758                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2759                                 break;
2760                         case 13:
2761                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2762                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2763                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2764                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2765                                 break;
2766                         case 14:
2767                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2768                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2769                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2770                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2771                                 break;
2772                         default:
2773                                 gb_tile_moden = 0;
2774                                 break;
2775                         }
2776                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2777                 }
2778         } else if (num_pipe_configs == 2) {
2779                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2780                         switch (reg_offset) {
2781                         case 0:
2782                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2783                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2784                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2785                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2786                                 break;
2787                         case 1:
2788                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2789                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2790                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2791                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2792                                 break;
2793                         case 2:
2794                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2795                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2796                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2797                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2798                                 break;
2799                         case 3:
2800                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2801                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2802                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2803                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2804                                 break;
2805                         case 4:
2806                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2807                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2808                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2809                                                  TILE_SPLIT(split_equal_to_row_size));
2810                                 break;
2811                         case 5:
2812                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2813                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2814                                 break;
2815                         case 6:
2816                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2817                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2818                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2819                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2820                                 break;
2821                         case 7:
2822                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2823                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2824                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2825                                                  TILE_SPLIT(split_equal_to_row_size));
2826                                 break;
2827                         case 8:
2828                                 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
2829                                 break;
2830                         case 9:
2831                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2832                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2833                                 break;
2834                         case 10:
2835                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2836                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2837                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2838                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2839                                 break;
2840                         case 11:
2841                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2842                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2843                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2844                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2845                                 break;
2846                         case 12:
2847                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2848                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2849                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2850                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2851                                 break;
2852                         case 13:
2853                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2854                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2855                                 break;
2856                         case 14:
2857                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2858                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2859                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2860                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2861                                 break;
2862                         case 16:
2863                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2864                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2865                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2866                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2867                                 break;
2868                         case 17:
2869                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2870                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2871                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2872                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2873                                 break;
2874                         case 27:
2875                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2876                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2877                                 break;
2878                         case 28:
2879                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2880                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2881                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2882                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2883                                 break;
2884                         case 29:
2885                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2886                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2887                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2888                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2889                                 break;
2890                         case 30:
2891                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2892                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2893                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2894                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2895                                 break;
2896                         default:
2897                                 gb_tile_moden = 0;
2898                                 break;
2899                         }
2900                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2901                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2902                 }
2903                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2904                         switch (reg_offset) {
2905                         case 0:
2906                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2907                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2908                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2909                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2910                                 break;
2911                         case 1:
2912                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2913                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2914                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2915                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2916                                 break;
2917                         case 2:
2918                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2919                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2920                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2921                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2922                                 break;
2923                         case 3:
2924                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2925                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2926                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2927                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2928                                 break;
2929                         case 4:
2930                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2931                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2932                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2933                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2934                                 break;
2935                         case 5:
2936                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2937                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2938                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2939                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2940                                 break;
2941                         case 6:
2942                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2943                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2944                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2945                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2946                                 break;
2947                         case 8:
2948                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2949                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2950                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2951                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2952                                 break;
2953                         case 9:
2954                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2955                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2956                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2957                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2958                                 break;
2959                         case 10:
2960                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2961                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2962                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2963                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2964                                 break;
2965                         case 11:
2966                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2967                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2968                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2969                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2970                                 break;
2971                         case 12:
2972                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2973                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2974                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2975                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2976                                 break;
2977                         case 13:
2978                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2979                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2980                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2981                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2982                                 break;
2983                         case 14:
2984                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2985                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2986                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2987                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2988                                 break;
2989                         default:
2990                                 gb_tile_moden = 0;
2991                                 break;
2992                         }
2993                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2994                 }
2995         } else
2996                 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
2997 }
2998
2999 /**
3000  * cik_select_se_sh - select which SE, SH to address
3001  *
3002  * @rdev: radeon_device pointer
3003  * @se_num: shader engine to address
3004  * @sh_num: sh block to address
3005  *
3006  * Select which SE, SH combinations to address. Certain
3007  * registers are instanced per SE or SH.  0xffffffff means
3008  * broadcast to all SEs or SHs (CIK).
3009  */
3010 static void cik_select_se_sh(struct radeon_device *rdev,
3011                              u32 se_num, u32 sh_num)
3012 {
3013         u32 data = INSTANCE_BROADCAST_WRITES;
3014
3015         if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3016                 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3017         else if (se_num == 0xffffffff)
3018                 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3019         else if (sh_num == 0xffffffff)
3020                 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3021         else
3022                 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3023         WREG32(GRBM_GFX_INDEX, data);
3024 }
3025
3026 /**
3027  * cik_create_bitmask - create a bitmask
3028  *
3029  * @bit_width: length of the mask
3030  *
3031  * create a variable length bit mask (CIK).
3032  * Returns the bitmask.
3033  */
3034 static u32 cik_create_bitmask(u32 bit_width)
3035 {
3036         u32 i, mask = 0;
3037
3038         for (i = 0; i < bit_width; i++) {
3039                 mask <<= 1;
3040                 mask |= 1;
3041         }
3042         return mask;
3043 }
3044
3045 /**
3046  * cik_select_se_sh - select which SE, SH to address
3047  *
3048  * @rdev: radeon_device pointer
3049  * @max_rb_num: max RBs (render backends) for the asic
3050  * @se_num: number of SEs (shader engines) for the asic
3051  * @sh_per_se: number of SH blocks per SE for the asic
3052  *
3053  * Calculates the bitmask of disabled RBs (CIK).
3054  * Returns the disabled RB bitmask.
3055  */
3056 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3057                               u32 max_rb_num, u32 se_num,
3058                               u32 sh_per_se)
3059 {
3060         u32 data, mask;
3061
3062         data = RREG32(CC_RB_BACKEND_DISABLE);
3063         if (data & 1)
3064                 data &= BACKEND_DISABLE_MASK;
3065         else
3066                 data = 0;
3067         data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3068
3069         data >>= BACKEND_DISABLE_SHIFT;
3070
3071         mask = cik_create_bitmask(max_rb_num / se_num / sh_per_se);
3072
3073         return data & mask;
3074 }
3075
3076 /**
3077  * cik_setup_rb - setup the RBs on the asic
3078  *
3079  * @rdev: radeon_device pointer
3080  * @se_num: number of SEs (shader engines) for the asic
3081  * @sh_per_se: number of SH blocks per SE for the asic
3082  * @max_rb_num: max RBs (render backends) for the asic
3083  *
3084  * Configures per-SE/SH RB registers (CIK).
3085  */
3086 static void cik_setup_rb(struct radeon_device *rdev,
3087                          u32 se_num, u32 sh_per_se,
3088                          u32 max_rb_num)
3089 {
3090         int i, j;
3091         u32 data, mask;
3092         u32 disabled_rbs = 0;
3093         u32 enabled_rbs = 0;
3094
3095         for (i = 0; i < se_num; i++) {
3096                 for (j = 0; j < sh_per_se; j++) {
3097                         cik_select_se_sh(rdev, i, j);
3098                         data = cik_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
3099                         if (rdev->family == CHIP_HAWAII)
3100                                 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3101                         else
3102                                 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3103                 }
3104         }
3105         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3106
3107         mask = 1;
3108         for (i = 0; i < max_rb_num; i++) {
3109                 if (!(disabled_rbs & mask))
3110                         enabled_rbs |= mask;
3111                 mask <<= 1;
3112         }
3113
3114         for (i = 0; i < se_num; i++) {
3115                 cik_select_se_sh(rdev, i, 0xffffffff);
3116                 data = 0;
3117                 for (j = 0; j < sh_per_se; j++) {
3118                         switch (enabled_rbs & 3) {
3119                         case 0:
3120                                 if (j == 0)
3121                                         data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3122                                 else
3123                                         data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3124                                 break;
3125                         case 1:
3126                                 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3127                                 break;
3128                         case 2:
3129                                 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3130                                 break;
3131                         case 3:
3132                         default:
3133                                 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3134                                 break;
3135                         }
3136                         enabled_rbs >>= 2;
3137                 }
3138                 WREG32(PA_SC_RASTER_CONFIG, data);
3139         }
3140         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3141 }
3142
3143 /**
3144  * cik_gpu_init - setup the 3D engine
3145  *
3146  * @rdev: radeon_device pointer
3147  *
3148  * Configures the 3D engine and tiling configuration
3149  * registers so that the 3D engine is usable.
3150  */
3151 static void cik_gpu_init(struct radeon_device *rdev)
3152 {
3153         u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3154         u32 mc_shared_chmap, mc_arb_ramcfg;
3155         u32 hdp_host_path_cntl;
3156         u32 tmp;
3157         int i, j;
3158
3159         switch (rdev->family) {
3160         case CHIP_BONAIRE:
3161                 rdev->config.cik.max_shader_engines = 2;
3162                 rdev->config.cik.max_tile_pipes = 4;
3163                 rdev->config.cik.max_cu_per_sh = 7;
3164                 rdev->config.cik.max_sh_per_se = 1;
3165                 rdev->config.cik.max_backends_per_se = 2;
3166                 rdev->config.cik.max_texture_channel_caches = 4;
3167                 rdev->config.cik.max_gprs = 256;
3168                 rdev->config.cik.max_gs_threads = 32;
3169                 rdev->config.cik.max_hw_contexts = 8;
3170
3171                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3172                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3173                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3174                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3175                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3176                 break;
3177         case CHIP_HAWAII:
3178                 rdev->config.cik.max_shader_engines = 4;
3179                 rdev->config.cik.max_tile_pipes = 16;
3180                 rdev->config.cik.max_cu_per_sh = 11;
3181                 rdev->config.cik.max_sh_per_se = 1;
3182                 rdev->config.cik.max_backends_per_se = 4;
3183                 rdev->config.cik.max_texture_channel_caches = 16;
3184                 rdev->config.cik.max_gprs = 256;
3185                 rdev->config.cik.max_gs_threads = 32;
3186                 rdev->config.cik.max_hw_contexts = 8;
3187
3188                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3189                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3190                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3191                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3192                 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3193                 break;
3194         case CHIP_KAVERI:
3195                 rdev->config.cik.max_shader_engines = 1;
3196                 rdev->config.cik.max_tile_pipes = 4;
3197                 if ((rdev->pdev->device == 0x1304) ||
3198                     (rdev->pdev->device == 0x1305) ||
3199                     (rdev->pdev->device == 0x130C) ||
3200                     (rdev->pdev->device == 0x130F) ||
3201                     (rdev->pdev->device == 0x1310) ||
3202                     (rdev->pdev->device == 0x1311) ||
3203                     (rdev->pdev->device == 0x131C)) {
3204                         rdev->config.cik.max_cu_per_sh = 8;
3205                         rdev->config.cik.max_backends_per_se = 2;
3206                 } else if ((rdev->pdev->device == 0x1309) ||
3207                            (rdev->pdev->device == 0x130A) ||
3208                            (rdev->pdev->device == 0x130D) ||
3209                            (rdev->pdev->device == 0x1313) ||
3210                            (rdev->pdev->device == 0x131D)) {
3211                         rdev->config.cik.max_cu_per_sh = 6;
3212                         rdev->config.cik.max_backends_per_se = 2;
3213                 } else if ((rdev->pdev->device == 0x1306) ||
3214                            (rdev->pdev->device == 0x1307) ||
3215                            (rdev->pdev->device == 0x130B) ||
3216                            (rdev->pdev->device == 0x130E) ||
3217                            (rdev->pdev->device == 0x1315) ||
3218                            (rdev->pdev->device == 0x131B)) {
3219                         rdev->config.cik.max_cu_per_sh = 4;
3220                         rdev->config.cik.max_backends_per_se = 1;
3221                 } else {
3222                         rdev->config.cik.max_cu_per_sh = 3;
3223                         rdev->config.cik.max_backends_per_se = 1;
3224                 }
3225                 rdev->config.cik.max_sh_per_se = 1;
3226                 rdev->config.cik.max_texture_channel_caches = 4;
3227                 rdev->config.cik.max_gprs = 256;
3228                 rdev->config.cik.max_gs_threads = 16;
3229                 rdev->config.cik.max_hw_contexts = 8;
3230
3231                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3232                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3233                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3234                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3235                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3236                 break;
3237         case CHIP_KABINI:
3238         default:
3239                 rdev->config.cik.max_shader_engines = 1;
3240                 rdev->config.cik.max_tile_pipes = 2;
3241                 rdev->config.cik.max_cu_per_sh = 2;
3242                 rdev->config.cik.max_sh_per_se = 1;
3243                 rdev->config.cik.max_backends_per_se = 1;
3244                 rdev->config.cik.max_texture_channel_caches = 2;
3245                 rdev->config.cik.max_gprs = 256;
3246                 rdev->config.cik.max_gs_threads = 16;
3247                 rdev->config.cik.max_hw_contexts = 8;
3248
3249                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3250                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3251                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3252                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3253                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3254                 break;
3255         }
3256
3257         /* Initialize HDP */
3258         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3259                 WREG32((0x2c14 + j), 0x00000000);
3260                 WREG32((0x2c18 + j), 0x00000000);
3261                 WREG32((0x2c1c + j), 0x00000000);
3262                 WREG32((0x2c20 + j), 0x00000000);
3263                 WREG32((0x2c24 + j), 0x00000000);
3264         }
3265
3266         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3267
3268         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3269
3270         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3271         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3272
3273         rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3274         rdev->config.cik.mem_max_burst_length_bytes = 256;
3275         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3276         rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3277         if (rdev->config.cik.mem_row_size_in_kb > 4)
3278                 rdev->config.cik.mem_row_size_in_kb = 4;
3279         /* XXX use MC settings? */
3280         rdev->config.cik.shader_engine_tile_size = 32;
3281         rdev->config.cik.num_gpus = 1;
3282         rdev->config.cik.multi_gpu_tile_size = 64;
3283
3284         /* fix up row size */
3285         gb_addr_config &= ~ROW_SIZE_MASK;
3286         switch (rdev->config.cik.mem_row_size_in_kb) {
3287         case 1:
3288         default:
3289                 gb_addr_config |= ROW_SIZE(0);
3290                 break;
3291         case 2:
3292                 gb_addr_config |= ROW_SIZE(1);
3293                 break;
3294         case 4:
3295                 gb_addr_config |= ROW_SIZE(2);
3296                 break;
3297         }
3298
3299         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3300          * not have bank info, so create a custom tiling dword.
3301          * bits 3:0   num_pipes
3302          * bits 7:4   num_banks
3303          * bits 11:8  group_size
3304          * bits 15:12 row_size
3305          */
3306         rdev->config.cik.tile_config = 0;
3307         switch (rdev->config.cik.num_tile_pipes) {
3308         case 1:
3309                 rdev->config.cik.tile_config |= (0 << 0);
3310                 break;
3311         case 2:
3312                 rdev->config.cik.tile_config |= (1 << 0);
3313                 break;
3314         case 4:
3315                 rdev->config.cik.tile_config |= (2 << 0);
3316                 break;
3317         case 8:
3318         default:
3319                 /* XXX what about 12? */
3320                 rdev->config.cik.tile_config |= (3 << 0);
3321                 break;
3322         }
3323         rdev->config.cik.tile_config |=
3324                 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3325         rdev->config.cik.tile_config |=
3326                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3327         rdev->config.cik.tile_config |=
3328                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3329
3330         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3331         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3332         WREG32(DMIF_ADDR_CALC, gb_addr_config);
3333         WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3334         WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3335         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3336         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3337         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3338
3339         cik_tiling_mode_table_init(rdev);
3340
3341         cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3342                      rdev->config.cik.max_sh_per_se,
3343                      rdev->config.cik.max_backends_per_se);
3344
3345         /* set HW defaults for 3D engine */
3346         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3347
3348         WREG32(SX_DEBUG_1, 0x20);
3349
3350         WREG32(TA_CNTL_AUX, 0x00010000);
3351
3352         tmp = RREG32(SPI_CONFIG_CNTL);
3353         tmp |= 0x03000000;
3354         WREG32(SPI_CONFIG_CNTL, tmp);
3355
3356         WREG32(SQ_CONFIG, 1);
3357
3358         WREG32(DB_DEBUG, 0);
3359
3360         tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3361         tmp |= 0x00000400;
3362         WREG32(DB_DEBUG2, tmp);
3363
3364         tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3365         tmp |= 0x00020200;
3366         WREG32(DB_DEBUG3, tmp);
3367
3368         tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3369         tmp |= 0x00018208;
3370         WREG32(CB_HW_CONTROL, tmp);
3371
3372         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3373
3374         WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3375                                  SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3376                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3377                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3378
3379         WREG32(VGT_NUM_INSTANCES, 1);
3380
3381         WREG32(CP_PERFMON_CNTL, 0);
3382
3383         WREG32(SQ_CONFIG, 0);
3384
3385         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3386                                           FORCE_EOV_MAX_REZ_CNT(255)));
3387
3388         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3389                AUTO_INVLD_EN(ES_AND_GS_AUTO));
3390
3391         WREG32(VGT_GS_VERTEX_REUSE, 16);
3392         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3393
3394         tmp = RREG32(HDP_MISC_CNTL);
3395         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3396         WREG32(HDP_MISC_CNTL, tmp);
3397
3398         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3399         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3400
3401         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3402         WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3403
3404         udelay(50);
3405 }
3406
3407 /*
3408  * GPU scratch registers helpers function.
3409  */
3410 /**
3411  * cik_scratch_init - setup driver info for CP scratch regs
3412  *
3413  * @rdev: radeon_device pointer
3414  *
3415  * Set up the number and offset of the CP scratch registers.
3416  * NOTE: use of CP scratch registers is a legacy inferface and
3417  * is not used by default on newer asics (r6xx+).  On newer asics,
3418  * memory buffers are used for fences rather than scratch regs.
3419  */
3420 static void cik_scratch_init(struct radeon_device *rdev)
3421 {
3422         int i;
3423
3424         rdev->scratch.num_reg = 7;
3425         rdev->scratch.reg_base = SCRATCH_REG0;
3426         for (i = 0; i < rdev->scratch.num_reg; i++) {
3427                 rdev->scratch.free[i] = true;
3428                 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3429         }
3430 }
3431
3432 /**
3433  * cik_ring_test - basic gfx ring test
3434  *
3435  * @rdev: radeon_device pointer
3436  * @ring: radeon_ring structure holding ring information
3437  *
3438  * Allocate a scratch register and write to it using the gfx ring (CIK).
3439  * Provides a basic gfx ring test to verify that the ring is working.
3440  * Used by cik_cp_gfx_resume();
3441  * Returns 0 on success, error on failure.
3442  */
3443 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3444 {
3445         uint32_t scratch;
3446         uint32_t tmp = 0;
3447         unsigned i;
3448         int r;
3449
3450         r = radeon_scratch_get(rdev, &scratch);
3451         if (r) {
3452                 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3453                 return r;
3454         }
3455         WREG32(scratch, 0xCAFEDEAD);
3456         r = radeon_ring_lock(rdev, ring, 3);
3457         if (r) {
3458                 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3459                 radeon_scratch_free(rdev, scratch);
3460                 return r;
3461         }
3462         radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3463         radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3464         radeon_ring_write(ring, 0xDEADBEEF);
3465         radeon_ring_unlock_commit(rdev, ring);
3466
3467         for (i = 0; i < rdev->usec_timeout; i++) {
3468                 tmp = RREG32(scratch);
3469                 if (tmp == 0xDEADBEEF)
3470                         break;
3471                 DRM_UDELAY(1);
3472         }
3473         if (i < rdev->usec_timeout) {
3474                 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3475         } else {
3476                 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3477                           ring->idx, scratch, tmp);
3478                 r = -EINVAL;
3479         }
3480         radeon_scratch_free(rdev, scratch);
3481         return r;
3482 }
3483
3484 /**
3485  * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3486  *
3487  * @rdev: radeon_device pointer
3488  * @fence: radeon fence object
3489  *
3490  * Emits a fence sequnce number on the gfx ring and flushes
3491  * GPU caches.
3492  */
3493 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3494                              struct radeon_fence *fence)
3495 {
3496         struct radeon_ring *ring = &rdev->ring[fence->ring];
3497         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3498
3499         /* EVENT_WRITE_EOP - flush caches, send int */
3500         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3501         radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3502                                  EOP_TC_ACTION_EN |
3503                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3504                                  EVENT_INDEX(5)));
3505         radeon_ring_write(ring, addr & 0xfffffffc);
3506         radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3507         radeon_ring_write(ring, fence->seq);
3508         radeon_ring_write(ring, 0);
3509         /* HDP flush */
3510         /* We should be using the new WAIT_REG_MEM special op packet here
3511          * but it causes the CP to hang
3512          */
3513         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3514         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3515                                  WRITE_DATA_DST_SEL(0)));
3516         radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3517         radeon_ring_write(ring, 0);
3518         radeon_ring_write(ring, 0);
3519 }
3520
3521 /**
3522  * cik_fence_compute_ring_emit - emit a fence on the compute ring
3523  *
3524  * @rdev: radeon_device pointer
3525  * @fence: radeon fence object
3526  *
3527  * Emits a fence sequnce number on the compute ring and flushes
3528  * GPU caches.
3529  */
3530 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3531                                  struct radeon_fence *fence)
3532 {
3533         struct radeon_ring *ring = &rdev->ring[fence->ring];
3534         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3535
3536         /* RELEASE_MEM - flush caches, send int */
3537         radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3538         radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3539                                  EOP_TC_ACTION_EN |
3540                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3541                                  EVENT_INDEX(5)));
3542         radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3543         radeon_ring_write(ring, addr & 0xfffffffc);
3544         radeon_ring_write(ring, upper_32_bits(addr));
3545         radeon_ring_write(ring, fence->seq);
3546         radeon_ring_write(ring, 0);
3547         /* HDP flush */
3548         /* We should be using the new WAIT_REG_MEM special op packet here
3549          * but it causes the CP to hang
3550          */
3551         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3552         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3553                                  WRITE_DATA_DST_SEL(0)));
3554         radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3555         radeon_ring_write(ring, 0);
3556         radeon_ring_write(ring, 0);
3557 }
3558
3559 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3560                              struct radeon_ring *ring,
3561                              struct radeon_semaphore *semaphore,
3562                              bool emit_wait)
3563 {
3564 /* TODO: figure out why semaphore cause lockups */
3565 #if 0
3566         uint64_t addr = semaphore->gpu_addr;
3567         unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3568
3569         radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3570         radeon_ring_write(ring, addr & 0xffffffff);
3571         radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3572
3573         return true;
3574 #else
3575         return false;
3576 #endif
3577 }
3578
3579 /**
3580  * cik_copy_cpdma - copy pages using the CP DMA engine
3581  *
3582  * @rdev: radeon_device pointer
3583  * @src_offset: src GPU address
3584  * @dst_offset: dst GPU address
3585  * @num_gpu_pages: number of GPU pages to xfer
3586  * @fence: radeon fence object
3587  *
3588  * Copy GPU paging using the CP DMA engine (CIK+).
3589  * Used by the radeon ttm implementation to move pages if
3590  * registered as the asic copy callback.
3591  */
3592 int cik_copy_cpdma(struct radeon_device *rdev,
3593                    uint64_t src_offset, uint64_t dst_offset,
3594                    unsigned num_gpu_pages,
3595                    struct radeon_fence **fence)
3596 {
3597         struct radeon_semaphore *sem = NULL;
3598         int ring_index = rdev->asic->copy.blit_ring_index;
3599         struct radeon_ring *ring = &rdev->ring[ring_index];
3600         u32 size_in_bytes, cur_size_in_bytes, control;
3601         int i, num_loops;
3602         int r = 0;
3603
3604         r = radeon_semaphore_create(rdev, &sem);
3605         if (r) {
3606                 DRM_ERROR("radeon: moving bo (%d).\n", r);
3607                 return r;
3608         }
3609
3610         size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3611         num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3612         r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3613         if (r) {
3614                 DRM_ERROR("radeon: moving bo (%d).\n", r);
3615                 radeon_semaphore_free(rdev, &sem, NULL);
3616                 return r;
3617         }
3618
3619         radeon_semaphore_sync_to(sem, *fence);
3620         radeon_semaphore_sync_rings(rdev, sem, ring->idx);
3621
3622         for (i = 0; i < num_loops; i++) {
3623                 cur_size_in_bytes = size_in_bytes;
3624                 if (cur_size_in_bytes > 0x1fffff)
3625                         cur_size_in_bytes = 0x1fffff;
3626                 size_in_bytes -= cur_size_in_bytes;
3627                 control = 0;
3628                 if (size_in_bytes == 0)
3629                         control |= PACKET3_DMA_DATA_CP_SYNC;
3630                 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3631                 radeon_ring_write(ring, control);
3632                 radeon_ring_write(ring, lower_32_bits(src_offset));
3633                 radeon_ring_write(ring, upper_32_bits(src_offset));
3634                 radeon_ring_write(ring, lower_32_bits(dst_offset));
3635                 radeon_ring_write(ring, upper_32_bits(dst_offset));
3636                 radeon_ring_write(ring, cur_size_in_bytes);
3637                 src_offset += cur_size_in_bytes;
3638                 dst_offset += cur_size_in_bytes;
3639         }
3640
3641         r = radeon_fence_emit(rdev, fence, ring->idx);
3642         if (r) {
3643                 radeon_ring_unlock_undo(rdev, ring);
3644                 return r;
3645         }
3646
3647         radeon_ring_unlock_commit(rdev, ring);
3648         radeon_semaphore_free(rdev, &sem, *fence);
3649
3650         return r;
3651 }
3652
3653 /*
3654  * IB stuff
3655  */
3656 /**
3657  * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3658  *
3659  * @rdev: radeon_device pointer
3660  * @ib: radeon indirect buffer object
3661  *
3662  * Emits an DE (drawing engine) or CE (constant engine) IB
3663  * on the gfx ring.  IBs are usually generated by userspace
3664  * acceleration drivers and submitted to the kernel for
3665  * sheduling on the ring.  This function schedules the IB
3666  * on the gfx ring for execution by the GPU.
3667  */
3668 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3669 {
3670         struct radeon_ring *ring = &rdev->ring[ib->ring];
3671         u32 header, control = INDIRECT_BUFFER_VALID;
3672
3673         if (ib->is_const_ib) {
3674                 /* set switch buffer packet before const IB */
3675                 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3676                 radeon_ring_write(ring, 0);
3677
3678                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3679         } else {
3680                 u32 next_rptr;
3681                 if (ring->rptr_save_reg) {
3682                         next_rptr = ring->wptr + 3 + 4;
3683                         radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3684                         radeon_ring_write(ring, ((ring->rptr_save_reg -
3685                                                   PACKET3_SET_UCONFIG_REG_START) >> 2));
3686                         radeon_ring_write(ring, next_rptr);
3687                 } else if (rdev->wb.enabled) {
3688                         next_rptr = ring->wptr + 5 + 4;
3689                         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3690                         radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3691                         radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3692                         radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3693                         radeon_ring_write(ring, next_rptr);
3694                 }
3695
3696                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3697         }
3698
3699         control |= ib->length_dw |
3700                 (ib->vm ? (ib->vm->id << 24) : 0);
3701
3702         radeon_ring_write(ring, header);
3703         radeon_ring_write(ring,
3704 #ifdef __BIG_ENDIAN
3705                           (2 << 0) |
3706 #endif
3707                           (ib->gpu_addr & 0xFFFFFFFC));
3708         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3709         radeon_ring_write(ring, control);
3710 }
3711
3712 /**
3713  * cik_ib_test - basic gfx ring IB test
3714  *
3715  * @rdev: radeon_device pointer
3716  * @ring: radeon_ring structure holding ring information
3717  *
3718  * Allocate an IB and execute it on the gfx ring (CIK).
3719  * Provides a basic gfx ring test to verify that IBs are working.
3720  * Returns 0 on success, error on failure.
3721  */
3722 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3723 {
3724         struct radeon_ib ib;
3725         uint32_t scratch;
3726         uint32_t tmp = 0;
3727         unsigned i;
3728         int r;
3729
3730         r = radeon_scratch_get(rdev, &scratch);
3731         if (r) {
3732                 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3733                 return r;
3734         }
3735         WREG32(scratch, 0xCAFEDEAD);
3736         r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3737         if (r) {
3738                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3739                 radeon_scratch_free(rdev, scratch);
3740                 return r;
3741         }
3742         ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3743         ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3744         ib.ptr[2] = 0xDEADBEEF;
3745         ib.length_dw = 3;
3746         r = radeon_ib_schedule(rdev, &ib, NULL);
3747         if (r) {
3748                 radeon_scratch_free(rdev, scratch);
3749                 radeon_ib_free(rdev, &ib);
3750                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3751                 return r;
3752         }
3753         r = radeon_fence_wait(ib.fence, false);
3754         if (r) {
3755                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3756                 radeon_scratch_free(rdev, scratch);
3757                 radeon_ib_free(rdev, &ib);
3758                 return r;
3759         }
3760         for (i = 0; i < rdev->usec_timeout; i++) {
3761                 tmp = RREG32(scratch);
3762                 if (tmp == 0xDEADBEEF)
3763                         break;
3764                 DRM_UDELAY(1);
3765         }
3766         if (i < rdev->usec_timeout) {
3767                 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3768         } else {
3769                 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3770                           scratch, tmp);
3771                 r = -EINVAL;
3772         }
3773         radeon_scratch_free(rdev, scratch);
3774         radeon_ib_free(rdev, &ib);
3775         return r;
3776 }
3777
3778 /*
3779  * CP.
3780  * On CIK, gfx and compute now have independant command processors.
3781  *
3782  * GFX
3783  * Gfx consists of a single ring and can process both gfx jobs and
3784  * compute jobs.  The gfx CP consists of three microengines (ME):
3785  * PFP - Pre-Fetch Parser
3786  * ME - Micro Engine
3787  * CE - Constant Engine
3788  * The PFP and ME make up what is considered the Drawing Engine (DE).
3789  * The CE is an asynchronous engine used for updating buffer desciptors
3790  * used by the DE so that they can be loaded into cache in parallel
3791  * while the DE is processing state update packets.
3792  *
3793  * Compute
3794  * The compute CP consists of two microengines (ME):
3795  * MEC1 - Compute MicroEngine 1
3796  * MEC2 - Compute MicroEngine 2
3797  * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3798  * The queues are exposed to userspace and are programmed directly
3799  * by the compute runtime.
3800  */
3801 /**
3802  * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3803  *
3804  * @rdev: radeon_device pointer
3805  * @enable: enable or disable the MEs
3806  *
3807  * Halts or unhalts the gfx MEs.
3808  */
3809 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3810 {
3811         if (enable)
3812                 WREG32(CP_ME_CNTL, 0);
3813         else {
3814                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3815                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3816         }
3817         udelay(50);
3818 }
3819
3820 /**
3821  * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3822  *
3823  * @rdev: radeon_device pointer
3824  *
3825  * Loads the gfx PFP, ME, and CE ucode.
3826  * Returns 0 for success, -EINVAL if the ucode is not available.
3827  */
3828 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3829 {
3830         const __be32 *fw_data;
3831         int i;
3832
3833         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3834                 return -EINVAL;
3835
3836         cik_cp_gfx_enable(rdev, false);
3837
3838         /* PFP */
3839         fw_data = (const __be32 *)rdev->pfp_fw->data;
3840         WREG32(CP_PFP_UCODE_ADDR, 0);
3841         for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3842                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3843         WREG32(CP_PFP_UCODE_ADDR, 0);
3844
3845         /* CE */
3846         fw_data = (const __be32 *)rdev->ce_fw->data;
3847         WREG32(CP_CE_UCODE_ADDR, 0);
3848         for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3849                 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3850         WREG32(CP_CE_UCODE_ADDR, 0);
3851
3852         /* ME */
3853         fw_data = (const __be32 *)rdev->me_fw->data;
3854         WREG32(CP_ME_RAM_WADDR, 0);
3855         for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3856                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3857         WREG32(CP_ME_RAM_WADDR, 0);
3858
3859         WREG32(CP_PFP_UCODE_ADDR, 0);
3860         WREG32(CP_CE_UCODE_ADDR, 0);
3861         WREG32(CP_ME_RAM_WADDR, 0);
3862         WREG32(CP_ME_RAM_RADDR, 0);
3863         return 0;
3864 }
3865
3866 /**
3867  * cik_cp_gfx_start - start the gfx ring
3868  *
3869  * @rdev: radeon_device pointer
3870  *
3871  * Enables the ring and loads the clear state context and other
3872  * packets required to init the ring.
3873  * Returns 0 for success, error for failure.
3874  */
3875 static int cik_cp_gfx_start(struct radeon_device *rdev)
3876 {
3877         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3878         int r, i;
3879
3880         /* init the CP */
3881         WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3882         WREG32(CP_ENDIAN_SWAP, 0);
3883         WREG32(CP_DEVICE_ID, 1);
3884
3885         cik_cp_gfx_enable(rdev, true);
3886
3887         r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3888         if (r) {
3889                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3890                 return r;
3891         }
3892
3893         /* init the CE partitions.  CE only used for gfx on CIK */
3894         radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3895         radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3896         radeon_ring_write(ring, 0xc000);
3897         radeon_ring_write(ring, 0xc000);
3898
3899         /* setup clear context state */
3900         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3901         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3902
3903         radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3904         radeon_ring_write(ring, 0x80000000);
3905         radeon_ring_write(ring, 0x80000000);
3906
3907         for (i = 0; i < cik_default_size; i++)
3908                 radeon_ring_write(ring, cik_default_state[i]);
3909
3910         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3911         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3912
3913         /* set clear context state */
3914         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3915         radeon_ring_write(ring, 0);
3916
3917         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3918         radeon_ring_write(ring, 0x00000316);
3919         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3920         radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3921
3922         radeon_ring_unlock_commit(rdev, ring);
3923
3924         return 0;
3925 }
3926
3927 /**
3928  * cik_cp_gfx_fini - stop the gfx ring
3929  *
3930  * @rdev: radeon_device pointer
3931  *
3932  * Stop the gfx ring and tear down the driver ring
3933  * info.
3934  */
3935 static void cik_cp_gfx_fini(struct radeon_device *rdev)
3936 {
3937         cik_cp_gfx_enable(rdev, false);
3938         radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3939 }
3940
3941 /**
3942  * cik_cp_gfx_resume - setup the gfx ring buffer registers
3943  *
3944  * @rdev: radeon_device pointer
3945  *
3946  * Program the location and size of the gfx ring buffer
3947  * and test it to make sure it's working.
3948  * Returns 0 for success, error for failure.
3949  */
3950 static int cik_cp_gfx_resume(struct radeon_device *rdev)
3951 {
3952         struct radeon_ring *ring;
3953         u32 tmp;
3954         u32 rb_bufsz;
3955         u64 rb_addr;
3956         int r;
3957
3958         WREG32(CP_SEM_WAIT_TIMER, 0x0);
3959         if (rdev->family != CHIP_HAWAII)
3960                 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3961
3962         /* Set the write pointer delay */
3963         WREG32(CP_RB_WPTR_DELAY, 0);
3964
3965         /* set the RB to use vmid 0 */
3966         WREG32(CP_RB_VMID, 0);
3967
3968         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3969
3970         /* ring 0 - compute and gfx */
3971         /* Set ring buffer size */
3972         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3973         rb_bufsz = order_base_2(ring->ring_size / 8);
3974         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3975 #ifdef __BIG_ENDIAN
3976         tmp |= BUF_SWAP_32BIT;
3977 #endif
3978         WREG32(CP_RB0_CNTL, tmp);
3979
3980         /* Initialize the ring buffer's read and write pointers */
3981         WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3982         ring->wptr = 0;
3983         WREG32(CP_RB0_WPTR, ring->wptr);
3984
3985         /* set the wb address wether it's enabled or not */
3986         WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3987         WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3988
3989         /* scratch register shadowing is no longer supported */
3990         WREG32(SCRATCH_UMSK, 0);
3991
3992         if (!rdev->wb.enabled)
3993                 tmp |= RB_NO_UPDATE;
3994
3995         mdelay(1);
3996         WREG32(CP_RB0_CNTL, tmp);
3997
3998         rb_addr = ring->gpu_addr >> 8;
3999         WREG32(CP_RB0_BASE, rb_addr);
4000         WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4001
4002         ring->rptr = RREG32(CP_RB0_RPTR);
4003
4004         /* start the ring */
4005         cik_cp_gfx_start(rdev);
4006         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4007         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4008         if (r) {
4009                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4010                 return r;
4011         }
4012         return 0;
4013 }
4014
4015 u32 cik_compute_ring_get_rptr(struct radeon_device *rdev,
4016                               struct radeon_ring *ring)
4017 {
4018         u32 rptr;
4019
4020
4021
4022         if (rdev->wb.enabled) {
4023                 rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
4024         } else {
4025                 mutex_lock(&rdev->srbm_mutex);
4026                 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4027                 rptr = RREG32(CP_HQD_PQ_RPTR);
4028                 cik_srbm_select(rdev, 0, 0, 0, 0);
4029                 mutex_unlock(&rdev->srbm_mutex);
4030         }
4031
4032         return rptr;
4033 }
4034
4035 u32 cik_compute_ring_get_wptr(struct radeon_device *rdev,
4036                               struct radeon_ring *ring)
4037 {
4038         u32 wptr;
4039
4040         if (rdev->wb.enabled) {
4041                 wptr = le32_to_cpu(rdev->wb.wb[ring->wptr_offs/4]);
4042         } else {
4043                 mutex_lock(&rdev->srbm_mutex);
4044                 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4045                 wptr = RREG32(CP_HQD_PQ_WPTR);
4046                 cik_srbm_select(rdev, 0, 0, 0, 0);
4047                 mutex_unlock(&rdev->srbm_mutex);
4048         }
4049
4050         return wptr;
4051 }
4052
4053 void cik_compute_ring_set_wptr(struct radeon_device *rdev,
4054                                struct radeon_ring *ring)
4055 {
4056         rdev->wb.wb[ring->wptr_offs/4] = cpu_to_le32(ring->wptr);
4057         WDOORBELL32(ring->doorbell_index, ring->wptr);
4058 }
4059
4060 /**
4061  * cik_cp_compute_enable - enable/disable the compute CP MEs
4062  *
4063  * @rdev: radeon_device pointer
4064  * @enable: enable or disable the MEs
4065  *
4066  * Halts or unhalts the compute MEs.
4067  */
4068 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4069 {
4070         if (enable)
4071                 WREG32(CP_MEC_CNTL, 0);
4072         else
4073                 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4074         udelay(50);
4075 }
4076
4077 /**
4078  * cik_cp_compute_load_microcode - load the compute CP ME ucode
4079  *
4080  * @rdev: radeon_device pointer
4081  *
4082  * Loads the compute MEC1&2 ucode.
4083  * Returns 0 for success, -EINVAL if the ucode is not available.
4084  */
4085 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4086 {
4087         const __be32 *fw_data;
4088         int i;
4089
4090         if (!rdev->mec_fw)
4091                 return -EINVAL;
4092
4093         cik_cp_compute_enable(rdev, false);
4094
4095         /* MEC1 */
4096         fw_data = (const __be32 *)rdev->mec_fw->data;
4097         WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4098         for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4099                 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4100         WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4101
4102         if (rdev->family == CHIP_KAVERI) {
4103                 /* MEC2 */
4104                 fw_data = (const __be32 *)rdev->mec_fw->data;
4105                 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4106                 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4107                         WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4108                 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4109         }
4110
4111         return 0;
4112 }
4113
4114 /**
4115  * cik_cp_compute_start - start the compute queues
4116  *
4117  * @rdev: radeon_device pointer
4118  *
4119  * Enable the compute queues.
4120  * Returns 0 for success, error for failure.
4121  */
4122 static int cik_cp_compute_start(struct radeon_device *rdev)
4123 {
4124         cik_cp_compute_enable(rdev, true);
4125
4126         return 0;
4127 }
4128
4129 /**
4130  * cik_cp_compute_fini - stop the compute queues
4131  *
4132  * @rdev: radeon_device pointer
4133  *
4134  * Stop the compute queues and tear down the driver queue
4135  * info.
4136  */
4137 static void cik_cp_compute_fini(struct radeon_device *rdev)
4138 {
4139         int i, idx, r;
4140
4141         cik_cp_compute_enable(rdev, false);
4142
4143         for (i = 0; i < 2; i++) {
4144                 if (i == 0)
4145                         idx = CAYMAN_RING_TYPE_CP1_INDEX;
4146                 else
4147                         idx = CAYMAN_RING_TYPE_CP2_INDEX;
4148
4149                 if (rdev->ring[idx].mqd_obj) {
4150                         r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4151                         if (unlikely(r != 0))
4152                                 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4153
4154                         radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4155                         radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4156
4157                         radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4158                         rdev->ring[idx].mqd_obj = NULL;
4159                 }
4160         }
4161 }
4162
4163 static void cik_mec_fini(struct radeon_device *rdev)
4164 {
4165         int r;
4166
4167         if (rdev->mec.hpd_eop_obj) {
4168                 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4169                 if (unlikely(r != 0))
4170                         dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4171                 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4172                 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4173
4174                 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4175                 rdev->mec.hpd_eop_obj = NULL;
4176         }
4177 }
4178
4179 #define MEC_HPD_SIZE 2048
4180
4181 static int cik_mec_init(struct radeon_device *rdev)
4182 {
4183         int r;
4184         u32 *hpd;
4185
4186         /*
4187          * KV:    2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4188          * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4189          */
4190         if (rdev->family == CHIP_KAVERI)
4191                 rdev->mec.num_mec = 2;
4192         else
4193                 rdev->mec.num_mec = 1;
4194         rdev->mec.num_pipe = 4;
4195         rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4196
4197         if (rdev->mec.hpd_eop_obj == NULL) {
4198                 r = radeon_bo_create(rdev,
4199                                      rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4200                                      PAGE_SIZE, true,
4201                                      RADEON_GEM_DOMAIN_GTT, NULL,
4202                                      &rdev->mec.hpd_eop_obj);
4203                 if (r) {
4204                         dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4205                         return r;
4206                 }
4207         }
4208
4209         r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4210         if (unlikely(r != 0)) {
4211                 cik_mec_fini(rdev);
4212                 return r;
4213         }
4214         r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4215                           &rdev->mec.hpd_eop_gpu_addr);
4216         if (r) {
4217                 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4218                 cik_mec_fini(rdev);
4219                 return r;
4220         }
4221         r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4222         if (r) {
4223                 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4224                 cik_mec_fini(rdev);
4225                 return r;
4226         }
4227
4228         /* clear memory.  Not sure if this is required or not */
4229         memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4230
4231         radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4232         radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4233
4234         return 0;
4235 }
4236
4237 struct hqd_registers
4238 {
4239         u32 cp_mqd_base_addr;
4240         u32 cp_mqd_base_addr_hi;
4241         u32 cp_hqd_active;
4242         u32 cp_hqd_vmid;
4243         u32 cp_hqd_persistent_state;
4244         u32 cp_hqd_pipe_priority;
4245         u32 cp_hqd_queue_priority;
4246         u32 cp_hqd_quantum;
4247         u32 cp_hqd_pq_base;
4248         u32 cp_hqd_pq_base_hi;
4249         u32 cp_hqd_pq_rptr;
4250         u32 cp_hqd_pq_rptr_report_addr;
4251         u32 cp_hqd_pq_rptr_report_addr_hi;
4252         u32 cp_hqd_pq_wptr_poll_addr;
4253         u32 cp_hqd_pq_wptr_poll_addr_hi;
4254         u32 cp_hqd_pq_doorbell_control;
4255         u32 cp_hqd_pq_wptr;
4256         u32 cp_hqd_pq_control;
4257         u32 cp_hqd_ib_base_addr;
4258         u32 cp_hqd_ib_base_addr_hi;
4259         u32 cp_hqd_ib_rptr;
4260         u32 cp_hqd_ib_control;
4261         u32 cp_hqd_iq_timer;
4262         u32 cp_hqd_iq_rptr;
4263         u32 cp_hqd_dequeue_request;
4264         u32 cp_hqd_dma_offload;
4265         u32 cp_hqd_sema_cmd;
4266         u32 cp_hqd_msg_type;
4267         u32 cp_hqd_atomic0_preop_lo;
4268         u32 cp_hqd_atomic0_preop_hi;
4269         u32 cp_hqd_atomic1_preop_lo;
4270         u32 cp_hqd_atomic1_preop_hi;
4271         u32 cp_hqd_hq_scheduler0;
4272         u32 cp_hqd_hq_scheduler1;
4273         u32 cp_mqd_control;
4274 };
4275
4276 struct bonaire_mqd
4277 {
4278         u32 header;
4279         u32 dispatch_initiator;
4280         u32 dimensions[3];
4281         u32 start_idx[3];
4282         u32 num_threads[3];
4283         u32 pipeline_stat_enable;
4284         u32 perf_counter_enable;
4285         u32 pgm[2];
4286         u32 tba[2];
4287         u32 tma[2];
4288         u32 pgm_rsrc[2];
4289         u32 vmid;
4290         u32 resource_limits;
4291         u32 static_thread_mgmt01[2];
4292         u32 tmp_ring_size;
4293         u32 static_thread_mgmt23[2];
4294         u32 restart[3];
4295         u32 thread_trace_enable;
4296         u32 reserved1;
4297         u32 user_data[16];
4298         u32 vgtcs_invoke_count[2];
4299         struct hqd_registers queue_state;
4300         u32 dequeue_cntr;
4301         u32 interrupt_queue[64];
4302 };
4303
4304 /**
4305  * cik_cp_compute_resume - setup the compute queue registers
4306  *
4307  * @rdev: radeon_device pointer
4308  *
4309  * Program the compute queues and test them to make sure they
4310  * are working.
4311  * Returns 0 for success, error for failure.
4312  */
4313 static int cik_cp_compute_resume(struct radeon_device *rdev)
4314 {
4315         int r, i, idx;
4316         u32 tmp;
4317         bool use_doorbell = true;
4318         u64 hqd_gpu_addr;
4319         u64 mqd_gpu_addr;
4320         u64 eop_gpu_addr;
4321         u64 wb_gpu_addr;
4322         u32 *buf;
4323         struct bonaire_mqd *mqd;
4324
4325         r = cik_cp_compute_start(rdev);
4326         if (r)
4327                 return r;
4328
4329         /* fix up chicken bits */
4330         tmp = RREG32(CP_CPF_DEBUG);
4331         tmp |= (1 << 23);
4332         WREG32(CP_CPF_DEBUG, tmp);
4333
4334         /* init the pipes */
4335         mutex_lock(&rdev->srbm_mutex);
4336         for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4337                 int me = (i < 4) ? 1 : 2;
4338                 int pipe = (i < 4) ? i : (i - 4);
4339
4340                 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4341
4342                 cik_srbm_select(rdev, me, pipe, 0, 0);
4343
4344                 /* write the EOP addr */
4345                 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4346                 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4347
4348                 /* set the VMID assigned */
4349                 WREG32(CP_HPD_EOP_VMID, 0);
4350
4351                 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4352                 tmp = RREG32(CP_HPD_EOP_CONTROL);
4353                 tmp &= ~EOP_SIZE_MASK;
4354                 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4355                 WREG32(CP_HPD_EOP_CONTROL, tmp);
4356         }
4357         cik_srbm_select(rdev, 0, 0, 0, 0);
4358         mutex_unlock(&rdev->srbm_mutex);
4359
4360         /* init the queues.  Just two for now. */
4361         for (i = 0; i < 2; i++) {
4362                 if (i == 0)
4363                         idx = CAYMAN_RING_TYPE_CP1_INDEX;
4364                 else
4365                         idx = CAYMAN_RING_TYPE_CP2_INDEX;
4366
4367                 if (rdev->ring[idx].mqd_obj == NULL) {
4368                         r = radeon_bo_create(rdev,
4369                                              sizeof(struct bonaire_mqd),
4370                                              PAGE_SIZE, true,
4371                                              RADEON_GEM_DOMAIN_GTT, NULL,
4372                                              &rdev->ring[idx].mqd_obj);
4373                         if (r) {
4374                                 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4375                                 return r;
4376                         }
4377                 }
4378
4379                 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4380                 if (unlikely(r != 0)) {
4381                         cik_cp_compute_fini(rdev);
4382                         return r;
4383                 }
4384                 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4385                                   &mqd_gpu_addr);
4386                 if (r) {
4387                         dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4388                         cik_cp_compute_fini(rdev);
4389                         return r;
4390                 }
4391                 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4392                 if (r) {
4393                         dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4394                         cik_cp_compute_fini(rdev);
4395                         return r;
4396                 }
4397
4398                 /* init the mqd struct */
4399                 memset(buf, 0, sizeof(struct bonaire_mqd));
4400
4401                 mqd = (struct bonaire_mqd *)buf;
4402                 mqd->header = 0xC0310800;
4403                 mqd->static_thread_mgmt01[0] = 0xffffffff;
4404                 mqd->static_thread_mgmt01[1] = 0xffffffff;
4405                 mqd->static_thread_mgmt23[0] = 0xffffffff;
4406                 mqd->static_thread_mgmt23[1] = 0xffffffff;
4407
4408                 mutex_lock(&rdev->srbm_mutex);
4409                 cik_srbm_select(rdev, rdev->ring[idx].me,
4410                                 rdev->ring[idx].pipe,
4411                                 rdev->ring[idx].queue, 0);
4412
4413                 /* disable wptr polling */
4414                 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4415                 tmp &= ~WPTR_POLL_EN;
4416                 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4417
4418                 /* enable doorbell? */
4419                 mqd->queue_state.cp_hqd_pq_doorbell_control =
4420                         RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4421                 if (use_doorbell)
4422                         mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4423                 else
4424                         mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4425                 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4426                        mqd->queue_state.cp_hqd_pq_doorbell_control);
4427
4428                 /* disable the queue if it's active */
4429                 mqd->queue_state.cp_hqd_dequeue_request = 0;
4430                 mqd->queue_state.cp_hqd_pq_rptr = 0;
4431                 mqd->queue_state.cp_hqd_pq_wptr= 0;
4432                 if (RREG32(CP_HQD_ACTIVE) & 1) {
4433                         WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4434                         for (i = 0; i < rdev->usec_timeout; i++) {
4435                                 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4436                                         break;
4437                                 udelay(1);
4438                         }
4439                         WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4440                         WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4441                         WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4442                 }
4443
4444                 /* set the pointer to the MQD */
4445                 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4446                 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4447                 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4448                 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4449                 /* set MQD vmid to 0 */
4450                 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4451                 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4452                 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4453
4454                 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4455                 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4456                 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4457                 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4458                 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4459                 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4460
4461                 /* set up the HQD, this is similar to CP_RB0_CNTL */
4462                 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4463                 mqd->queue_state.cp_hqd_pq_control &=
4464                         ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4465
4466                 mqd->queue_state.cp_hqd_pq_control |=
4467                         order_base_2(rdev->ring[idx].ring_size / 8);
4468                 mqd->queue_state.cp_hqd_pq_control |=
4469                         (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4470 #ifdef __BIG_ENDIAN
4471                 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4472 #endif
4473                 mqd->queue_state.cp_hqd_pq_control &=
4474                         ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4475                 mqd->queue_state.cp_hqd_pq_control |=
4476                         PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4477                 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4478
4479                 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4480                 if (i == 0)
4481                         wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4482                 else
4483                         wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4484                 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4485                 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4486                 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4487                 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4488                        mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4489
4490                 /* set the wb address wether it's enabled or not */
4491                 if (i == 0)
4492                         wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4493                 else
4494                         wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4495                 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4496                 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4497                         upper_32_bits(wb_gpu_addr) & 0xffff;
4498                 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4499                        mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4500                 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4501                        mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4502
4503                 /* enable the doorbell if requested */
4504                 if (use_doorbell) {
4505                         mqd->queue_state.cp_hqd_pq_doorbell_control =
4506                                 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4507                         mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4508                         mqd->queue_state.cp_hqd_pq_doorbell_control |=
4509                                 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
4510                         mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4511                         mqd->queue_state.cp_hqd_pq_doorbell_control &=
4512                                 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4513
4514                 } else {
4515                         mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4516                 }
4517                 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4518                        mqd->queue_state.cp_hqd_pq_doorbell_control);
4519
4520                 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4521                 rdev->ring[idx].wptr = 0;
4522                 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4523                 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4524                 rdev->ring[idx].rptr = RREG32(CP_HQD_PQ_RPTR);
4525                 mqd->queue_state.cp_hqd_pq_rptr = rdev->ring[idx].rptr;
4526
4527                 /* set the vmid for the queue */
4528                 mqd->queue_state.cp_hqd_vmid = 0;
4529                 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4530
4531                 /* activate the queue */
4532                 mqd->queue_state.cp_hqd_active = 1;
4533                 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4534
4535                 cik_srbm_select(rdev, 0, 0, 0, 0);
4536                 mutex_unlock(&rdev->srbm_mutex);
4537
4538                 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4539                 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4540
4541                 rdev->ring[idx].ready = true;
4542                 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4543                 if (r)
4544                         rdev->ring[idx].ready = false;
4545         }
4546
4547         return 0;
4548 }
4549
4550 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4551 {
4552         cik_cp_gfx_enable(rdev, enable);
4553         cik_cp_compute_enable(rdev, enable);
4554 }
4555
4556 static int cik_cp_load_microcode(struct radeon_device *rdev)
4557 {
4558         int r;
4559
4560         r = cik_cp_gfx_load_microcode(rdev);
4561         if (r)
4562                 return r;
4563         r = cik_cp_compute_load_microcode(rdev);
4564         if (r)
4565                 return r;
4566
4567         return 0;
4568 }
4569
4570 static void cik_cp_fini(struct radeon_device *rdev)
4571 {
4572         cik_cp_gfx_fini(rdev);
4573         cik_cp_compute_fini(rdev);
4574 }
4575
4576 static int cik_cp_resume(struct radeon_device *rdev)
4577 {
4578         int r;
4579
4580         cik_enable_gui_idle_interrupt(rdev, false);
4581
4582         r = cik_cp_load_microcode(rdev);
4583         if (r)
4584                 return r;
4585
4586         r = cik_cp_gfx_resume(rdev);
4587         if (r)
4588                 return r;
4589         r = cik_cp_compute_resume(rdev);
4590         if (r)
4591                 return r;
4592
4593         cik_enable_gui_idle_interrupt(rdev, true);
4594
4595         return 0;
4596 }
4597
4598 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4599 {
4600         dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
4601                 RREG32(GRBM_STATUS));
4602         dev_info(rdev->dev, "  GRBM_STATUS2=0x%08X\n",
4603                 RREG32(GRBM_STATUS2));
4604         dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
4605                 RREG32(GRBM_STATUS_SE0));
4606         dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
4607                 RREG32(GRBM_STATUS_SE1));
4608         dev_info(rdev->dev, "  GRBM_STATUS_SE2=0x%08X\n",
4609                 RREG32(GRBM_STATUS_SE2));
4610         dev_info(rdev->dev, "  GRBM_STATUS_SE3=0x%08X\n",
4611                 RREG32(GRBM_STATUS_SE3));
4612         dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
4613                 RREG32(SRBM_STATUS));
4614         dev_info(rdev->dev, "  SRBM_STATUS2=0x%08X\n",
4615                 RREG32(SRBM_STATUS2));
4616         dev_info(rdev->dev, "  SDMA0_STATUS_REG   = 0x%08X\n",
4617                 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4618         dev_info(rdev->dev, "  SDMA1_STATUS_REG   = 0x%08X\n",
4619                  RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
4620         dev_info(rdev->dev, "  CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4621         dev_info(rdev->dev, "  CP_STALLED_STAT1 = 0x%08x\n",
4622                  RREG32(CP_STALLED_STAT1));
4623         dev_info(rdev->dev, "  CP_STALLED_STAT2 = 0x%08x\n",
4624                  RREG32(CP_STALLED_STAT2));
4625         dev_info(rdev->dev, "  CP_STALLED_STAT3 = 0x%08x\n",
4626                  RREG32(CP_STALLED_STAT3));
4627         dev_info(rdev->dev, "  CP_CPF_BUSY_STAT = 0x%08x\n",
4628                  RREG32(CP_CPF_BUSY_STAT));
4629         dev_info(rdev->dev, "  CP_CPF_STALLED_STAT1 = 0x%08x\n",
4630                  RREG32(CP_CPF_STALLED_STAT1));
4631         dev_info(rdev->dev, "  CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4632         dev_info(rdev->dev, "  CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4633         dev_info(rdev->dev, "  CP_CPC_STALLED_STAT1 = 0x%08x\n",
4634                  RREG32(CP_CPC_STALLED_STAT1));
4635         dev_info(rdev->dev, "  CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
4636 }
4637
4638 /**
4639  * cik_gpu_check_soft_reset - check which blocks are busy
4640  *
4641  * @rdev: radeon_device pointer
4642  *
4643  * Check which blocks are busy and return the relevant reset
4644  * mask to be used by cik_gpu_soft_reset().
4645  * Returns a mask of the blocks to be reset.
4646  */
4647 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
4648 {
4649         u32 reset_mask = 0;
4650         u32 tmp;
4651
4652         /* GRBM_STATUS */
4653         tmp = RREG32(GRBM_STATUS);
4654         if (tmp & (PA_BUSY | SC_BUSY |
4655                    BCI_BUSY | SX_BUSY |
4656                    TA_BUSY | VGT_BUSY |
4657                    DB_BUSY | CB_BUSY |
4658                    GDS_BUSY | SPI_BUSY |
4659                    IA_BUSY | IA_BUSY_NO_DMA))
4660                 reset_mask |= RADEON_RESET_GFX;
4661
4662         if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4663                 reset_mask |= RADEON_RESET_CP;
4664
4665         /* GRBM_STATUS2 */
4666         tmp = RREG32(GRBM_STATUS2);
4667         if (tmp & RLC_BUSY)
4668                 reset_mask |= RADEON_RESET_RLC;
4669
4670         /* SDMA0_STATUS_REG */
4671         tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4672         if (!(tmp & SDMA_IDLE))
4673                 reset_mask |= RADEON_RESET_DMA;
4674
4675         /* SDMA1_STATUS_REG */
4676         tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4677         if (!(tmp & SDMA_IDLE))
4678                 reset_mask |= RADEON_RESET_DMA1;
4679
4680         /* SRBM_STATUS2 */
4681         tmp = RREG32(SRBM_STATUS2);
4682         if (tmp & SDMA_BUSY)
4683                 reset_mask |= RADEON_RESET_DMA;
4684
4685         if (tmp & SDMA1_BUSY)
4686                 reset_mask |= RADEON_RESET_DMA1;
4687
4688         /* SRBM_STATUS */
4689         tmp = RREG32(SRBM_STATUS);
4690
4691         if (tmp & IH_BUSY)
4692                 reset_mask |= RADEON_RESET_IH;
4693
4694         if (tmp & SEM_BUSY)
4695                 reset_mask |= RADEON_RESET_SEM;
4696
4697         if (tmp & GRBM_RQ_PENDING)
4698                 reset_mask |= RADEON_RESET_GRBM;
4699
4700         if (tmp & VMC_BUSY)
4701                 reset_mask |= RADEON_RESET_VMC;
4702
4703         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4704                    MCC_BUSY | MCD_BUSY))
4705                 reset_mask |= RADEON_RESET_MC;
4706
4707         if (evergreen_is_display_hung(rdev))
4708                 reset_mask |= RADEON_RESET_DISPLAY;
4709
4710         /* Skip MC reset as it's mostly likely not hung, just busy */
4711         if (reset_mask & RADEON_RESET_MC) {
4712                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4713                 reset_mask &= ~RADEON_RESET_MC;
4714         }
4715
4716         return reset_mask;
4717 }
4718
4719 /**
4720  * cik_gpu_soft_reset - soft reset GPU
4721  *
4722  * @rdev: radeon_device pointer
4723  * @reset_mask: mask of which blocks to reset
4724  *
4725  * Soft reset the blocks specified in @reset_mask.
4726  */
4727 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4728 {
4729         struct evergreen_mc_save save;
4730         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4731         u32 tmp;
4732
4733         if (reset_mask == 0)
4734                 return;
4735
4736         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4737
4738         cik_print_gpu_status_regs(rdev);
4739         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
4740                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4741         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4742                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4743
4744         /* disable CG/PG */
4745         cik_fini_pg(rdev);
4746         cik_fini_cg(rdev);
4747
4748         /* stop the rlc */
4749         cik_rlc_stop(rdev);
4750
4751         /* Disable GFX parsing/prefetching */
4752         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4753
4754         /* Disable MEC parsing/prefetching */
4755         WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4756
4757         if (reset_mask & RADEON_RESET_DMA) {
4758                 /* sdma0 */
4759                 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4760                 tmp |= SDMA_HALT;
4761                 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4762         }
4763         if (reset_mask & RADEON_RESET_DMA1) {
4764                 /* sdma1 */
4765                 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4766                 tmp |= SDMA_HALT;
4767                 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4768         }
4769
4770         evergreen_mc_stop(rdev, &save);
4771         if (evergreen_mc_wait_for_idle(rdev)) {
4772                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4773         }
4774
4775         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4776                 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4777
4778         if (reset_mask & RADEON_RESET_CP) {
4779                 grbm_soft_reset |= SOFT_RESET_CP;
4780
4781                 srbm_soft_reset |= SOFT_RESET_GRBM;
4782         }
4783
4784         if (reset_mask & RADEON_RESET_DMA)
4785                 srbm_soft_reset |= SOFT_RESET_SDMA;
4786
4787         if (reset_mask & RADEON_RESET_DMA1)
4788                 srbm_soft_reset |= SOFT_RESET_SDMA1;
4789
4790         if (reset_mask & RADEON_RESET_DISPLAY)
4791                 srbm_soft_reset |= SOFT_RESET_DC;
4792
4793         if (reset_mask & RADEON_RESET_RLC)
4794                 grbm_soft_reset |= SOFT_RESET_RLC;
4795
4796         if (reset_mask & RADEON_RESET_SEM)
4797                 srbm_soft_reset |= SOFT_RESET_SEM;
4798
4799         if (reset_mask & RADEON_RESET_IH)
4800                 srbm_soft_reset |= SOFT_RESET_IH;
4801
4802         if (reset_mask & RADEON_RESET_GRBM)
4803                 srbm_soft_reset |= SOFT_RESET_GRBM;
4804
4805         if (reset_mask & RADEON_RESET_VMC)
4806                 srbm_soft_reset |= SOFT_RESET_VMC;
4807
4808         if (!(rdev->flags & RADEON_IS_IGP)) {
4809                 if (reset_mask & RADEON_RESET_MC)
4810                         srbm_soft_reset |= SOFT_RESET_MC;
4811         }
4812
4813         if (grbm_soft_reset) {
4814                 tmp = RREG32(GRBM_SOFT_RESET);
4815                 tmp |= grbm_soft_reset;
4816                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4817                 WREG32(GRBM_SOFT_RESET, tmp);
4818                 tmp = RREG32(GRBM_SOFT_RESET);
4819
4820                 udelay(50);
4821
4822                 tmp &= ~grbm_soft_reset;
4823                 WREG32(GRBM_SOFT_RESET, tmp);
4824                 tmp = RREG32(GRBM_SOFT_RESET);
4825         }
4826
4827         if (srbm_soft_reset) {
4828                 tmp = RREG32(SRBM_SOFT_RESET);
4829                 tmp |= srbm_soft_reset;
4830                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4831                 WREG32(SRBM_SOFT_RESET, tmp);
4832                 tmp = RREG32(SRBM_SOFT_RESET);
4833
4834                 udelay(50);
4835
4836                 tmp &= ~srbm_soft_reset;
4837                 WREG32(SRBM_SOFT_RESET, tmp);
4838                 tmp = RREG32(SRBM_SOFT_RESET);
4839         }
4840
4841         /* Wait a little for things to settle down */
4842         udelay(50);
4843
4844         evergreen_mc_resume(rdev, &save);
4845         udelay(50);
4846
4847         cik_print_gpu_status_regs(rdev);
4848 }
4849
4850 /**
4851  * cik_asic_reset - soft reset GPU
4852  *
4853  * @rdev: radeon_device pointer
4854  *
4855  * Look up which blocks are hung and attempt
4856  * to reset them.
4857  * Returns 0 for success.
4858  */
4859 int cik_asic_reset(struct radeon_device *rdev)
4860 {
4861         u32 reset_mask;
4862
4863         reset_mask = cik_gpu_check_soft_reset(rdev);
4864
4865         if (reset_mask)
4866                 r600_set_bios_scratch_engine_hung(rdev, true);
4867
4868         cik_gpu_soft_reset(rdev, reset_mask);
4869
4870         reset_mask = cik_gpu_check_soft_reset(rdev);
4871
4872         if (!reset_mask)
4873                 r600_set_bios_scratch_engine_hung(rdev, false);
4874
4875         return 0;
4876 }
4877
4878 /**
4879  * cik_gfx_is_lockup - check if the 3D engine is locked up
4880  *
4881  * @rdev: radeon_device pointer
4882  * @ring: radeon_ring structure holding ring information
4883  *
4884  * Check if the 3D engine is locked up (CIK).
4885  * Returns true if the engine is locked, false if not.
4886  */
4887 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4888 {
4889         u32 reset_mask = cik_gpu_check_soft_reset(rdev);
4890
4891         if (!(reset_mask & (RADEON_RESET_GFX |
4892                             RADEON_RESET_COMPUTE |
4893                             RADEON_RESET_CP))) {
4894                 radeon_ring_lockup_update(ring);
4895                 return false;
4896         }
4897         /* force CP activities */
4898         radeon_ring_force_activity(rdev, ring);
4899         return radeon_ring_test_lockup(rdev, ring);
4900 }
4901
4902 /* MC */
4903 /**
4904  * cik_mc_program - program the GPU memory controller
4905  *
4906  * @rdev: radeon_device pointer
4907  *
4908  * Set the location of vram, gart, and AGP in the GPU's
4909  * physical address space (CIK).
4910  */
4911 static void cik_mc_program(struct radeon_device *rdev)
4912 {
4913         struct evergreen_mc_save save;
4914         u32 tmp;
4915         int i, j;
4916
4917         /* Initialize HDP */
4918         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4919                 WREG32((0x2c14 + j), 0x00000000);
4920                 WREG32((0x2c18 + j), 0x00000000);
4921                 WREG32((0x2c1c + j), 0x00000000);
4922                 WREG32((0x2c20 + j), 0x00000000);
4923                 WREG32((0x2c24 + j), 0x00000000);
4924         }
4925         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4926
4927         evergreen_mc_stop(rdev, &save);
4928         if (radeon_mc_wait_for_idle(rdev)) {
4929                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4930         }
4931         /* Lockout access through VGA aperture*/
4932         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4933         /* Update configuration */
4934         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4935                rdev->mc.vram_start >> 12);
4936         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4937                rdev->mc.vram_end >> 12);
4938         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4939                rdev->vram_scratch.gpu_addr >> 12);
4940         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4941         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4942         WREG32(MC_VM_FB_LOCATION, tmp);
4943         /* XXX double check these! */
4944         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4945         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4946         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4947         WREG32(MC_VM_AGP_BASE, 0);
4948         WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4949         WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4950         if (radeon_mc_wait_for_idle(rdev)) {
4951                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4952         }
4953         evergreen_mc_resume(rdev, &save);
4954         /* we need to own VRAM, so turn off the VGA renderer here
4955          * to stop it overwriting our objects */
4956         rv515_vga_render_disable(rdev);
4957 }
4958
4959 /**
4960  * cik_mc_init - initialize the memory controller driver params
4961  *
4962  * @rdev: radeon_device pointer
4963  *
4964  * Look up the amount of vram, vram width, and decide how to place
4965  * vram and gart within the GPU's physical address space (CIK).
4966  * Returns 0 for success.
4967  */
4968 static int cik_mc_init(struct radeon_device *rdev)
4969 {
4970         u32 tmp;
4971         int chansize, numchan;
4972
4973         /* Get VRAM informations */
4974         rdev->mc.vram_is_ddr = true;
4975         tmp = RREG32(MC_ARB_RAMCFG);
4976         if (tmp & CHANSIZE_MASK) {
4977                 chansize = 64;
4978         } else {
4979                 chansize = 32;
4980         }
4981         tmp = RREG32(MC_SHARED_CHMAP);
4982         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4983         case 0:
4984         default:
4985                 numchan = 1;
4986                 break;
4987         case 1:
4988                 numchan = 2;
4989                 break;
4990         case 2:
4991                 numchan = 4;
4992                 break;
4993         case 3:
4994                 numchan = 8;
4995                 break;
4996         case 4:
4997                 numchan = 3;
4998                 break;
4999         case 5:
5000                 numchan = 6;
5001                 break;
5002         case 6:
5003                 numchan = 10;
5004                 break;
5005         case 7:
5006                 numchan = 12;
5007                 break;
5008         case 8:
5009                 numchan = 16;
5010                 break;
5011         }
5012         rdev->mc.vram_width = numchan * chansize;
5013         /* Could aper size report 0 ? */
5014         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5015         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5016         /* size in MB on si */
5017         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5018         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5019         rdev->mc.visible_vram_size = rdev->mc.aper_size;
5020         si_vram_gtt_location(rdev, &rdev->mc);
5021         radeon_update_bandwidth_info(rdev);
5022
5023         return 0;
5024 }
5025
5026 /*
5027  * GART
5028  * VMID 0 is the physical GPU addresses as used by the kernel.
5029  * VMIDs 1-15 are used for userspace clients and are handled
5030  * by the radeon vm/hsa code.
5031  */
5032 /**
5033  * cik_pcie_gart_tlb_flush - gart tlb flush callback
5034  *
5035  * @rdev: radeon_device pointer
5036  *
5037  * Flush the TLB for the VMID 0 page table (CIK).
5038  */
5039 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5040 {
5041         /* flush hdp cache */
5042         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5043
5044         /* bits 0-15 are the VM contexts0-15 */
5045         WREG32(VM_INVALIDATE_REQUEST, 0x1);
5046 }
5047
5048 /**
5049  * cik_pcie_gart_enable - gart enable
5050  *
5051  * @rdev: radeon_device pointer
5052  *
5053  * This sets up the TLBs, programs the page tables for VMID0,
5054  * sets up the hw for VMIDs 1-15 which are allocated on
5055  * demand, and sets up the global locations for the LDS, GDS,
5056  * and GPUVM for FSA64 clients (CIK).
5057  * Returns 0 for success, errors for failure.
5058  */
5059 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5060 {
5061         int r, i;
5062
5063         if (rdev->gart.robj == NULL) {
5064                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5065                 return -EINVAL;
5066         }
5067         r = radeon_gart_table_vram_pin(rdev);
5068         if (r)
5069                 return r;
5070         radeon_gart_restore(rdev);
5071         /* Setup TLB control */
5072         WREG32(MC_VM_MX_L1_TLB_CNTL,
5073                (0xA << 7) |
5074                ENABLE_L1_TLB |
5075                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5076                ENABLE_ADVANCED_DRIVER_MODEL |
5077                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5078         /* Setup L2 cache */
5079         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5080                ENABLE_L2_FRAGMENT_PROCESSING |
5081                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5082                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5083                EFFECTIVE_L2_QUEUE_SIZE(7) |
5084                CONTEXT1_IDENTITY_ACCESS_MODE(1));
5085         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5086         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5087                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5088         /* setup context0 */
5089         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5090         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5091         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5092         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5093                         (u32)(rdev->dummy_page.addr >> 12));
5094         WREG32(VM_CONTEXT0_CNTL2, 0);
5095         WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5096                                   RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5097
5098         WREG32(0x15D4, 0);
5099         WREG32(0x15D8, 0);
5100         WREG32(0x15DC, 0);
5101
5102         /* empty context1-15 */
5103         /* FIXME start with 4G, once using 2 level pt switch to full
5104          * vm size space
5105          */
5106         /* set vm size, must be a multiple of 4 */
5107         WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5108         WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5109         for (i = 1; i < 16; i++) {
5110                 if (i < 8)
5111                         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5112                                rdev->gart.table_addr >> 12);
5113                 else
5114                         WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5115                                rdev->gart.table_addr >> 12);
5116         }
5117
5118         /* enable context1-15 */
5119         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5120                (u32)(rdev->dummy_page.addr >> 12));
5121         WREG32(VM_CONTEXT1_CNTL2, 4);
5122         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5123                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5124                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5125                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5126                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5127                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5128                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5129                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5130                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5131                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5132                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5133                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5134                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5135
5136         /* TC cache setup ??? */
5137         WREG32(TC_CFG_L1_LOAD_POLICY0, 0);
5138         WREG32(TC_CFG_L1_LOAD_POLICY1, 0);
5139         WREG32(TC_CFG_L1_STORE_POLICY, 0);
5140
5141         WREG32(TC_CFG_L2_LOAD_POLICY0, 0);
5142         WREG32(TC_CFG_L2_LOAD_POLICY1, 0);
5143         WREG32(TC_CFG_L2_STORE_POLICY0, 0);
5144         WREG32(TC_CFG_L2_STORE_POLICY1, 0);
5145         WREG32(TC_CFG_L2_ATOMIC_POLICY, 0);
5146
5147         WREG32(TC_CFG_L1_VOLATILE, 0);
5148         WREG32(TC_CFG_L2_VOLATILE, 0);
5149
5150         if (rdev->family == CHIP_KAVERI) {
5151                 u32 tmp = RREG32(CHUB_CONTROL);
5152                 tmp &= ~BYPASS_VM;
5153                 WREG32(CHUB_CONTROL, tmp);
5154         }
5155
5156         /* XXX SH_MEM regs */
5157         /* where to put LDS, scratch, GPUVM in FSA64 space */
5158         mutex_lock(&rdev->srbm_mutex);
5159         for (i = 0; i < 16; i++) {
5160                 cik_srbm_select(rdev, 0, 0, 0, i);
5161                 /* CP and shaders */
5162                 WREG32(SH_MEM_CONFIG, 0);
5163                 WREG32(SH_MEM_APE1_BASE, 1);
5164                 WREG32(SH_MEM_APE1_LIMIT, 0);
5165                 WREG32(SH_MEM_BASES, 0);
5166                 /* SDMA GFX */
5167                 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5168                 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5169                 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5170                 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5171                 /* XXX SDMA RLC - todo */
5172         }
5173         cik_srbm_select(rdev, 0, 0, 0, 0);
5174         mutex_unlock(&rdev->srbm_mutex);
5175
5176         cik_pcie_gart_tlb_flush(rdev);
5177         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5178                  (unsigned)(rdev->mc.gtt_size >> 20),
5179                  (unsigned long long)rdev->gart.table_addr);
5180         rdev->gart.ready = true;
5181         return 0;
5182 }
5183
5184 /**
5185  * cik_pcie_gart_disable - gart disable
5186  *
5187  * @rdev: radeon_device pointer
5188  *
5189  * This disables all VM page table (CIK).
5190  */
5191 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5192 {
5193         /* Disable all tables */
5194         WREG32(VM_CONTEXT0_CNTL, 0);
5195         WREG32(VM_CONTEXT1_CNTL, 0);
5196         /* Setup TLB control */
5197         WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5198                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5199         /* Setup L2 cache */
5200         WREG32(VM_L2_CNTL,
5201                ENABLE_L2_FRAGMENT_PROCESSING |
5202                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5203                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5204                EFFECTIVE_L2_QUEUE_SIZE(7) |
5205                CONTEXT1_IDENTITY_ACCESS_MODE(1));
5206         WREG32(VM_L2_CNTL2, 0);
5207         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5208                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5209         radeon_gart_table_vram_unpin(rdev);
5210 }
5211
5212 /**
5213  * cik_pcie_gart_fini - vm fini callback
5214  *
5215  * @rdev: radeon_device pointer
5216  *
5217  * Tears down the driver GART/VM setup (CIK).
5218  */
5219 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5220 {
5221         cik_pcie_gart_disable(rdev);
5222         radeon_gart_table_vram_free(rdev);
5223         radeon_gart_fini(rdev);
5224 }
5225
5226 /* vm parser */
5227 /**
5228  * cik_ib_parse - vm ib_parse callback
5229  *
5230  * @rdev: radeon_device pointer
5231  * @ib: indirect buffer pointer
5232  *
5233  * CIK uses hw IB checking so this is a nop (CIK).
5234  */
5235 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5236 {
5237         return 0;
5238 }
5239
5240 /*
5241  * vm
5242  * VMID 0 is the physical GPU addresses as used by the kernel.
5243  * VMIDs 1-15 are used for userspace clients and are handled
5244  * by the radeon vm/hsa code.
5245  */
5246 /**
5247  * cik_vm_init - cik vm init callback
5248  *
5249  * @rdev: radeon_device pointer
5250  *
5251  * Inits cik specific vm parameters (number of VMs, base of vram for
5252  * VMIDs 1-15) (CIK).
5253  * Returns 0 for success.
5254  */
5255 int cik_vm_init(struct radeon_device *rdev)
5256 {
5257         /* number of VMs */
5258         rdev->vm_manager.nvm = 16;
5259         /* base offset of vram pages */
5260         if (rdev->flags & RADEON_IS_IGP) {
5261                 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5262                 tmp <<= 22;
5263                 rdev->vm_manager.vram_base_offset = tmp;
5264         } else
5265                 rdev->vm_manager.vram_base_offset = 0;
5266
5267         return 0;
5268 }
5269
5270 /**
5271  * cik_vm_fini - cik vm fini callback
5272  *
5273  * @rdev: radeon_device pointer
5274  *
5275  * Tear down any asic specific VM setup (CIK).
5276  */
5277 void cik_vm_fini(struct radeon_device *rdev)
5278 {
5279 }
5280
5281 /**
5282  * cik_vm_decode_fault - print human readable fault info
5283  *
5284  * @rdev: radeon_device pointer
5285  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5286  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5287  *
5288  * Print human readable fault information (CIK).
5289  */
5290 static void cik_vm_decode_fault(struct radeon_device *rdev,
5291                                 u32 status, u32 addr, u32 mc_client)
5292 {
5293         u32 mc_id;
5294         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5295         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5296         char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5297                 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5298
5299         if (rdev->family == CHIP_HAWAII)
5300                 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5301         else
5302                 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5303
5304         printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5305                protections, vmid, addr,
5306                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5307                block, mc_client, mc_id);
5308 }
5309
5310 /**
5311  * cik_vm_flush - cik vm flush using the CP
5312  *
5313  * @rdev: radeon_device pointer
5314  *
5315  * Update the page table base and flush the VM TLB
5316  * using the CP (CIK).
5317  */
5318 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5319 {
5320         struct radeon_ring *ring = &rdev->ring[ridx];
5321
5322         if (vm == NULL)
5323                 return;
5324
5325         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5326         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5327                                  WRITE_DATA_DST_SEL(0)));
5328         if (vm->id < 8) {
5329                 radeon_ring_write(ring,
5330                                   (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5331         } else {
5332                 radeon_ring_write(ring,
5333                                   (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5334         }
5335         radeon_ring_write(ring, 0);
5336         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5337
5338         /* update SH_MEM_* regs */
5339         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5340         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5341                                  WRITE_DATA_DST_SEL(0)));
5342         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5343         radeon_ring_write(ring, 0);
5344         radeon_ring_write(ring, VMID(vm->id));
5345
5346         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5347         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5348                                  WRITE_DATA_DST_SEL(0)));
5349         radeon_ring_write(ring, SH_MEM_BASES >> 2);
5350         radeon_ring_write(ring, 0);
5351
5352         radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5353         radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5354         radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5355         radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5356
5357         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5358         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5359                                  WRITE_DATA_DST_SEL(0)));
5360         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5361         radeon_ring_write(ring, 0);
5362         radeon_ring_write(ring, VMID(0));
5363
5364         /* HDP flush */
5365         /* We should be using the WAIT_REG_MEM packet here like in
5366          * cik_fence_ring_emit(), but it causes the CP to hang in this
5367          * context...
5368          */
5369         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5370         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5371                                  WRITE_DATA_DST_SEL(0)));
5372         radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5373         radeon_ring_write(ring, 0);
5374         radeon_ring_write(ring, 0);
5375
5376         /* bits 0-15 are the VM contexts0-15 */
5377         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5378         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5379                                  WRITE_DATA_DST_SEL(0)));
5380         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5381         radeon_ring_write(ring, 0);
5382         radeon_ring_write(ring, 1 << vm->id);
5383
5384         /* compute doesn't have PFP */
5385         if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
5386                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5387                 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5388                 radeon_ring_write(ring, 0x0);
5389         }
5390 }
5391
5392 /*
5393  * RLC
5394  * The RLC is a multi-purpose microengine that handles a
5395  * variety of functions, the most important of which is
5396  * the interrupt controller.
5397  */
5398 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5399                                           bool enable)
5400 {
5401         u32 tmp = RREG32(CP_INT_CNTL_RING0);
5402
5403         if (enable)
5404                 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5405         else
5406                 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5407         WREG32(CP_INT_CNTL_RING0, tmp);
5408 }
5409
5410 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5411 {
5412         u32 tmp;
5413
5414         tmp = RREG32(RLC_LB_CNTL);
5415         if (enable)
5416                 tmp |= LOAD_BALANCE_ENABLE;
5417         else
5418                 tmp &= ~LOAD_BALANCE_ENABLE;
5419         WREG32(RLC_LB_CNTL, tmp);
5420 }
5421
5422 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5423 {
5424         u32 i, j, k;
5425         u32 mask;
5426
5427         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5428                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5429                         cik_select_se_sh(rdev, i, j);
5430                         for (k = 0; k < rdev->usec_timeout; k++) {
5431                                 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5432                                         break;
5433                                 udelay(1);
5434                         }
5435                 }
5436         }
5437         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5438
5439         mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5440         for (k = 0; k < rdev->usec_timeout; k++) {
5441                 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5442                         break;
5443                 udelay(1);
5444         }
5445 }
5446
5447 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5448 {
5449         u32 tmp;
5450
5451         tmp = RREG32(RLC_CNTL);
5452         if (tmp != rlc)
5453                 WREG32(RLC_CNTL, rlc);
5454 }
5455
5456 static u32 cik_halt_rlc(struct radeon_device *rdev)
5457 {
5458         u32 data, orig;
5459
5460         orig = data = RREG32(RLC_CNTL);
5461
5462         if (data & RLC_ENABLE) {
5463                 u32 i;
5464
5465                 data &= ~RLC_ENABLE;
5466                 WREG32(RLC_CNTL, data);
5467
5468                 for (i = 0; i < rdev->usec_timeout; i++) {
5469                         if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5470                                 break;
5471                         udelay(1);
5472                 }
5473
5474                 cik_wait_for_rlc_serdes(rdev);
5475         }
5476
5477         return orig;
5478 }
5479
5480 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5481 {
5482         u32 tmp, i, mask;
5483
5484         tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5485         WREG32(RLC_GPR_REG2, tmp);
5486
5487         mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5488         for (i = 0; i < rdev->usec_timeout; i++) {
5489                 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5490                         break;
5491                 udelay(1);
5492         }
5493
5494         for (i = 0; i < rdev->usec_timeout; i++) {
5495                 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5496                         break;
5497                 udelay(1);
5498         }
5499 }
5500
5501 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5502 {
5503         u32 tmp;
5504
5505         tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5506         WREG32(RLC_GPR_REG2, tmp);
5507 }
5508
5509 /**
5510  * cik_rlc_stop - stop the RLC ME
5511  *
5512  * @rdev: radeon_device pointer
5513  *
5514  * Halt the RLC ME (MicroEngine) (CIK).
5515  */
5516 static void cik_rlc_stop(struct radeon_device *rdev)
5517 {
5518         WREG32(RLC_CNTL, 0);
5519
5520         cik_enable_gui_idle_interrupt(rdev, false);
5521
5522         cik_wait_for_rlc_serdes(rdev);
5523 }
5524
5525 /**
5526  * cik_rlc_start - start the RLC ME
5527  *
5528  * @rdev: radeon_device pointer
5529  *
5530  * Unhalt the RLC ME (MicroEngine) (CIK).
5531  */
5532 static void cik_rlc_start(struct radeon_device *rdev)
5533 {
5534         WREG32(RLC_CNTL, RLC_ENABLE);
5535
5536         cik_enable_gui_idle_interrupt(rdev, true);
5537
5538         udelay(50);
5539 }
5540
5541 /**
5542  * cik_rlc_resume - setup the RLC hw
5543  *
5544  * @rdev: radeon_device pointer
5545  *
5546  * Initialize the RLC registers, load the ucode,
5547  * and start the RLC (CIK).
5548  * Returns 0 for success, -EINVAL if the ucode is not available.
5549  */
5550 static int cik_rlc_resume(struct radeon_device *rdev)
5551 {
5552         u32 i, size, tmp;
5553         const __be32 *fw_data;
5554
5555         if (!rdev->rlc_fw)
5556                 return -EINVAL;
5557
5558         switch (rdev->family) {
5559         case CHIP_BONAIRE:
5560         case CHIP_HAWAII:
5561         default:
5562                 size = BONAIRE_RLC_UCODE_SIZE;
5563                 break;
5564         case CHIP_KAVERI:
5565                 size = KV_RLC_UCODE_SIZE;
5566                 break;
5567         case CHIP_KABINI:
5568                 size = KB_RLC_UCODE_SIZE;
5569                 break;
5570         }
5571
5572         cik_rlc_stop(rdev);
5573
5574         /* disable CG */
5575         tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5576         WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5577
5578         si_rlc_reset(rdev);
5579
5580         cik_init_pg(rdev);
5581
5582         cik_init_cg(rdev);
5583
5584         WREG32(RLC_LB_CNTR_INIT, 0);
5585         WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5586
5587         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5588         WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5589         WREG32(RLC_LB_PARAMS, 0x00600408);
5590         WREG32(RLC_LB_CNTL, 0x80000004);
5591
5592         WREG32(RLC_MC_CNTL, 0);
5593         WREG32(RLC_UCODE_CNTL, 0);
5594
5595         fw_data = (const __be32 *)rdev->rlc_fw->data;
5596                 WREG32(RLC_GPM_UCODE_ADDR, 0);
5597         for (i = 0; i < size; i++)
5598                 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5599         WREG32(RLC_GPM_UCODE_ADDR, 0);
5600
5601         /* XXX - find out what chips support lbpw */
5602         cik_enable_lbpw(rdev, false);
5603
5604         if (rdev->family == CHIP_BONAIRE)
5605                 WREG32(RLC_DRIVER_DMA_STATUS, 0);
5606
5607         cik_rlc_start(rdev);
5608
5609         return 0;
5610 }
5611
5612 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5613 {
5614         u32 data, orig, tmp, tmp2;
5615
5616         orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5617
5618         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5619                 cik_enable_gui_idle_interrupt(rdev, true);
5620
5621                 tmp = cik_halt_rlc(rdev);
5622
5623                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5624                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5625                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5626                 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5627                 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5628
5629                 cik_update_rlc(rdev, tmp);
5630
5631                 data |= CGCG_EN | CGLS_EN;
5632         } else {
5633                 cik_enable_gui_idle_interrupt(rdev, false);
5634
5635                 RREG32(CB_CGTT_SCLK_CTRL);
5636                 RREG32(CB_CGTT_SCLK_CTRL);
5637                 RREG32(CB_CGTT_SCLK_CTRL);
5638                 RREG32(CB_CGTT_SCLK_CTRL);
5639
5640                 data &= ~(CGCG_EN | CGLS_EN);
5641         }
5642
5643         if (orig != data)
5644                 WREG32(RLC_CGCG_CGLS_CTRL, data);
5645
5646 }
5647
5648 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5649 {
5650         u32 data, orig, tmp = 0;
5651
5652         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5653                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5654                         if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5655                                 orig = data = RREG32(CP_MEM_SLP_CNTL);
5656                                 data |= CP_MEM_LS_EN;
5657                                 if (orig != data)
5658                                         WREG32(CP_MEM_SLP_CNTL, data);
5659                         }
5660                 }
5661
5662                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5663                 data &= 0xfffffffd;
5664                 if (orig != data)
5665                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5666
5667                 tmp = cik_halt_rlc(rdev);
5668
5669                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5670                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5671                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5672                 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5673                 WREG32(RLC_SERDES_WR_CTRL, data);
5674
5675                 cik_update_rlc(rdev, tmp);
5676
5677                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5678                         orig = data = RREG32(CGTS_SM_CTRL_REG);
5679                         data &= ~SM_MODE_MASK;
5680                         data |= SM_MODE(0x2);
5681                         data |= SM_MODE_ENABLE;
5682                         data &= ~CGTS_OVERRIDE;
5683                         if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5684                             (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5685                                 data &= ~CGTS_LS_OVERRIDE;
5686                         data &= ~ON_MONITOR_ADD_MASK;
5687                         data |= ON_MONITOR_ADD_EN;
5688                         data |= ON_MONITOR_ADD(0x96);
5689                         if (orig != data)
5690                                 WREG32(CGTS_SM_CTRL_REG, data);
5691                 }
5692         } else {
5693                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5694                 data |= 0x00000002;
5695                 if (orig != data)
5696                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5697
5698                 data = RREG32(RLC_MEM_SLP_CNTL);
5699                 if (data & RLC_MEM_LS_EN) {
5700                         data &= ~RLC_MEM_LS_EN;
5701                         WREG32(RLC_MEM_SLP_CNTL, data);
5702                 }
5703
5704                 data = RREG32(CP_MEM_SLP_CNTL);
5705                 if (data & CP_MEM_LS_EN) {
5706                         data &= ~CP_MEM_LS_EN;
5707                         WREG32(CP_MEM_SLP_CNTL, data);
5708                 }
5709
5710                 orig = data = RREG32(CGTS_SM_CTRL_REG);
5711                 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5712                 if (orig != data)
5713                         WREG32(CGTS_SM_CTRL_REG, data);
5714
5715                 tmp = cik_halt_rlc(rdev);
5716
5717                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5718                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5719                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5720                 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5721                 WREG32(RLC_SERDES_WR_CTRL, data);
5722
5723                 cik_update_rlc(rdev, tmp);
5724         }
5725 }
5726
5727 static const u32 mc_cg_registers[] =
5728 {
5729         MC_HUB_MISC_HUB_CG,
5730         MC_HUB_MISC_SIP_CG,
5731         MC_HUB_MISC_VM_CG,
5732         MC_XPB_CLK_GAT,
5733         ATC_MISC_CG,
5734         MC_CITF_MISC_WR_CG,
5735         MC_CITF_MISC_RD_CG,
5736         MC_CITF_MISC_VM_CG,
5737         VM_L2_CG,
5738 };
5739
5740 static void cik_enable_mc_ls(struct radeon_device *rdev,
5741                              bool enable)
5742 {
5743         int i;
5744         u32 orig, data;
5745
5746         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5747                 orig = data = RREG32(mc_cg_registers[i]);
5748                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5749                         data |= MC_LS_ENABLE;
5750                 else
5751                         data &= ~MC_LS_ENABLE;
5752                 if (data != orig)
5753                         WREG32(mc_cg_registers[i], data);
5754         }
5755 }
5756
5757 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5758                                bool enable)
5759 {
5760         int i;
5761         u32 orig, data;
5762
5763         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5764                 orig = data = RREG32(mc_cg_registers[i]);
5765                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5766                         data |= MC_CG_ENABLE;
5767                 else
5768                         data &= ~MC_CG_ENABLE;
5769                 if (data != orig)
5770                         WREG32(mc_cg_registers[i], data);
5771         }
5772 }
5773
5774 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
5775                                  bool enable)
5776 {
5777         u32 orig, data;
5778
5779         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5780                 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
5781                 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
5782         } else {
5783                 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
5784                 data |= 0xff000000;
5785                 if (data != orig)
5786                         WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
5787
5788                 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
5789                 data |= 0xff000000;
5790                 if (data != orig)
5791                         WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
5792         }
5793 }
5794
5795 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
5796                                  bool enable)
5797 {
5798         u32 orig, data;
5799
5800         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
5801                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5802                 data |= 0x100;
5803                 if (orig != data)
5804                         WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5805
5806                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5807                 data |= 0x100;
5808                 if (orig != data)
5809                         WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5810         } else {
5811                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5812                 data &= ~0x100;
5813                 if (orig != data)
5814                         WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5815
5816                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5817                 data &= ~0x100;
5818                 if (orig != data)
5819                         WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5820         }
5821 }
5822
5823 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
5824                                 bool enable)
5825 {
5826         u32 orig, data;
5827
5828         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5829                 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5830                 data = 0xfff;
5831                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
5832
5833                 orig = data = RREG32(UVD_CGC_CTRL);
5834                 data |= DCM;
5835                 if (orig != data)
5836                         WREG32(UVD_CGC_CTRL, data);
5837         } else {
5838                 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5839                 data &= ~0xfff;
5840                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
5841
5842                 orig = data = RREG32(UVD_CGC_CTRL);
5843                 data &= ~DCM;
5844                 if (orig != data)
5845                         WREG32(UVD_CGC_CTRL, data);
5846         }
5847 }
5848
5849 static void cik_enable_bif_mgls(struct radeon_device *rdev,
5850                                bool enable)
5851 {
5852         u32 orig, data;
5853
5854         orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
5855
5856         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5857                 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5858                         REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5859         else
5860                 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5861                           REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5862
5863         if (orig != data)
5864                 WREG32_PCIE_PORT(PCIE_CNTL2, data);
5865 }
5866
5867 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
5868                                 bool enable)
5869 {
5870         u32 orig, data;
5871
5872         orig = data = RREG32(HDP_HOST_PATH_CNTL);
5873
5874         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5875                 data &= ~CLOCK_GATING_DIS;
5876         else
5877                 data |= CLOCK_GATING_DIS;
5878
5879         if (orig != data)
5880                 WREG32(HDP_HOST_PATH_CNTL, data);
5881 }
5882
5883 static void cik_enable_hdp_ls(struct radeon_device *rdev,
5884                               bool enable)
5885 {
5886         u32 orig, data;
5887
5888         orig = data = RREG32(HDP_MEM_POWER_LS);
5889
5890         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5891                 data |= HDP_LS_ENABLE;
5892         else
5893                 data &= ~HDP_LS_ENABLE;
5894
5895         if (orig != data)
5896                 WREG32(HDP_MEM_POWER_LS, data);
5897 }
5898
5899 void cik_update_cg(struct radeon_device *rdev,
5900                    u32 block, bool enable)
5901 {
5902
5903         if (block & RADEON_CG_BLOCK_GFX) {
5904                 cik_enable_gui_idle_interrupt(rdev, false);
5905                 /* order matters! */
5906                 if (enable) {
5907                         cik_enable_mgcg(rdev, true);
5908                         cik_enable_cgcg(rdev, true);
5909                 } else {
5910                         cik_enable_cgcg(rdev, false);
5911                         cik_enable_mgcg(rdev, false);
5912                 }
5913                 cik_enable_gui_idle_interrupt(rdev, true);
5914         }
5915
5916         if (block & RADEON_CG_BLOCK_MC) {
5917                 if (!(rdev->flags & RADEON_IS_IGP)) {
5918                         cik_enable_mc_mgcg(rdev, enable);
5919                         cik_enable_mc_ls(rdev, enable);
5920                 }
5921         }
5922
5923         if (block & RADEON_CG_BLOCK_SDMA) {
5924                 cik_enable_sdma_mgcg(rdev, enable);
5925                 cik_enable_sdma_mgls(rdev, enable);
5926         }
5927
5928         if (block & RADEON_CG_BLOCK_BIF) {
5929                 cik_enable_bif_mgls(rdev, enable);
5930         }
5931
5932         if (block & RADEON_CG_BLOCK_UVD) {
5933                 if (rdev->has_uvd)
5934                         cik_enable_uvd_mgcg(rdev, enable);
5935         }
5936
5937         if (block & RADEON_CG_BLOCK_HDP) {
5938                 cik_enable_hdp_mgcg(rdev, enable);
5939                 cik_enable_hdp_ls(rdev, enable);
5940         }
5941 }
5942
5943 static void cik_init_cg(struct radeon_device *rdev)
5944 {
5945
5946         cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
5947
5948         if (rdev->has_uvd)
5949                 si_init_uvd_internal_cg(rdev);
5950
5951         cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5952                              RADEON_CG_BLOCK_SDMA |
5953                              RADEON_CG_BLOCK_BIF |
5954                              RADEON_CG_BLOCK_UVD |
5955                              RADEON_CG_BLOCK_HDP), true);
5956 }
5957
5958 static void cik_fini_cg(struct radeon_device *rdev)
5959 {
5960         cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5961                              RADEON_CG_BLOCK_SDMA |
5962                              RADEON_CG_BLOCK_BIF |
5963                              RADEON_CG_BLOCK_UVD |
5964                              RADEON_CG_BLOCK_HDP), false);
5965
5966         cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
5967 }
5968
5969 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
5970                                           bool enable)
5971 {
5972         u32 data, orig;
5973
5974         orig = data = RREG32(RLC_PG_CNTL);
5975         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
5976                 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5977         else
5978                 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5979         if (orig != data)
5980                 WREG32(RLC_PG_CNTL, data);
5981 }
5982
5983 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
5984                                           bool enable)
5985 {
5986         u32 data, orig;
5987
5988         orig = data = RREG32(RLC_PG_CNTL);
5989         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
5990                 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5991         else
5992                 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5993         if (orig != data)
5994                 WREG32(RLC_PG_CNTL, data);
5995 }
5996
5997 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
5998 {
5999         u32 data, orig;
6000
6001         orig = data = RREG32(RLC_PG_CNTL);
6002         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6003                 data &= ~DISABLE_CP_PG;
6004         else
6005                 data |= DISABLE_CP_PG;
6006         if (orig != data)
6007                 WREG32(RLC_PG_CNTL, data);
6008 }
6009
6010 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6011 {
6012         u32 data, orig;
6013
6014         orig = data = RREG32(RLC_PG_CNTL);
6015         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6016                 data &= ~DISABLE_GDS_PG;
6017         else
6018                 data |= DISABLE_GDS_PG;
6019         if (orig != data)
6020                 WREG32(RLC_PG_CNTL, data);
6021 }
6022
6023 #define CP_ME_TABLE_SIZE    96
6024 #define CP_ME_TABLE_OFFSET  2048
6025 #define CP_MEC_TABLE_OFFSET 4096
6026
6027 void cik_init_cp_pg_table(struct radeon_device *rdev)
6028 {
6029         const __be32 *fw_data;
6030         volatile u32 *dst_ptr;
6031         int me, i, max_me = 4;
6032         u32 bo_offset = 0;
6033         u32 table_offset;
6034
6035         if (rdev->family == CHIP_KAVERI)
6036                 max_me = 5;
6037
6038         if (rdev->rlc.cp_table_ptr == NULL)
6039                 return;
6040
6041         /* write the cp table buffer */
6042         dst_ptr = rdev->rlc.cp_table_ptr;
6043         for (me = 0; me < max_me; me++) {
6044                 if (me == 0) {
6045                         fw_data = (const __be32 *)rdev->ce_fw->data;
6046                         table_offset = CP_ME_TABLE_OFFSET;
6047                 } else if (me == 1) {
6048                         fw_data = (const __be32 *)rdev->pfp_fw->data;
6049                         table_offset = CP_ME_TABLE_OFFSET;
6050                 } else if (me == 2) {
6051                         fw_data = (const __be32 *)rdev->me_fw->data;
6052                         table_offset = CP_ME_TABLE_OFFSET;
6053                 } else {
6054                         fw_data = (const __be32 *)rdev->mec_fw->data;
6055                         table_offset = CP_MEC_TABLE_OFFSET;
6056                 }
6057
6058                 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
6059                         dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6060                 }
6061                 bo_offset += CP_ME_TABLE_SIZE;
6062         }
6063 }
6064
6065 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6066                                 bool enable)
6067 {
6068         u32 data, orig;
6069
6070         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6071                 orig = data = RREG32(RLC_PG_CNTL);
6072                 data |= GFX_PG_ENABLE;
6073                 if (orig != data)
6074                         WREG32(RLC_PG_CNTL, data);
6075
6076                 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6077                 data |= AUTO_PG_EN;
6078                 if (orig != data)
6079                         WREG32(RLC_AUTO_PG_CTRL, data);
6080         } else {
6081                 orig = data = RREG32(RLC_PG_CNTL);
6082                 data &= ~GFX_PG_ENABLE;
6083                 if (orig != data)
6084                         WREG32(RLC_PG_CNTL, data);
6085
6086                 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6087                 data &= ~AUTO_PG_EN;
6088                 if (orig != data)
6089                         WREG32(RLC_AUTO_PG_CTRL, data);
6090
6091                 data = RREG32(DB_RENDER_CONTROL);
6092         }
6093 }
6094
6095 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6096 {
6097         u32 mask = 0, tmp, tmp1;
6098         int i;
6099
6100         cik_select_se_sh(rdev, se, sh);
6101         tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6102         tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6103         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6104
6105         tmp &= 0xffff0000;
6106
6107         tmp |= tmp1;
6108         tmp >>= 16;
6109
6110         for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6111                 mask <<= 1;
6112                 mask |= 1;
6113         }
6114
6115         return (~tmp) & mask;
6116 }
6117
6118 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6119 {
6120         u32 i, j, k, active_cu_number = 0;
6121         u32 mask, counter, cu_bitmap;
6122         u32 tmp = 0;
6123
6124         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6125                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6126                         mask = 1;
6127                         cu_bitmap = 0;
6128                         counter = 0;
6129                         for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6130                                 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6131                                         if (counter < 2)
6132                                                 cu_bitmap |= mask;
6133                                         counter ++;
6134                                 }
6135                                 mask <<= 1;
6136                         }
6137
6138                         active_cu_number += counter;
6139                         tmp |= (cu_bitmap << (i * 16 + j * 8));
6140                 }
6141         }
6142
6143         WREG32(RLC_PG_AO_CU_MASK, tmp);
6144
6145         tmp = RREG32(RLC_MAX_PG_CU);
6146         tmp &= ~MAX_PU_CU_MASK;
6147         tmp |= MAX_PU_CU(active_cu_number);
6148         WREG32(RLC_MAX_PG_CU, tmp);
6149 }
6150
6151 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6152                                        bool enable)
6153 {
6154         u32 data, orig;
6155
6156         orig = data = RREG32(RLC_PG_CNTL);
6157         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6158                 data |= STATIC_PER_CU_PG_ENABLE;
6159         else
6160                 data &= ~STATIC_PER_CU_PG_ENABLE;
6161         if (orig != data)
6162                 WREG32(RLC_PG_CNTL, data);
6163 }
6164
6165 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6166                                         bool enable)
6167 {
6168         u32 data, orig;
6169
6170         orig = data = RREG32(RLC_PG_CNTL);
6171         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6172                 data |= DYN_PER_CU_PG_ENABLE;
6173         else
6174                 data &= ~DYN_PER_CU_PG_ENABLE;
6175         if (orig != data)
6176                 WREG32(RLC_PG_CNTL, data);
6177 }
6178
6179 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6180 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET    0x3D
6181
6182 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6183 {
6184         u32 data, orig;
6185         u32 i;
6186
6187         if (rdev->rlc.cs_data) {
6188                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6189                 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6190                 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6191                 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6192         } else {
6193                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6194                 for (i = 0; i < 3; i++)
6195                         WREG32(RLC_GPM_SCRATCH_DATA, 0);
6196         }
6197         if (rdev->rlc.reg_list) {
6198                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6199                 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6200                         WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6201         }
6202
6203         orig = data = RREG32(RLC_PG_CNTL);
6204         data |= GFX_PG_SRC;
6205         if (orig != data)
6206                 WREG32(RLC_PG_CNTL, data);
6207
6208         WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6209         WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6210
6211         data = RREG32(CP_RB_WPTR_POLL_CNTL);
6212         data &= ~IDLE_POLL_COUNT_MASK;
6213         data |= IDLE_POLL_COUNT(0x60);
6214         WREG32(CP_RB_WPTR_POLL_CNTL, data);
6215
6216         data = 0x10101010;
6217         WREG32(RLC_PG_DELAY, data);
6218
6219         data = RREG32(RLC_PG_DELAY_2);
6220         data &= ~0xff;
6221         data |= 0x3;
6222         WREG32(RLC_PG_DELAY_2, data);
6223
6224         data = RREG32(RLC_AUTO_PG_CTRL);
6225         data &= ~GRBM_REG_SGIT_MASK;
6226         data |= GRBM_REG_SGIT(0x700);
6227         WREG32(RLC_AUTO_PG_CTRL, data);
6228
6229 }
6230
6231 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6232 {
6233         cik_enable_gfx_cgpg(rdev, enable);
6234         cik_enable_gfx_static_mgpg(rdev, enable);
6235         cik_enable_gfx_dynamic_mgpg(rdev, enable);
6236 }
6237
6238 u32 cik_get_csb_size(struct radeon_device *rdev)
6239 {
6240         u32 count = 0;
6241         const struct cs_section_def *sect = NULL;
6242         const struct cs_extent_def *ext = NULL;
6243
6244         if (rdev->rlc.cs_data == NULL)
6245                 return 0;
6246
6247         /* begin clear state */
6248         count += 2;
6249         /* context control state */
6250         count += 3;
6251
6252         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6253                 for (ext = sect->section; ext->extent != NULL; ++ext) {
6254                         if (sect->id == SECT_CONTEXT)
6255                                 count += 2 + ext->reg_count;
6256                         else
6257                                 return 0;
6258                 }
6259         }
6260         /* pa_sc_raster_config/pa_sc_raster_config1 */
6261         count += 4;
6262         /* end clear state */
6263         count += 2;
6264         /* clear state */
6265         count += 2;
6266
6267         return count;
6268 }
6269
6270 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6271 {
6272         u32 count = 0, i;
6273         const struct cs_section_def *sect = NULL;
6274         const struct cs_extent_def *ext = NULL;
6275
6276         if (rdev->rlc.cs_data == NULL)
6277                 return;
6278         if (buffer == NULL)
6279                 return;
6280
6281         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6282         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6283
6284         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6285         buffer[count++] = cpu_to_le32(0x80000000);
6286         buffer[count++] = cpu_to_le32(0x80000000);
6287
6288         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6289                 for (ext = sect->section; ext->extent != NULL; ++ext) {
6290                         if (sect->id == SECT_CONTEXT) {
6291                                 buffer[count++] =
6292                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6293                                 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
6294                                 for (i = 0; i < ext->reg_count; i++)
6295                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
6296                         } else {
6297                                 return;
6298                         }
6299                 }
6300         }
6301
6302         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6303         buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
6304         switch (rdev->family) {
6305         case CHIP_BONAIRE:
6306                 buffer[count++] = cpu_to_le32(0x16000012);
6307                 buffer[count++] = cpu_to_le32(0x00000000);
6308                 break;
6309         case CHIP_KAVERI:
6310                 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6311                 buffer[count++] = cpu_to_le32(0x00000000);
6312                 break;
6313         case CHIP_KABINI:
6314                 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6315                 buffer[count++] = cpu_to_le32(0x00000000);
6316                 break;
6317         case CHIP_HAWAII:
6318                 buffer[count++] = 0x3a00161a;
6319                 buffer[count++] = 0x0000002e;
6320                 break;
6321         default:
6322                 buffer[count++] = cpu_to_le32(0x00000000);
6323                 buffer[count++] = cpu_to_le32(0x00000000);
6324                 break;
6325         }
6326
6327         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6328         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
6329
6330         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6331         buffer[count++] = cpu_to_le32(0);
6332 }
6333
6334 static void cik_init_pg(struct radeon_device *rdev)
6335 {
6336         if (rdev->pg_flags) {
6337                 cik_enable_sck_slowdown_on_pu(rdev, true);
6338                 cik_enable_sck_slowdown_on_pd(rdev, true);
6339                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6340                         cik_init_gfx_cgpg(rdev);
6341                         cik_enable_cp_pg(rdev, true);
6342                         cik_enable_gds_pg(rdev, true);
6343                 }
6344                 cik_init_ao_cu_mask(rdev);
6345                 cik_update_gfx_pg(rdev, true);
6346         }
6347 }
6348
6349 static void cik_fini_pg(struct radeon_device *rdev)
6350 {
6351         if (rdev->pg_flags) {
6352                 cik_update_gfx_pg(rdev, false);
6353                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6354                         cik_enable_cp_pg(rdev, false);
6355                         cik_enable_gds_pg(rdev, false);
6356                 }
6357         }
6358 }
6359
6360 /*
6361  * Interrupts
6362  * Starting with r6xx, interrupts are handled via a ring buffer.
6363  * Ring buffers are areas of GPU accessible memory that the GPU
6364  * writes interrupt vectors into and the host reads vectors out of.
6365  * There is a rptr (read pointer) that determines where the
6366  * host is currently reading, and a wptr (write pointer)
6367  * which determines where the GPU has written.  When the
6368  * pointers are equal, the ring is idle.  When the GPU
6369  * writes vectors to the ring buffer, it increments the
6370  * wptr.  When there is an interrupt, the host then starts
6371  * fetching commands and processing them until the pointers are
6372  * equal again at which point it updates the rptr.
6373  */
6374
6375 /**
6376  * cik_enable_interrupts - Enable the interrupt ring buffer
6377  *
6378  * @rdev: radeon_device pointer
6379  *
6380  * Enable the interrupt ring buffer (CIK).
6381  */
6382 static void cik_enable_interrupts(struct radeon_device *rdev)
6383 {
6384         u32 ih_cntl = RREG32(IH_CNTL);
6385         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6386
6387         ih_cntl |= ENABLE_INTR;
6388         ih_rb_cntl |= IH_RB_ENABLE;
6389         WREG32(IH_CNTL, ih_cntl);
6390         WREG32(IH_RB_CNTL, ih_rb_cntl);
6391         rdev->ih.enabled = true;
6392 }
6393
6394 /**
6395  * cik_disable_interrupts - Disable the interrupt ring buffer
6396  *
6397  * @rdev: radeon_device pointer
6398  *
6399  * Disable the interrupt ring buffer (CIK).
6400  */
6401 static void cik_disable_interrupts(struct radeon_device *rdev)
6402 {
6403         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6404         u32 ih_cntl = RREG32(IH_CNTL);
6405
6406         ih_rb_cntl &= ~IH_RB_ENABLE;
6407         ih_cntl &= ~ENABLE_INTR;
6408         WREG32(IH_RB_CNTL, ih_rb_cntl);
6409         WREG32(IH_CNTL, ih_cntl);
6410         /* set rptr, wptr to 0 */
6411         WREG32(IH_RB_RPTR, 0);
6412         WREG32(IH_RB_WPTR, 0);
6413         rdev->ih.enabled = false;
6414         rdev->ih.rptr = 0;
6415 }
6416
6417 /**
6418  * cik_disable_interrupt_state - Disable all interrupt sources
6419  *
6420  * @rdev: radeon_device pointer
6421  *
6422  * Clear all interrupt enable bits used by the driver (CIK).
6423  */
6424 static void cik_disable_interrupt_state(struct radeon_device *rdev)
6425 {
6426         u32 tmp;
6427
6428         /* gfx ring */
6429         tmp = RREG32(CP_INT_CNTL_RING0) &
6430                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6431         WREG32(CP_INT_CNTL_RING0, tmp);
6432         /* sdma */
6433         tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6434         WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6435         tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6436         WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
6437         /* compute queues */
6438         WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6439         WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6440         WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6441         WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6442         WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6443         WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6444         WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6445         WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6446         /* grbm */
6447         WREG32(GRBM_INT_CNTL, 0);
6448         /* vline/vblank, etc. */
6449         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6450         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6451         if (rdev->num_crtc >= 4) {
6452                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6453                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6454         }
6455         if (rdev->num_crtc >= 6) {
6456                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6457                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6458         }
6459
6460         /* dac hotplug */
6461         WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6462
6463         /* digital hotplug */
6464         tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6465         WREG32(DC_HPD1_INT_CONTROL, tmp);
6466         tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6467         WREG32(DC_HPD2_INT_CONTROL, tmp);
6468         tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6469         WREG32(DC_HPD3_INT_CONTROL, tmp);
6470         tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6471         WREG32(DC_HPD4_INT_CONTROL, tmp);
6472         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6473         WREG32(DC_HPD5_INT_CONTROL, tmp);
6474         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6475         WREG32(DC_HPD6_INT_CONTROL, tmp);
6476
6477 }
6478
6479 /**
6480  * cik_irq_init - init and enable the interrupt ring
6481  *
6482  * @rdev: radeon_device pointer
6483  *
6484  * Allocate a ring buffer for the interrupt controller,
6485  * enable the RLC, disable interrupts, enable the IH
6486  * ring buffer and enable it (CIK).
6487  * Called at device load and reume.
6488  * Returns 0 for success, errors for failure.
6489  */
6490 static int cik_irq_init(struct radeon_device *rdev)
6491 {
6492         int ret = 0;
6493         int rb_bufsz;
6494         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6495
6496         /* allocate ring */
6497         ret = r600_ih_ring_alloc(rdev);
6498         if (ret)
6499                 return ret;
6500
6501         /* disable irqs */
6502         cik_disable_interrupts(rdev);
6503
6504         /* init rlc */
6505         ret = cik_rlc_resume(rdev);
6506         if (ret) {
6507                 r600_ih_ring_fini(rdev);
6508                 return ret;
6509         }
6510
6511         /* setup interrupt control */
6512         /* XXX this should actually be a bus address, not an MC address. same on older asics */
6513         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6514         interrupt_cntl = RREG32(INTERRUPT_CNTL);
6515         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6516          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6517          */
6518         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6519         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6520         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6521         WREG32(INTERRUPT_CNTL, interrupt_cntl);
6522
6523         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
6524         rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
6525
6526         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6527                       IH_WPTR_OVERFLOW_CLEAR |
6528                       (rb_bufsz << 1));
6529
6530         if (rdev->wb.enabled)
6531                 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6532
6533         /* set the writeback address whether it's enabled or not */
6534         WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6535         WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6536
6537         WREG32(IH_RB_CNTL, ih_rb_cntl);
6538
6539         /* set rptr, wptr to 0 */
6540         WREG32(IH_RB_RPTR, 0);
6541         WREG32(IH_RB_WPTR, 0);
6542
6543         /* Default settings for IH_CNTL (disabled at first) */
6544         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6545         /* RPTR_REARM only works if msi's are enabled */
6546         if (rdev->msi_enabled)
6547                 ih_cntl |= RPTR_REARM;
6548         WREG32(IH_CNTL, ih_cntl);
6549
6550         /* force the active interrupt state to all disabled */
6551         cik_disable_interrupt_state(rdev);
6552
6553         pci_set_master(rdev->pdev);
6554
6555         /* enable irqs */
6556         cik_enable_interrupts(rdev);
6557
6558         return ret;
6559 }
6560
6561 /**
6562  * cik_irq_set - enable/disable interrupt sources
6563  *
6564  * @rdev: radeon_device pointer
6565  *
6566  * Enable interrupt sources on the GPU (vblanks, hpd,
6567  * etc.) (CIK).
6568  * Returns 0 for success, errors for failure.
6569  */
6570 int cik_irq_set(struct radeon_device *rdev)
6571 {
6572         u32 cp_int_cntl;
6573         u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6574         u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
6575         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6576         u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6577         u32 grbm_int_cntl = 0;
6578         u32 dma_cntl, dma_cntl1;
6579         u32 thermal_int;
6580
6581         if (!rdev->irq.installed) {
6582                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6583                 return -EINVAL;
6584         }
6585         /* don't enable anything if the ih is disabled */
6586         if (!rdev->ih.enabled) {
6587                 cik_disable_interrupts(rdev);
6588                 /* force the active interrupt state to all disabled */
6589                 cik_disable_interrupt_state(rdev);
6590                 return 0;
6591         }
6592
6593         cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6594                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6595         cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6596
6597         hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6598         hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6599         hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6600         hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6601         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6602         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6603
6604         dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6605         dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6606
6607         cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6608         cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6609         cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6610         cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6611         cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6612         cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6613         cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6614         cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6615
6616         if (rdev->flags & RADEON_IS_IGP)
6617                 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6618                         ~(THERM_INTH_MASK | THERM_INTL_MASK);
6619         else
6620                 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6621                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6622
6623         /* enable CP interrupts on all rings */
6624         if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6625                 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6626                 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6627         }
6628         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6629                 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6630                 DRM_DEBUG("si_irq_set: sw int cp1\n");
6631                 if (ring->me == 1) {
6632                         switch (ring->pipe) {
6633                         case 0:
6634                                 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6635                                 break;
6636                         case 1:
6637                                 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6638                                 break;
6639                         case 2:
6640                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6641                                 break;
6642                         case 3:
6643                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6644                                 break;
6645                         default:
6646                                 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6647                                 break;
6648                         }
6649                 } else if (ring->me == 2) {
6650                         switch (ring->pipe) {
6651                         case 0:
6652                                 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6653                                 break;
6654                         case 1:
6655                                 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6656                                 break;
6657                         case 2:
6658                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6659                                 break;
6660                         case 3:
6661                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6662                                 break;
6663                         default:
6664                                 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6665                                 break;
6666                         }
6667                 } else {
6668                         DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6669                 }
6670         }
6671         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6672                 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6673                 DRM_DEBUG("si_irq_set: sw int cp2\n");
6674                 if (ring->me == 1) {
6675                         switch (ring->pipe) {
6676                         case 0:
6677                                 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6678                                 break;
6679                         case 1:
6680                                 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6681                                 break;
6682                         case 2:
6683                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6684                                 break;
6685                         case 3:
6686                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6687                                 break;
6688                         default:
6689                                 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6690                                 break;
6691                         }
6692                 } else if (ring->me == 2) {
6693                         switch (ring->pipe) {
6694                         case 0:
6695                                 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6696                                 break;
6697                         case 1:
6698                                 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6699                                 break;
6700                         case 2:
6701                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6702                                 break;
6703                         case 3:
6704                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6705                                 break;
6706                         default:
6707                                 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6708                                 break;
6709                         }
6710                 } else {
6711                         DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6712                 }
6713         }
6714
6715         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6716                 DRM_DEBUG("cik_irq_set: sw int dma\n");
6717                 dma_cntl |= TRAP_ENABLE;
6718         }
6719
6720         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6721                 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6722                 dma_cntl1 |= TRAP_ENABLE;
6723         }
6724
6725         if (rdev->irq.crtc_vblank_int[0] ||
6726             atomic_read(&rdev->irq.pflip[0])) {
6727                 DRM_DEBUG("cik_irq_set: vblank 0\n");
6728                 crtc1 |= VBLANK_INTERRUPT_MASK;
6729         }
6730         if (rdev->irq.crtc_vblank_int[1] ||
6731             atomic_read(&rdev->irq.pflip[1])) {
6732                 DRM_DEBUG("cik_irq_set: vblank 1\n");
6733                 crtc2 |= VBLANK_INTERRUPT_MASK;
6734         }
6735         if (rdev->irq.crtc_vblank_int[2] ||
6736             atomic_read(&rdev->irq.pflip[2])) {
6737                 DRM_DEBUG("cik_irq_set: vblank 2\n");
6738                 crtc3 |= VBLANK_INTERRUPT_MASK;
6739         }
6740         if (rdev->irq.crtc_vblank_int[3] ||
6741             atomic_read(&rdev->irq.pflip[3])) {
6742                 DRM_DEBUG("cik_irq_set: vblank 3\n");
6743                 crtc4 |= VBLANK_INTERRUPT_MASK;
6744         }
6745         if (rdev->irq.crtc_vblank_int[4] ||
6746             atomic_read(&rdev->irq.pflip[4])) {
6747                 DRM_DEBUG("cik_irq_set: vblank 4\n");
6748                 crtc5 |= VBLANK_INTERRUPT_MASK;
6749         }
6750         if (rdev->irq.crtc_vblank_int[5] ||
6751             atomic_read(&rdev->irq.pflip[5])) {
6752                 DRM_DEBUG("cik_irq_set: vblank 5\n");
6753                 crtc6 |= VBLANK_INTERRUPT_MASK;
6754         }
6755         if (rdev->irq.hpd[0]) {
6756                 DRM_DEBUG("cik_irq_set: hpd 1\n");
6757                 hpd1 |= DC_HPDx_INT_EN;
6758         }
6759         if (rdev->irq.hpd[1]) {
6760                 DRM_DEBUG("cik_irq_set: hpd 2\n");
6761                 hpd2 |= DC_HPDx_INT_EN;
6762         }
6763         if (rdev->irq.hpd[2]) {
6764                 DRM_DEBUG("cik_irq_set: hpd 3\n");
6765                 hpd3 |= DC_HPDx_INT_EN;
6766         }
6767         if (rdev->irq.hpd[3]) {
6768                 DRM_DEBUG("cik_irq_set: hpd 4\n");
6769                 hpd4 |= DC_HPDx_INT_EN;
6770         }
6771         if (rdev->irq.hpd[4]) {
6772                 DRM_DEBUG("cik_irq_set: hpd 5\n");
6773                 hpd5 |= DC_HPDx_INT_EN;
6774         }
6775         if (rdev->irq.hpd[5]) {
6776                 DRM_DEBUG("cik_irq_set: hpd 6\n");
6777                 hpd6 |= DC_HPDx_INT_EN;
6778         }
6779
6780         if (rdev->irq.dpm_thermal) {
6781                 DRM_DEBUG("dpm thermal\n");
6782                 if (rdev->flags & RADEON_IS_IGP)
6783                         thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
6784                 else
6785                         thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
6786         }
6787
6788         WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6789
6790         WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
6791         WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
6792
6793         WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
6794         WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
6795         WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
6796         WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
6797         WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
6798         WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
6799         WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
6800         WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
6801
6802         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6803
6804         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6805         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6806         if (rdev->num_crtc >= 4) {
6807                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6808                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6809         }
6810         if (rdev->num_crtc >= 6) {
6811                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6812                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6813         }
6814
6815         WREG32(DC_HPD1_INT_CONTROL, hpd1);
6816         WREG32(DC_HPD2_INT_CONTROL, hpd2);
6817         WREG32(DC_HPD3_INT_CONTROL, hpd3);
6818         WREG32(DC_HPD4_INT_CONTROL, hpd4);
6819         WREG32(DC_HPD5_INT_CONTROL, hpd5);
6820         WREG32(DC_HPD6_INT_CONTROL, hpd6);
6821
6822         if (rdev->flags & RADEON_IS_IGP)
6823                 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
6824         else
6825                 WREG32_SMC(CG_THERMAL_INT, thermal_int);
6826
6827         return 0;
6828 }
6829
6830 /**
6831  * cik_irq_ack - ack interrupt sources
6832  *
6833  * @rdev: radeon_device pointer
6834  *
6835  * Ack interrupt sources on the GPU (vblanks, hpd,
6836  * etc.) (CIK).  Certain interrupts sources are sw
6837  * generated and do not require an explicit ack.
6838  */
6839 static inline void cik_irq_ack(struct radeon_device *rdev)
6840 {
6841         u32 tmp;
6842
6843         rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6844         rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6845         rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6846         rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6847         rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6848         rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6849         rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
6850
6851         if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
6852                 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6853         if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
6854                 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6855         if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6856                 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6857         if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6858                 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6859
6860         if (rdev->num_crtc >= 4) {
6861                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6862                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6863                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6864                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6865                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6866                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6867                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6868                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6869         }
6870
6871         if (rdev->num_crtc >= 6) {
6872                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6873                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6874                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6875                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6876                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6877                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6878                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6879                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6880         }
6881
6882         if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
6883                 tmp = RREG32(DC_HPD1_INT_CONTROL);
6884                 tmp |= DC_HPDx_INT_ACK;
6885                 WREG32(DC_HPD1_INT_CONTROL, tmp);
6886         }
6887         if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
6888                 tmp = RREG32(DC_HPD2_INT_CONTROL);
6889                 tmp |= DC_HPDx_INT_ACK;
6890                 WREG32(DC_HPD2_INT_CONTROL, tmp);
6891         }
6892         if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6893                 tmp = RREG32(DC_HPD3_INT_CONTROL);
6894                 tmp |= DC_HPDx_INT_ACK;
6895                 WREG32(DC_HPD3_INT_CONTROL, tmp);
6896         }
6897         if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6898                 tmp = RREG32(DC_HPD4_INT_CONTROL);
6899                 tmp |= DC_HPDx_INT_ACK;
6900                 WREG32(DC_HPD4_INT_CONTROL, tmp);
6901         }
6902         if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6903                 tmp = RREG32(DC_HPD5_INT_CONTROL);
6904                 tmp |= DC_HPDx_INT_ACK;
6905                 WREG32(DC_HPD5_INT_CONTROL, tmp);
6906         }
6907         if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6908                 tmp = RREG32(DC_HPD5_INT_CONTROL);
6909                 tmp |= DC_HPDx_INT_ACK;
6910                 WREG32(DC_HPD6_INT_CONTROL, tmp);
6911         }
6912 }
6913
6914 /**
6915  * cik_irq_disable - disable interrupts
6916  *
6917  * @rdev: radeon_device pointer
6918  *
6919  * Disable interrupts on the hw (CIK).
6920  */
6921 static void cik_irq_disable(struct radeon_device *rdev)
6922 {
6923         cik_disable_interrupts(rdev);
6924         /* Wait and acknowledge irq */
6925         mdelay(1);
6926         cik_irq_ack(rdev);
6927         cik_disable_interrupt_state(rdev);
6928 }
6929
6930 /**
6931  * cik_irq_disable - disable interrupts for suspend
6932  *
6933  * @rdev: radeon_device pointer
6934  *
6935  * Disable interrupts and stop the RLC (CIK).
6936  * Used for suspend.
6937  */
6938 static void cik_irq_suspend(struct radeon_device *rdev)
6939 {
6940         cik_irq_disable(rdev);
6941         cik_rlc_stop(rdev);
6942 }
6943
6944 /**
6945  * cik_irq_fini - tear down interrupt support
6946  *
6947  * @rdev: radeon_device pointer
6948  *
6949  * Disable interrupts on the hw and free the IH ring
6950  * buffer (CIK).
6951  * Used for driver unload.
6952  */
6953 static void cik_irq_fini(struct radeon_device *rdev)
6954 {
6955         cik_irq_suspend(rdev);
6956         r600_ih_ring_fini(rdev);
6957 }
6958
6959 /**
6960  * cik_get_ih_wptr - get the IH ring buffer wptr
6961  *
6962  * @rdev: radeon_device pointer
6963  *
6964  * Get the IH ring buffer wptr from either the register
6965  * or the writeback memory buffer (CIK).  Also check for
6966  * ring buffer overflow and deal with it.
6967  * Used by cik_irq_process().
6968  * Returns the value of the wptr.
6969  */
6970 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
6971 {
6972         u32 wptr, tmp;
6973
6974         if (rdev->wb.enabled)
6975                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6976         else
6977                 wptr = RREG32(IH_RB_WPTR);
6978
6979         if (wptr & RB_OVERFLOW) {
6980                 /* When a ring buffer overflow happen start parsing interrupt
6981                  * from the last not overwritten vector (wptr + 16). Hopefully
6982                  * this should allow us to catchup.
6983                  */
6984                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6985                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6986                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6987                 tmp = RREG32(IH_RB_CNTL);
6988                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6989                 WREG32(IH_RB_CNTL, tmp);
6990         }
6991         return (wptr & rdev->ih.ptr_mask);
6992 }
6993
6994 /*        CIK IV Ring
6995  * Each IV ring entry is 128 bits:
6996  * [7:0]    - interrupt source id
6997  * [31:8]   - reserved
6998  * [59:32]  - interrupt source data
6999  * [63:60]  - reserved
7000  * [71:64]  - RINGID
7001  *            CP:
7002  *            ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7003  *            QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7004  *                     - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7005  *            ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7006  *            PIPE_ID - ME0 0=3D
7007  *                    - ME1&2 compute dispatcher (4 pipes each)
7008  *            SDMA:
7009  *            INSTANCE_ID [1:0], QUEUE_ID[1:0]
7010  *            INSTANCE_ID - 0 = sdma0, 1 = sdma1
7011  *            QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7012  * [79:72]  - VMID
7013  * [95:80]  - PASID
7014  * [127:96] - reserved
7015  */
7016 /**
7017  * cik_irq_process - interrupt handler
7018  *
7019  * @rdev: radeon_device pointer
7020  *
7021  * Interrupt hander (CIK).  Walk the IH ring,
7022  * ack interrupts and schedule work to handle
7023  * interrupt events.
7024  * Returns irq process return code.
7025  */
7026 int cik_irq_process(struct radeon_device *rdev)
7027 {
7028         struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7029         struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7030         u32 wptr;
7031         u32 rptr;
7032         u32 src_id, src_data, ring_id;
7033         u8 me_id, pipe_id, queue_id;
7034         u32 ring_index;
7035         bool queue_hotplug = false;
7036         bool queue_reset = false;
7037         u32 addr, status, mc_client;
7038         bool queue_thermal = false;
7039
7040         if (!rdev->ih.enabled || rdev->shutdown)
7041                 return IRQ_NONE;
7042
7043         wptr = cik_get_ih_wptr(rdev);
7044
7045 restart_ih:
7046         /* is somebody else already processing irqs? */
7047         if (atomic_xchg(&rdev->ih.lock, 1))
7048                 return IRQ_NONE;
7049
7050         rptr = rdev->ih.rptr;
7051         DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7052
7053         /* Order reading of wptr vs. reading of IH ring data */
7054         rmb();
7055
7056         /* display interrupts */
7057         cik_irq_ack(rdev);
7058
7059         while (rptr != wptr) {
7060                 /* wptr/rptr are in bytes! */
7061                 ring_index = rptr / 4;
7062                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7063                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7064                 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7065
7066                 switch (src_id) {
7067                 case 1: /* D1 vblank/vline */
7068                         switch (src_data) {
7069                         case 0: /* D1 vblank */
7070                                 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7071                                         if (rdev->irq.crtc_vblank_int[0]) {
7072                                                 drm_handle_vblank(rdev->ddev, 0);
7073                                                 rdev->pm.vblank_sync = true;
7074                                                 wake_up(&rdev->irq.vblank_queue);
7075                                         }
7076                                         if (atomic_read(&rdev->irq.pflip[0]))
7077                                                 radeon_crtc_handle_flip(rdev, 0);
7078                                         rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7079                                         DRM_DEBUG("IH: D1 vblank\n");
7080                                 }
7081                                 break;
7082                         case 1: /* D1 vline */
7083                                 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7084                                         rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7085                                         DRM_DEBUG("IH: D1 vline\n");
7086                                 }
7087                                 break;
7088                         default:
7089                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7090                                 break;
7091                         }
7092                         break;
7093                 case 2: /* D2 vblank/vline */
7094                         switch (src_data) {
7095                         case 0: /* D2 vblank */
7096                                 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7097                                         if (rdev->irq.crtc_vblank_int[1]) {
7098                                                 drm_handle_vblank(rdev->ddev, 1);
7099                                                 rdev->pm.vblank_sync = true;
7100                                                 wake_up(&rdev->irq.vblank_queue);
7101                                         }
7102                                         if (atomic_read(&rdev->irq.pflip[1]))
7103                                                 radeon_crtc_handle_flip(rdev, 1);
7104                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7105                                         DRM_DEBUG("IH: D2 vblank\n");
7106                                 }
7107                                 break;
7108                         case 1: /* D2 vline */
7109                                 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7110                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7111                                         DRM_DEBUG("IH: D2 vline\n");
7112                                 }
7113                                 break;
7114                         default:
7115                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7116                                 break;
7117                         }
7118                         break;
7119                 case 3: /* D3 vblank/vline */
7120                         switch (src_data) {
7121                         case 0: /* D3 vblank */
7122                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7123                                         if (rdev->irq.crtc_vblank_int[2]) {
7124                                                 drm_handle_vblank(rdev->ddev, 2);
7125                                                 rdev->pm.vblank_sync = true;
7126                                                 wake_up(&rdev->irq.vblank_queue);
7127                                         }
7128                                         if (atomic_read(&rdev->irq.pflip[2]))
7129                                                 radeon_crtc_handle_flip(rdev, 2);
7130                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7131                                         DRM_DEBUG("IH: D3 vblank\n");
7132                                 }
7133                                 break;
7134                         case 1: /* D3 vline */
7135                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7136                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7137                                         DRM_DEBUG("IH: D3 vline\n");
7138                                 }
7139                                 break;
7140                         default:
7141                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7142                                 break;
7143                         }
7144                         break;
7145                 case 4: /* D4 vblank/vline */
7146                         switch (src_data) {
7147                         case 0: /* D4 vblank */
7148                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7149                                         if (rdev->irq.crtc_vblank_int[3]) {
7150                                                 drm_handle_vblank(rdev->ddev, 3);
7151                                                 rdev->pm.vblank_sync = true;
7152                                                 wake_up(&rdev->irq.vblank_queue);
7153                                         }
7154                                         if (atomic_read(&rdev->irq.pflip[3]))
7155                                                 radeon_crtc_handle_flip(rdev, 3);
7156                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7157                                         DRM_DEBUG("IH: D4 vblank\n");
7158                                 }
7159                                 break;
7160                         case 1: /* D4 vline */
7161                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7162                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7163                                         DRM_DEBUG("IH: D4 vline\n");
7164                                 }
7165                                 break;
7166                         default:
7167                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7168                                 break;
7169                         }
7170                         break;
7171                 case 5: /* D5 vblank/vline */
7172                         switch (src_data) {
7173                         case 0: /* D5 vblank */
7174                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7175                                         if (rdev->irq.crtc_vblank_int[4]) {
7176                                                 drm_handle_vblank(rdev->ddev, 4);
7177                                                 rdev->pm.vblank_sync = true;
7178                                                 wake_up(&rdev->irq.vblank_queue);
7179                                         }
7180                                         if (atomic_read(&rdev->irq.pflip[4]))
7181                                                 radeon_crtc_handle_flip(rdev, 4);
7182                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7183                                         DRM_DEBUG("IH: D5 vblank\n");
7184                                 }
7185                                 break;
7186                         case 1: /* D5 vline */
7187                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7188                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7189                                         DRM_DEBUG("IH: D5 vline\n");
7190                                 }
7191                                 break;
7192                         default:
7193                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7194                                 break;
7195                         }
7196                         break;
7197                 case 6: /* D6 vblank/vline */
7198                         switch (src_data) {
7199                         case 0: /* D6 vblank */
7200                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7201                                         if (rdev->irq.crtc_vblank_int[5]) {
7202                                                 drm_handle_vblank(rdev->ddev, 5);
7203                                                 rdev->pm.vblank_sync = true;
7204                                                 wake_up(&rdev->irq.vblank_queue);
7205                                         }
7206                                         if (atomic_read(&rdev->irq.pflip[5]))
7207                                                 radeon_crtc_handle_flip(rdev, 5);
7208                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7209                                         DRM_DEBUG("IH: D6 vblank\n");
7210                                 }
7211                                 break;
7212                         case 1: /* D6 vline */
7213                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7214                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7215                                         DRM_DEBUG("IH: D6 vline\n");
7216                                 }
7217                                 break;
7218                         default:
7219                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7220                                 break;
7221                         }
7222                         break;
7223                 case 42: /* HPD hotplug */
7224                         switch (src_data) {
7225                         case 0:
7226                                 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7227                                         rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7228                                         queue_hotplug = true;
7229                                         DRM_DEBUG("IH: HPD1\n");
7230                                 }
7231                                 break;
7232                         case 1:
7233                                 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7234                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7235                                         queue_hotplug = true;
7236                                         DRM_DEBUG("IH: HPD2\n");
7237                                 }
7238                                 break;
7239                         case 2:
7240                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7241                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7242                                         queue_hotplug = true;
7243                                         DRM_DEBUG("IH: HPD3\n");
7244                                 }
7245                                 break;
7246                         case 3:
7247                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7248                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7249                                         queue_hotplug = true;
7250                                         DRM_DEBUG("IH: HPD4\n");
7251                                 }
7252                                 break;
7253                         case 4:
7254                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7255                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7256                                         queue_hotplug = true;
7257                                         DRM_DEBUG("IH: HPD5\n");
7258                                 }
7259                                 break;
7260                         case 5:
7261                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7262                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7263                                         queue_hotplug = true;
7264                                         DRM_DEBUG("IH: HPD6\n");
7265                                 }
7266                                 break;
7267                         default:
7268                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7269                                 break;
7270                         }
7271                         break;
7272                 case 124: /* UVD */
7273                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7274                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7275                         break;
7276                 case 146:
7277                 case 147:
7278                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7279                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7280                         mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
7281                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7282                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
7283                                 addr);
7284                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
7285                                 status);
7286                         cik_vm_decode_fault(rdev, status, addr, mc_client);
7287                         /* reset addr and status */
7288                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7289                         break;
7290                 case 176: /* GFX RB CP_INT */
7291                 case 177: /* GFX IB CP_INT */
7292                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7293                         break;
7294                 case 181: /* CP EOP event */
7295                         DRM_DEBUG("IH: CP EOP\n");
7296                         /* XXX check the bitfield order! */
7297                         me_id = (ring_id & 0x60) >> 5;
7298                         pipe_id = (ring_id & 0x18) >> 3;
7299                         queue_id = (ring_id & 0x7) >> 0;
7300                         switch (me_id) {
7301                         case 0:
7302                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7303                                 break;
7304                         case 1:
7305                         case 2:
7306                                 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7307                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7308                                 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7309                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7310                                 break;
7311                         }
7312                         break;
7313                 case 184: /* CP Privileged reg access */
7314                         DRM_ERROR("Illegal register access in command stream\n");
7315                         /* XXX check the bitfield order! */
7316                         me_id = (ring_id & 0x60) >> 5;
7317                         pipe_id = (ring_id & 0x18) >> 3;
7318                         queue_id = (ring_id & 0x7) >> 0;
7319                         switch (me_id) {
7320                         case 0:
7321                                 /* This results in a full GPU reset, but all we need to do is soft
7322                                  * reset the CP for gfx
7323                                  */
7324                                 queue_reset = true;
7325                                 break;
7326                         case 1:
7327                                 /* XXX compute */
7328                                 queue_reset = true;
7329                                 break;
7330                         case 2:
7331                                 /* XXX compute */
7332                                 queue_reset = true;
7333                                 break;
7334                         }
7335                         break;
7336                 case 185: /* CP Privileged inst */
7337                         DRM_ERROR("Illegal instruction in command stream\n");
7338                         /* XXX check the bitfield order! */
7339                         me_id = (ring_id & 0x60) >> 5;
7340                         pipe_id = (ring_id & 0x18) >> 3;
7341                         queue_id = (ring_id & 0x7) >> 0;
7342                         switch (me_id) {
7343                         case 0:
7344                                 /* This results in a full GPU reset, but all we need to do is soft
7345                                  * reset the CP for gfx
7346                                  */
7347                                 queue_reset = true;
7348                                 break;
7349                         case 1:
7350                                 /* XXX compute */
7351                                 queue_reset = true;
7352                                 break;
7353                         case 2:
7354                                 /* XXX compute */
7355                                 queue_reset = true;
7356                                 break;
7357                         }
7358                         break;
7359                 case 224: /* SDMA trap event */
7360                         /* XXX check the bitfield order! */
7361                         me_id = (ring_id & 0x3) >> 0;
7362                         queue_id = (ring_id & 0xc) >> 2;
7363                         DRM_DEBUG("IH: SDMA trap\n");
7364                         switch (me_id) {
7365                         case 0:
7366                                 switch (queue_id) {
7367                                 case 0:
7368                                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7369                                         break;
7370                                 case 1:
7371                                         /* XXX compute */
7372                                         break;
7373                                 case 2:
7374                                         /* XXX compute */
7375                                         break;
7376                                 }
7377                                 break;
7378                         case 1:
7379                                 switch (queue_id) {
7380                                 case 0:
7381                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7382                                         break;
7383                                 case 1:
7384                                         /* XXX compute */
7385                                         break;
7386                                 case 2:
7387                                         /* XXX compute */
7388                                         break;
7389                                 }
7390                                 break;
7391                         }
7392                         break;
7393                 case 230: /* thermal low to high */
7394                         DRM_DEBUG("IH: thermal low to high\n");
7395                         rdev->pm.dpm.thermal.high_to_low = false;
7396                         queue_thermal = true;
7397                         break;
7398                 case 231: /* thermal high to low */
7399                         DRM_DEBUG("IH: thermal high to low\n");
7400                         rdev->pm.dpm.thermal.high_to_low = true;
7401                         queue_thermal = true;
7402                         break;
7403                 case 233: /* GUI IDLE */
7404                         DRM_DEBUG("IH: GUI idle\n");
7405                         break;
7406                 case 241: /* SDMA Privileged inst */
7407                 case 247: /* SDMA Privileged inst */
7408                         DRM_ERROR("Illegal instruction in SDMA command stream\n");
7409                         /* XXX check the bitfield order! */
7410                         me_id = (ring_id & 0x3) >> 0;
7411                         queue_id = (ring_id & 0xc) >> 2;
7412                         switch (me_id) {
7413                         case 0:
7414                                 switch (queue_id) {
7415                                 case 0:
7416                                         queue_reset = true;
7417                                         break;
7418                                 case 1:
7419                                         /* XXX compute */
7420                                         queue_reset = true;
7421                                         break;
7422                                 case 2:
7423                                         /* XXX compute */
7424                                         queue_reset = true;
7425                                         break;
7426                                 }
7427                                 break;
7428                         case 1:
7429                                 switch (queue_id) {
7430                                 case 0:
7431                                         queue_reset = true;
7432                                         break;
7433                                 case 1:
7434                                         /* XXX compute */
7435                                         queue_reset = true;
7436                                         break;
7437                                 case 2:
7438                                         /* XXX compute */
7439                                         queue_reset = true;
7440                                         break;
7441                                 }
7442                                 break;
7443                         }
7444                         break;
7445                 default:
7446                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7447                         break;
7448                 }
7449
7450                 /* wptr/rptr are in bytes! */
7451                 rptr += 16;
7452                 rptr &= rdev->ih.ptr_mask;
7453         }
7454         if (queue_hotplug)
7455                 schedule_work(&rdev->hotplug_work);
7456         if (queue_reset)
7457                 schedule_work(&rdev->reset_work);
7458         if (queue_thermal)
7459                 schedule_work(&rdev->pm.dpm.thermal.work);
7460         rdev->ih.rptr = rptr;
7461         WREG32(IH_RB_RPTR, rdev->ih.rptr);
7462         atomic_set(&rdev->ih.lock, 0);
7463
7464         /* make sure wptr hasn't changed while processing */
7465         wptr = cik_get_ih_wptr(rdev);
7466         if (wptr != rptr)
7467                 goto restart_ih;
7468
7469         return IRQ_HANDLED;
7470 }
7471
7472 /*
7473  * startup/shutdown callbacks
7474  */
7475 /**
7476  * cik_startup - program the asic to a functional state
7477  *
7478  * @rdev: radeon_device pointer
7479  *
7480  * Programs the asic to a functional state (CIK).
7481  * Called by cik_init() and cik_resume().
7482  * Returns 0 for success, error for failure.
7483  */
7484 static int cik_startup(struct radeon_device *rdev)
7485 {
7486         struct radeon_ring *ring;
7487         int r;
7488
7489         /* enable pcie gen2/3 link */
7490         cik_pcie_gen3_enable(rdev);
7491         /* enable aspm */
7492         cik_program_aspm(rdev);
7493
7494         /* scratch needs to be initialized before MC */
7495         r = r600_vram_scratch_init(rdev);
7496         if (r)
7497                 return r;
7498
7499         cik_mc_program(rdev);
7500
7501         if (rdev->flags & RADEON_IS_IGP) {
7502                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7503                     !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
7504                         r = cik_init_microcode(rdev);
7505                         if (r) {
7506                                 DRM_ERROR("Failed to load firmware!\n");
7507                                 return r;
7508                         }
7509                 }
7510         } else {
7511                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7512                     !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
7513                     !rdev->mc_fw) {
7514                         r = cik_init_microcode(rdev);
7515                         if (r) {
7516                                 DRM_ERROR("Failed to load firmware!\n");
7517                                 return r;
7518                         }
7519                 }
7520
7521                 r = ci_mc_load_microcode(rdev);
7522                 if (r) {
7523                         DRM_ERROR("Failed to load MC firmware!\n");
7524                         return r;
7525                 }
7526         }
7527
7528         r = cik_pcie_gart_enable(rdev);
7529         if (r)
7530                 return r;
7531         cik_gpu_init(rdev);
7532
7533         /* allocate rlc buffers */
7534         if (rdev->flags & RADEON_IS_IGP) {
7535                 if (rdev->family == CHIP_KAVERI) {
7536                         rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7537                         rdev->rlc.reg_list_size =
7538                                 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7539                 } else {
7540                         rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7541                         rdev->rlc.reg_list_size =
7542                                 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7543                 }
7544         }
7545         rdev->rlc.cs_data = ci_cs_data;
7546         rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
7547         r = sumo_rlc_init(rdev);
7548         if (r) {
7549                 DRM_ERROR("Failed to init rlc BOs!\n");
7550                 return r;
7551         }
7552
7553         /* allocate wb buffer */
7554         r = radeon_wb_init(rdev);
7555         if (r)
7556                 return r;
7557
7558         /* allocate mec buffers */
7559         r = cik_mec_init(rdev);
7560         if (r) {
7561                 DRM_ERROR("Failed to init MEC BOs!\n");
7562                 return r;
7563         }
7564
7565         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7566         if (r) {
7567                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7568                 return r;
7569         }
7570
7571         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7572         if (r) {
7573                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7574                 return r;
7575         }
7576
7577         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7578         if (r) {
7579                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7580                 return r;
7581         }
7582
7583         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7584         if (r) {
7585                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7586                 return r;
7587         }
7588
7589         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7590         if (r) {
7591                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7592                 return r;
7593         }
7594
7595         r = radeon_uvd_resume(rdev);
7596         if (!r) {
7597                 r = uvd_v4_2_resume(rdev);
7598                 if (!r) {
7599                         r = radeon_fence_driver_start_ring(rdev,
7600                                                            R600_RING_TYPE_UVD_INDEX);
7601                         if (r)
7602                                 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7603                 }
7604         }
7605         if (r)
7606                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7607
7608         /* Enable IRQ */
7609         if (!rdev->irq.installed) {
7610                 r = radeon_irq_kms_init(rdev);
7611                 if (r)
7612                         return r;
7613         }
7614
7615         r = cik_irq_init(rdev);
7616         if (r) {
7617                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7618                 radeon_irq_kms_fini(rdev);
7619                 return r;
7620         }
7621         cik_irq_set(rdev);
7622
7623         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7624         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
7625                              CP_RB0_RPTR, CP_RB0_WPTR,
7626                              PACKET3(PACKET3_NOP, 0x3FFF));
7627         if (r)
7628                 return r;
7629
7630         /* set up the compute queues */
7631         /* type-2 packets are deprecated on MEC, use type-3 instead */
7632         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7633         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7634                              CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
7635                              PACKET3(PACKET3_NOP, 0x3FFF));
7636         if (r)
7637                 return r;
7638         ring->me = 1; /* first MEC */
7639         ring->pipe = 0; /* first pipe */
7640         ring->queue = 0; /* first queue */
7641         ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7642
7643         /* type-2 packets are deprecated on MEC, use type-3 instead */
7644         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7645         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7646                              CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
7647                              PACKET3(PACKET3_NOP, 0x3FFF));
7648         if (r)
7649                 return r;
7650         /* dGPU only have 1 MEC */
7651         ring->me = 1; /* first MEC */
7652         ring->pipe = 0; /* first pipe */
7653         ring->queue = 1; /* second queue */
7654         ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7655
7656         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7657         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7658                              SDMA0_GFX_RB_RPTR + SDMA0_REGISTER_OFFSET,
7659                              SDMA0_GFX_RB_WPTR + SDMA0_REGISTER_OFFSET,
7660                              SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7661         if (r)
7662                 return r;
7663
7664         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7665         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7666                              SDMA0_GFX_RB_RPTR + SDMA1_REGISTER_OFFSET,
7667                              SDMA0_GFX_RB_WPTR + SDMA1_REGISTER_OFFSET,
7668                              SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7669         if (r)
7670                 return r;
7671
7672         r = cik_cp_resume(rdev);
7673         if (r)
7674                 return r;
7675
7676         r = cik_sdma_resume(rdev);
7677         if (r)
7678                 return r;
7679
7680         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7681         if (ring->ring_size) {
7682                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7683                                      UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
7684                                      RADEON_CP_PACKET2);
7685                 if (!r)
7686                         r = uvd_v1_0_init(rdev);
7687                 if (r)
7688                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7689         }
7690
7691         r = radeon_ib_pool_init(rdev);
7692         if (r) {
7693                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7694                 return r;
7695         }
7696
7697         r = radeon_vm_manager_init(rdev);
7698         if (r) {
7699                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7700                 return r;
7701         }
7702
7703         r = dce6_audio_init(rdev);
7704         if (r)
7705                 return r;
7706
7707         return 0;
7708 }
7709
7710 /**
7711  * cik_resume - resume the asic to a functional state
7712  *
7713  * @rdev: radeon_device pointer
7714  *
7715  * Programs the asic to a functional state (CIK).
7716  * Called at resume.
7717  * Returns 0 for success, error for failure.
7718  */
7719 int cik_resume(struct radeon_device *rdev)
7720 {
7721         int r;
7722
7723         /* post card */
7724         atom_asic_init(rdev->mode_info.atom_context);
7725
7726         /* init golden registers */
7727         cik_init_golden_registers(rdev);
7728
7729         rdev->accel_working = true;
7730         r = cik_startup(rdev);
7731         if (r) {
7732                 DRM_ERROR("cik startup failed on resume\n");
7733                 rdev->accel_working = false;
7734                 return r;
7735         }
7736
7737         return r;
7738
7739 }
7740
7741 /**
7742  * cik_suspend - suspend the asic
7743  *
7744  * @rdev: radeon_device pointer
7745  *
7746  * Bring the chip into a state suitable for suspend (CIK).
7747  * Called at suspend.
7748  * Returns 0 for success.
7749  */
7750 int cik_suspend(struct radeon_device *rdev)
7751 {
7752         dce6_audio_fini(rdev);
7753         radeon_vm_manager_fini(rdev);
7754         cik_cp_enable(rdev, false);
7755         cik_sdma_enable(rdev, false);
7756         uvd_v1_0_fini(rdev);
7757         radeon_uvd_suspend(rdev);
7758         cik_fini_pg(rdev);
7759         cik_fini_cg(rdev);
7760         cik_irq_suspend(rdev);
7761         radeon_wb_disable(rdev);
7762         cik_pcie_gart_disable(rdev);
7763         return 0;
7764 }
7765
7766 /* Plan is to move initialization in that function and use
7767  * helper function so that radeon_device_init pretty much
7768  * do nothing more than calling asic specific function. This
7769  * should also allow to remove a bunch of callback function
7770  * like vram_info.
7771  */
7772 /**
7773  * cik_init - asic specific driver and hw init
7774  *
7775  * @rdev: radeon_device pointer
7776  *
7777  * Setup asic specific driver variables and program the hw
7778  * to a functional state (CIK).
7779  * Called at driver startup.
7780  * Returns 0 for success, errors for failure.
7781  */
7782 int cik_init(struct radeon_device *rdev)
7783 {
7784         struct radeon_ring *ring;
7785         int r;
7786
7787         /* Read BIOS */
7788         if (!radeon_get_bios(rdev)) {
7789                 if (ASIC_IS_AVIVO(rdev))
7790                         return -EINVAL;
7791         }
7792         /* Must be an ATOMBIOS */
7793         if (!rdev->is_atom_bios) {
7794                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7795                 return -EINVAL;
7796         }
7797         r = radeon_atombios_init(rdev);
7798         if (r)
7799                 return r;
7800
7801         /* Post card if necessary */
7802         if (!radeon_card_posted(rdev)) {
7803                 if (!rdev->bios) {
7804                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7805                         return -EINVAL;
7806                 }
7807                 DRM_INFO("GPU not posted. posting now...\n");
7808                 atom_asic_init(rdev->mode_info.atom_context);
7809         }
7810         /* init golden registers */
7811         cik_init_golden_registers(rdev);
7812         /* Initialize scratch registers */
7813         cik_scratch_init(rdev);
7814         /* Initialize surface registers */
7815         radeon_surface_init(rdev);
7816         /* Initialize clocks */
7817         radeon_get_clock_info(rdev->ddev);
7818
7819         /* Fence driver */
7820         r = radeon_fence_driver_init(rdev);
7821         if (r)
7822                 return r;
7823
7824         /* initialize memory controller */
7825         r = cik_mc_init(rdev);
7826         if (r)
7827                 return r;
7828         /* Memory manager */
7829         r = radeon_bo_init(rdev);
7830         if (r)
7831                 return r;
7832
7833         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7834         ring->ring_obj = NULL;
7835         r600_ring_init(rdev, ring, 1024 * 1024);
7836
7837         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7838         ring->ring_obj = NULL;
7839         r600_ring_init(rdev, ring, 1024 * 1024);
7840         r = radeon_doorbell_get(rdev, &ring->doorbell_index);
7841         if (r)
7842                 return r;
7843
7844         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7845         ring->ring_obj = NULL;
7846         r600_ring_init(rdev, ring, 1024 * 1024);
7847         r = radeon_doorbell_get(rdev, &ring->doorbell_index);
7848         if (r)
7849                 return r;
7850
7851         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7852         ring->ring_obj = NULL;
7853         r600_ring_init(rdev, ring, 256 * 1024);
7854
7855         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7856         ring->ring_obj = NULL;
7857         r600_ring_init(rdev, ring, 256 * 1024);
7858
7859         r = radeon_uvd_init(rdev);
7860         if (!r) {
7861                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7862                 ring->ring_obj = NULL;
7863                 r600_ring_init(rdev, ring, 4096);
7864         }
7865
7866         rdev->ih.ring_obj = NULL;
7867         r600_ih_ring_init(rdev, 64 * 1024);
7868
7869         r = r600_pcie_gart_init(rdev);
7870         if (r)
7871                 return r;
7872
7873         rdev->accel_working = true;
7874         r = cik_startup(rdev);
7875         if (r) {
7876                 dev_err(rdev->dev, "disabling GPU acceleration\n");
7877                 cik_cp_fini(rdev);
7878                 cik_sdma_fini(rdev);
7879                 cik_irq_fini(rdev);
7880                 sumo_rlc_fini(rdev);
7881                 cik_mec_fini(rdev);
7882                 radeon_wb_fini(rdev);
7883                 radeon_ib_pool_fini(rdev);
7884                 radeon_vm_manager_fini(rdev);
7885                 radeon_irq_kms_fini(rdev);
7886                 cik_pcie_gart_fini(rdev);
7887                 rdev->accel_working = false;
7888         }
7889
7890         /* Don't start up if the MC ucode is missing.
7891          * The default clocks and voltages before the MC ucode
7892          * is loaded are not suffient for advanced operations.
7893          */
7894         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
7895                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
7896                 return -EINVAL;
7897         }
7898
7899         return 0;
7900 }
7901
7902 /**
7903  * cik_fini - asic specific driver and hw fini
7904  *
7905  * @rdev: radeon_device pointer
7906  *
7907  * Tear down the asic specific driver variables and program the hw
7908  * to an idle state (CIK).
7909  * Called at driver unload.
7910  */
7911 void cik_fini(struct radeon_device *rdev)
7912 {
7913         cik_cp_fini(rdev);
7914         cik_sdma_fini(rdev);
7915         cik_fini_pg(rdev);
7916         cik_fini_cg(rdev);
7917         cik_irq_fini(rdev);
7918         sumo_rlc_fini(rdev);
7919         cik_mec_fini(rdev);
7920         radeon_wb_fini(rdev);
7921         radeon_vm_manager_fini(rdev);
7922         radeon_ib_pool_fini(rdev);
7923         radeon_irq_kms_fini(rdev);
7924         uvd_v1_0_fini(rdev);
7925         radeon_uvd_fini(rdev);
7926         cik_pcie_gart_fini(rdev);
7927         r600_vram_scratch_fini(rdev);
7928         radeon_gem_fini(rdev);
7929         radeon_fence_driver_fini(rdev);
7930         radeon_bo_fini(rdev);
7931         radeon_atombios_fini(rdev);
7932         kfree(rdev->bios);
7933         rdev->bios = NULL;
7934 }
7935
7936 void dce8_program_fmt(struct drm_encoder *encoder)
7937 {
7938         struct drm_device *dev = encoder->dev;
7939         struct radeon_device *rdev = dev->dev_private;
7940         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
7941         struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
7942         struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
7943         int bpc = 0;
7944         u32 tmp = 0;
7945         enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
7946
7947         if (connector) {
7948                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
7949                 bpc = radeon_get_monitor_bpc(connector);
7950                 dither = radeon_connector->dither;
7951         }
7952
7953         /* LVDS/eDP FMT is set up by atom */
7954         if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
7955                 return;
7956
7957         /* not needed for analog */
7958         if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
7959             (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
7960                 return;
7961
7962         if (bpc == 0)
7963                 return;
7964
7965         switch (bpc) {
7966         case 6:
7967                 if (dither == RADEON_FMT_DITHER_ENABLE)
7968                         /* XXX sort out optimal dither settings */
7969                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7970                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
7971                 else
7972                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
7973                 break;
7974         case 8:
7975                 if (dither == RADEON_FMT_DITHER_ENABLE)
7976                         /* XXX sort out optimal dither settings */
7977                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7978                                 FMT_RGB_RANDOM_ENABLE |
7979                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
7980                 else
7981                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
7982                 break;
7983         case 10:
7984                 if (dither == RADEON_FMT_DITHER_ENABLE)
7985                         /* XXX sort out optimal dither settings */
7986                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7987                                 FMT_RGB_RANDOM_ENABLE |
7988                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
7989                 else
7990                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
7991                 break;
7992         default:
7993                 /* not needed */
7994                 break;
7995         }
7996
7997         WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
7998 }
7999
8000 /* display watermark setup */
8001 /**
8002  * dce8_line_buffer_adjust - Set up the line buffer
8003  *
8004  * @rdev: radeon_device pointer
8005  * @radeon_crtc: the selected display controller
8006  * @mode: the current display mode on the selected display
8007  * controller
8008  *
8009  * Setup up the line buffer allocation for
8010  * the selected display controller (CIK).
8011  * Returns the line buffer size in pixels.
8012  */
8013 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8014                                    struct radeon_crtc *radeon_crtc,
8015                                    struct drm_display_mode *mode)
8016 {
8017         u32 tmp, buffer_alloc, i;
8018         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8019         /*
8020          * Line Buffer Setup
8021          * There are 6 line buffers, one for each display controllers.
8022          * There are 3 partitions per LB. Select the number of partitions
8023          * to enable based on the display width.  For display widths larger
8024          * than 4096, you need use to use 2 display controllers and combine
8025          * them using the stereo blender.
8026          */
8027         if (radeon_crtc->base.enabled && mode) {
8028                 if (mode->crtc_hdisplay < 1920) {
8029                         tmp = 1;
8030                         buffer_alloc = 2;
8031                 } else if (mode->crtc_hdisplay < 2560) {
8032                         tmp = 2;
8033                         buffer_alloc = 2;
8034                 } else if (mode->crtc_hdisplay < 4096) {
8035                         tmp = 0;
8036                         buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8037                 } else {
8038                         DRM_DEBUG_KMS("Mode too big for LB!\n");
8039                         tmp = 0;
8040                         buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8041                 }
8042         } else {
8043                 tmp = 1;
8044                 buffer_alloc = 0;
8045         }
8046
8047         WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8048                LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8049
8050         WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8051                DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8052         for (i = 0; i < rdev->usec_timeout; i++) {
8053                 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8054                     DMIF_BUFFERS_ALLOCATED_COMPLETED)
8055                         break;
8056                 udelay(1);
8057         }
8058
8059         if (radeon_crtc->base.enabled && mode) {
8060                 switch (tmp) {
8061                 case 0:
8062                 default:
8063                         return 4096 * 2;
8064                 case 1:
8065                         return 1920 * 2;
8066                 case 2:
8067                         return 2560 * 2;
8068                 }
8069         }
8070
8071         /* controller not enabled, so no lb used */
8072         return 0;
8073 }
8074
8075 /**
8076  * cik_get_number_of_dram_channels - get the number of dram channels
8077  *
8078  * @rdev: radeon_device pointer
8079  *
8080  * Look up the number of video ram channels (CIK).
8081  * Used for display watermark bandwidth calculations
8082  * Returns the number of dram channels
8083  */
8084 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8085 {
8086         u32 tmp = RREG32(MC_SHARED_CHMAP);
8087
8088         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8089         case 0:
8090         default:
8091                 return 1;
8092         case 1:
8093                 return 2;
8094         case 2:
8095                 return 4;
8096         case 3:
8097                 return 8;
8098         case 4:
8099                 return 3;
8100         case 5:
8101                 return 6;
8102         case 6:
8103                 return 10;
8104         case 7:
8105                 return 12;
8106         case 8:
8107                 return 16;
8108         }
8109 }
8110
8111 struct dce8_wm_params {
8112         u32 dram_channels; /* number of dram channels */
8113         u32 yclk;          /* bandwidth per dram data pin in kHz */
8114         u32 sclk;          /* engine clock in kHz */
8115         u32 disp_clk;      /* display clock in kHz */
8116         u32 src_width;     /* viewport width */
8117         u32 active_time;   /* active display time in ns */
8118         u32 blank_time;    /* blank time in ns */
8119         bool interlaced;    /* mode is interlaced */
8120         fixed20_12 vsc;    /* vertical scale ratio */
8121         u32 num_heads;     /* number of active crtcs */
8122         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8123         u32 lb_size;       /* line buffer allocated to pipe */
8124         u32 vtaps;         /* vertical scaler taps */
8125 };
8126
8127 /**
8128  * dce8_dram_bandwidth - get the dram bandwidth
8129  *
8130  * @wm: watermark calculation data
8131  *
8132  * Calculate the raw dram bandwidth (CIK).
8133  * Used for display watermark bandwidth calculations
8134  * Returns the dram bandwidth in MBytes/s
8135  */
8136 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8137 {
8138         /* Calculate raw DRAM Bandwidth */
8139         fixed20_12 dram_efficiency; /* 0.7 */
8140         fixed20_12 yclk, dram_channels, bandwidth;
8141         fixed20_12 a;
8142
8143         a.full = dfixed_const(1000);
8144         yclk.full = dfixed_const(wm->yclk);
8145         yclk.full = dfixed_div(yclk, a);
8146         dram_channels.full = dfixed_const(wm->dram_channels * 4);
8147         a.full = dfixed_const(10);
8148         dram_efficiency.full = dfixed_const(7);
8149         dram_efficiency.full = dfixed_div(dram_efficiency, a);
8150         bandwidth.full = dfixed_mul(dram_channels, yclk);
8151         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8152
8153         return dfixed_trunc(bandwidth);
8154 }
8155
8156 /**
8157  * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8158  *
8159  * @wm: watermark calculation data
8160  *
8161  * Calculate the dram bandwidth used for display (CIK).
8162  * Used for display watermark bandwidth calculations
8163  * Returns the dram bandwidth for display in MBytes/s
8164  */
8165 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8166 {
8167         /* Calculate DRAM Bandwidth and the part allocated to display. */
8168         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8169         fixed20_12 yclk, dram_channels, bandwidth;
8170         fixed20_12 a;
8171
8172         a.full = dfixed_const(1000);
8173         yclk.full = dfixed_const(wm->yclk);
8174         yclk.full = dfixed_div(yclk, a);
8175         dram_channels.full = dfixed_const(wm->dram_channels * 4);
8176         a.full = dfixed_const(10);
8177         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8178         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8179         bandwidth.full = dfixed_mul(dram_channels, yclk);
8180         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8181
8182         return dfixed_trunc(bandwidth);
8183 }
8184
8185 /**
8186  * dce8_data_return_bandwidth - get the data return bandwidth
8187  *
8188  * @wm: watermark calculation data
8189  *
8190  * Calculate the data return bandwidth used for display (CIK).
8191  * Used for display watermark bandwidth calculations
8192  * Returns the data return bandwidth in MBytes/s
8193  */
8194 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8195 {
8196         /* Calculate the display Data return Bandwidth */
8197         fixed20_12 return_efficiency; /* 0.8 */
8198         fixed20_12 sclk, bandwidth;
8199         fixed20_12 a;
8200
8201         a.full = dfixed_const(1000);
8202         sclk.full = dfixed_const(wm->sclk);
8203         sclk.full = dfixed_div(sclk, a);
8204         a.full = dfixed_const(10);
8205         return_efficiency.full = dfixed_const(8);
8206         return_efficiency.full = dfixed_div(return_efficiency, a);
8207         a.full = dfixed_const(32);
8208         bandwidth.full = dfixed_mul(a, sclk);
8209         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8210
8211         return dfixed_trunc(bandwidth);
8212 }
8213
8214 /**
8215  * dce8_dmif_request_bandwidth - get the dmif bandwidth
8216  *
8217  * @wm: watermark calculation data
8218  *
8219  * Calculate the dmif bandwidth used for display (CIK).
8220  * Used for display watermark bandwidth calculations
8221  * Returns the dmif bandwidth in MBytes/s
8222  */
8223 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8224 {
8225         /* Calculate the DMIF Request Bandwidth */
8226         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8227         fixed20_12 disp_clk, bandwidth;
8228         fixed20_12 a, b;
8229
8230         a.full = dfixed_const(1000);
8231         disp_clk.full = dfixed_const(wm->disp_clk);
8232         disp_clk.full = dfixed_div(disp_clk, a);
8233         a.full = dfixed_const(32);
8234         b.full = dfixed_mul(a, disp_clk);
8235
8236         a.full = dfixed_const(10);
8237         disp_clk_request_efficiency.full = dfixed_const(8);
8238         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8239
8240         bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8241
8242         return dfixed_trunc(bandwidth);
8243 }
8244
8245 /**
8246  * dce8_available_bandwidth - get the min available bandwidth
8247  *
8248  * @wm: watermark calculation data
8249  *
8250  * Calculate the min available bandwidth used for display (CIK).
8251  * Used for display watermark bandwidth calculations
8252  * Returns the min available bandwidth in MBytes/s
8253  */
8254 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8255 {
8256         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8257         u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8258         u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8259         u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8260
8261         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8262 }
8263
8264 /**
8265  * dce8_average_bandwidth - get the average available bandwidth
8266  *
8267  * @wm: watermark calculation data
8268  *
8269  * Calculate the average available bandwidth used for display (CIK).
8270  * Used for display watermark bandwidth calculations
8271  * Returns the average available bandwidth in MBytes/s
8272  */
8273 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8274 {
8275         /* Calculate the display mode Average Bandwidth
8276          * DisplayMode should contain the source and destination dimensions,
8277          * timing, etc.
8278          */
8279         fixed20_12 bpp;
8280         fixed20_12 line_time;
8281         fixed20_12 src_width;
8282         fixed20_12 bandwidth;
8283         fixed20_12 a;
8284
8285         a.full = dfixed_const(1000);
8286         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8287         line_time.full = dfixed_div(line_time, a);
8288         bpp.full = dfixed_const(wm->bytes_per_pixel);
8289         src_width.full = dfixed_const(wm->src_width);
8290         bandwidth.full = dfixed_mul(src_width, bpp);
8291         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8292         bandwidth.full = dfixed_div(bandwidth, line_time);
8293
8294         return dfixed_trunc(bandwidth);
8295 }
8296
8297 /**
8298  * dce8_latency_watermark - get the latency watermark
8299  *
8300  * @wm: watermark calculation data
8301  *
8302  * Calculate the latency watermark (CIK).
8303  * Used for display watermark bandwidth calculations
8304  * Returns the latency watermark in ns
8305  */
8306 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8307 {
8308         /* First calculate the latency in ns */
8309         u32 mc_latency = 2000; /* 2000 ns. */
8310         u32 available_bandwidth = dce8_available_bandwidth(wm);
8311         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
8312         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
8313         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
8314         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
8315                 (wm->num_heads * cursor_line_pair_return_time);
8316         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
8317         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
8318         u32 tmp, dmif_size = 12288;
8319         fixed20_12 a, b, c;
8320
8321         if (wm->num_heads == 0)
8322                 return 0;
8323
8324         a.full = dfixed_const(2);
8325         b.full = dfixed_const(1);
8326         if ((wm->vsc.full > a.full) ||
8327             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
8328             (wm->vtaps >= 5) ||
8329             ((wm->vsc.full >= a.full) && wm->interlaced))
8330                 max_src_lines_per_dst_line = 4;
8331         else
8332                 max_src_lines_per_dst_line = 2;
8333
8334         a.full = dfixed_const(available_bandwidth);
8335         b.full = dfixed_const(wm->num_heads);
8336         a.full = dfixed_div(a, b);
8337
8338         b.full = dfixed_const(mc_latency + 512);
8339         c.full = dfixed_const(wm->disp_clk);
8340         b.full = dfixed_div(b, c);
8341
8342         c.full = dfixed_const(dmif_size);
8343         b.full = dfixed_div(c, b);
8344
8345         tmp = min(dfixed_trunc(a), dfixed_trunc(b));
8346
8347         b.full = dfixed_const(1000);
8348         c.full = dfixed_const(wm->disp_clk);
8349         b.full = dfixed_div(c, b);
8350         c.full = dfixed_const(wm->bytes_per_pixel);
8351         b.full = dfixed_mul(b, c);
8352
8353         lb_fill_bw = min(tmp, dfixed_trunc(b));
8354
8355         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
8356         b.full = dfixed_const(1000);
8357         c.full = dfixed_const(lb_fill_bw);
8358         b.full = dfixed_div(c, b);
8359         a.full = dfixed_div(a, b);
8360         line_fill_time = dfixed_trunc(a);
8361
8362         if (line_fill_time < wm->active_time)
8363                 return latency;
8364         else
8365                 return latency + (line_fill_time - wm->active_time);
8366
8367 }
8368
8369 /**
8370  * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8371  * average and available dram bandwidth
8372  *
8373  * @wm: watermark calculation data
8374  *
8375  * Check if the display average bandwidth fits in the display
8376  * dram bandwidth (CIK).
8377  * Used for display watermark bandwidth calculations
8378  * Returns true if the display fits, false if not.
8379  */
8380 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8381 {
8382         if (dce8_average_bandwidth(wm) <=
8383             (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
8384                 return true;
8385         else
8386                 return false;
8387 }
8388
8389 /**
8390  * dce8_average_bandwidth_vs_available_bandwidth - check
8391  * average and available bandwidth
8392  *
8393  * @wm: watermark calculation data
8394  *
8395  * Check if the display average bandwidth fits in the display
8396  * available bandwidth (CIK).
8397  * Used for display watermark bandwidth calculations
8398  * Returns true if the display fits, false if not.
8399  */
8400 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
8401 {
8402         if (dce8_average_bandwidth(wm) <=
8403             (dce8_available_bandwidth(wm) / wm->num_heads))
8404                 return true;
8405         else
8406                 return false;
8407 }
8408
8409 /**
8410  * dce8_check_latency_hiding - check latency hiding
8411  *
8412  * @wm: watermark calculation data
8413  *
8414  * Check latency hiding (CIK).
8415  * Used for display watermark bandwidth calculations
8416  * Returns true if the display fits, false if not.
8417  */
8418 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
8419 {
8420         u32 lb_partitions = wm->lb_size / wm->src_width;
8421         u32 line_time = wm->active_time + wm->blank_time;
8422         u32 latency_tolerant_lines;
8423         u32 latency_hiding;
8424         fixed20_12 a;
8425
8426         a.full = dfixed_const(1);
8427         if (wm->vsc.full > a.full)
8428                 latency_tolerant_lines = 1;
8429         else {
8430                 if (lb_partitions <= (wm->vtaps + 1))
8431                         latency_tolerant_lines = 1;
8432                 else
8433                         latency_tolerant_lines = 2;
8434         }
8435
8436         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8437
8438         if (dce8_latency_watermark(wm) <= latency_hiding)
8439                 return true;
8440         else
8441                 return false;
8442 }
8443
8444 /**
8445  * dce8_program_watermarks - program display watermarks
8446  *
8447  * @rdev: radeon_device pointer
8448  * @radeon_crtc: the selected display controller
8449  * @lb_size: line buffer size
8450  * @num_heads: number of display controllers in use
8451  *
8452  * Calculate and program the display watermarks for the
8453  * selected display controller (CIK).
8454  */
8455 static void dce8_program_watermarks(struct radeon_device *rdev,
8456                                     struct radeon_crtc *radeon_crtc,
8457                                     u32 lb_size, u32 num_heads)
8458 {
8459         struct drm_display_mode *mode = &radeon_crtc->base.mode;
8460         struct dce8_wm_params wm_low, wm_high;
8461         u32 pixel_period;
8462         u32 line_time = 0;
8463         u32 latency_watermark_a = 0, latency_watermark_b = 0;
8464         u32 tmp, wm_mask;
8465
8466         if (radeon_crtc->base.enabled && num_heads && mode) {
8467                 pixel_period = 1000000 / (u32)mode->clock;
8468                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8469
8470                 /* watermark for high clocks */
8471                 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8472                     rdev->pm.dpm_enabled) {
8473                         wm_high.yclk =
8474                                 radeon_dpm_get_mclk(rdev, false) * 10;
8475                         wm_high.sclk =
8476                                 radeon_dpm_get_sclk(rdev, false) * 10;
8477                 } else {
8478                         wm_high.yclk = rdev->pm.current_mclk * 10;
8479                         wm_high.sclk = rdev->pm.current_sclk * 10;
8480                 }
8481
8482                 wm_high.disp_clk = mode->clock;
8483                 wm_high.src_width = mode->crtc_hdisplay;
8484                 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8485                 wm_high.blank_time = line_time - wm_high.active_time;
8486                 wm_high.interlaced = false;
8487                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8488                         wm_high.interlaced = true;
8489                 wm_high.vsc = radeon_crtc->vsc;
8490                 wm_high.vtaps = 1;
8491                 if (radeon_crtc->rmx_type != RMX_OFF)
8492                         wm_high.vtaps = 2;
8493                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8494                 wm_high.lb_size = lb_size;
8495                 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8496                 wm_high.num_heads = num_heads;
8497
8498                 /* set for high clocks */
8499                 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
8500
8501                 /* possibly force display priority to high */
8502                 /* should really do this at mode validation time... */
8503                 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8504                     !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8505                     !dce8_check_latency_hiding(&wm_high) ||
8506                     (rdev->disp_priority == 2)) {
8507                         DRM_DEBUG_KMS("force priority to high\n");
8508                 }
8509
8510                 /* watermark for low clocks */
8511                 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8512                     rdev->pm.dpm_enabled) {
8513                         wm_low.yclk =
8514                                 radeon_dpm_get_mclk(rdev, true) * 10;
8515                         wm_low.sclk =
8516                                 radeon_dpm_get_sclk(rdev, true) * 10;
8517                 } else {
8518                         wm_low.yclk = rdev->pm.current_mclk * 10;
8519                         wm_low.sclk = rdev->pm.current_sclk * 10;
8520                 }
8521
8522                 wm_low.disp_clk = mode->clock;
8523                 wm_low.src_width = mode->crtc_hdisplay;
8524                 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8525                 wm_low.blank_time = line_time - wm_low.active_time;
8526                 wm_low.interlaced = false;
8527                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8528                         wm_low.interlaced = true;
8529                 wm_low.vsc = radeon_crtc->vsc;
8530                 wm_low.vtaps = 1;
8531                 if (radeon_crtc->rmx_type != RMX_OFF)
8532                         wm_low.vtaps = 2;
8533                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8534                 wm_low.lb_size = lb_size;
8535                 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
8536                 wm_low.num_heads = num_heads;
8537
8538                 /* set for low clocks */
8539                 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
8540
8541                 /* possibly force display priority to high */
8542                 /* should really do this at mode validation time... */
8543                 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
8544                     !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
8545                     !dce8_check_latency_hiding(&wm_low) ||
8546                     (rdev->disp_priority == 2)) {
8547                         DRM_DEBUG_KMS("force priority to high\n");
8548                 }
8549         }
8550
8551         /* select wm A */
8552         wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8553         tmp = wm_mask;
8554         tmp &= ~LATENCY_WATERMARK_MASK(3);
8555         tmp |= LATENCY_WATERMARK_MASK(1);
8556         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8557         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8558                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
8559                 LATENCY_HIGH_WATERMARK(line_time)));
8560         /* select wm B */
8561         tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8562         tmp &= ~LATENCY_WATERMARK_MASK(3);
8563         tmp |= LATENCY_WATERMARK_MASK(2);
8564         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8565         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8566                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
8567                 LATENCY_HIGH_WATERMARK(line_time)));
8568         /* restore original selection */
8569         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
8570
8571         /* save values for DPM */
8572         radeon_crtc->line_time = line_time;
8573         radeon_crtc->wm_high = latency_watermark_a;
8574         radeon_crtc->wm_low = latency_watermark_b;
8575 }
8576
8577 /**
8578  * dce8_bandwidth_update - program display watermarks
8579  *
8580  * @rdev: radeon_device pointer
8581  *
8582  * Calculate and program the display watermarks and line
8583  * buffer allocation (CIK).
8584  */
8585 void dce8_bandwidth_update(struct radeon_device *rdev)
8586 {
8587         struct drm_display_mode *mode = NULL;
8588         u32 num_heads = 0, lb_size;
8589         int i;
8590
8591         radeon_update_display_priority(rdev);
8592
8593         for (i = 0; i < rdev->num_crtc; i++) {
8594                 if (rdev->mode_info.crtcs[i]->base.enabled)
8595                         num_heads++;
8596         }
8597         for (i = 0; i < rdev->num_crtc; i++) {
8598                 mode = &rdev->mode_info.crtcs[i]->base.mode;
8599                 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
8600                 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
8601         }
8602 }
8603
8604 /**
8605  * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8606  *
8607  * @rdev: radeon_device pointer
8608  *
8609  * Fetches a GPU clock counter snapshot (SI).
8610  * Returns the 64 bit clock counter snapshot.
8611  */
8612 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8613 {
8614         uint64_t clock;
8615
8616         mutex_lock(&rdev->gpu_clock_mutex);
8617         WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8618         clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8619                 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8620         mutex_unlock(&rdev->gpu_clock_mutex);
8621         return clock;
8622 }
8623
8624 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
8625                               u32 cntl_reg, u32 status_reg)
8626 {
8627         int r, i;
8628         struct atom_clock_dividers dividers;
8629         uint32_t tmp;
8630
8631         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8632                                            clock, false, &dividers);
8633         if (r)
8634                 return r;
8635
8636         tmp = RREG32_SMC(cntl_reg);
8637         tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
8638         tmp |= dividers.post_divider;
8639         WREG32_SMC(cntl_reg, tmp);
8640
8641         for (i = 0; i < 100; i++) {
8642                 if (RREG32_SMC(status_reg) & DCLK_STATUS)
8643                         break;
8644                 mdelay(10);
8645         }
8646         if (i == 100)
8647                 return -ETIMEDOUT;
8648
8649         return 0;
8650 }
8651
8652 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
8653 {
8654         int r = 0;
8655
8656         r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
8657         if (r)
8658                 return r;
8659
8660         r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
8661         return r;
8662 }
8663
8664 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
8665 {
8666         struct pci_dev *root = rdev->pdev->bus->self;
8667         int bridge_pos, gpu_pos;
8668         u32 speed_cntl, mask, current_data_rate;
8669         int ret, i;
8670         u16 tmp16;
8671
8672         if (radeon_pcie_gen2 == 0)
8673                 return;
8674
8675         if (rdev->flags & RADEON_IS_IGP)
8676                 return;
8677
8678         if (!(rdev->flags & RADEON_IS_PCIE))
8679                 return;
8680
8681         ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
8682         if (ret != 0)
8683                 return;
8684
8685         if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
8686                 return;
8687
8688         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8689         current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
8690                 LC_CURRENT_DATA_RATE_SHIFT;
8691         if (mask & DRM_PCIE_SPEED_80) {
8692                 if (current_data_rate == 2) {
8693                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
8694                         return;
8695                 }
8696                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
8697         } else if (mask & DRM_PCIE_SPEED_50) {
8698                 if (current_data_rate == 1) {
8699                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
8700                         return;
8701                 }
8702                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
8703         }
8704
8705         bridge_pos = pci_pcie_cap(root);
8706         if (!bridge_pos)
8707                 return;
8708
8709         gpu_pos = pci_pcie_cap(rdev->pdev);
8710         if (!gpu_pos)
8711                 return;
8712
8713         if (mask & DRM_PCIE_SPEED_80) {
8714                 /* re-try equalization if gen3 is not already enabled */
8715                 if (current_data_rate != 2) {
8716                         u16 bridge_cfg, gpu_cfg;
8717                         u16 bridge_cfg2, gpu_cfg2;
8718                         u32 max_lw, current_lw, tmp;
8719
8720                         pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8721                         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8722
8723                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
8724                         pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8725
8726                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
8727                         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8728
8729                         tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8730                         max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
8731                         current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
8732
8733                         if (current_lw < max_lw) {
8734                                 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8735                                 if (tmp & LC_RENEGOTIATION_SUPPORT) {
8736                                         tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
8737                                         tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
8738                                         tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
8739                                         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
8740                                 }
8741                         }
8742
8743                         for (i = 0; i < 10; i++) {
8744                                 /* check status */
8745                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
8746                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
8747                                         break;
8748
8749                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8750                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8751
8752                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
8753                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
8754
8755                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8756                                 tmp |= LC_SET_QUIESCE;
8757                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8758
8759                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8760                                 tmp |= LC_REDO_EQ;
8761                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8762
8763                                 mdelay(100);
8764
8765                                 /* linkctl */
8766                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
8767                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8768                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
8769                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8770
8771                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
8772                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8773                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
8774                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8775
8776                                 /* linkctl2 */
8777                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
8778                                 tmp16 &= ~((1 << 4) | (7 << 9));
8779                                 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
8780                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
8781
8782                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8783                                 tmp16 &= ~((1 << 4) | (7 << 9));
8784                                 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
8785                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8786
8787                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8788                                 tmp &= ~LC_SET_QUIESCE;
8789                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8790                         }
8791                 }
8792         }
8793
8794         /* set the link speed */
8795         speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
8796         speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
8797         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8798
8799         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8800         tmp16 &= ~0xf;
8801         if (mask & DRM_PCIE_SPEED_80)
8802                 tmp16 |= 3; /* gen3 */
8803         else if (mask & DRM_PCIE_SPEED_50)
8804                 tmp16 |= 2; /* gen2 */
8805         else
8806                 tmp16 |= 1; /* gen1 */
8807         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8808
8809         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8810         speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
8811         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8812
8813         for (i = 0; i < rdev->usec_timeout; i++) {
8814                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8815                 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
8816                         break;
8817                 udelay(1);
8818         }
8819 }
8820
8821 static void cik_program_aspm(struct radeon_device *rdev)
8822 {
8823         u32 data, orig;
8824         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
8825         bool disable_clkreq = false;
8826
8827         if (radeon_aspm == 0)
8828                 return;
8829
8830         /* XXX double check IGPs */
8831         if (rdev->flags & RADEON_IS_IGP)
8832                 return;
8833
8834         if (!(rdev->flags & RADEON_IS_PCIE))
8835                 return;
8836
8837         orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8838         data &= ~LC_XMIT_N_FTS_MASK;
8839         data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
8840         if (orig != data)
8841                 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
8842
8843         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
8844         data |= LC_GO_TO_RECOVERY;
8845         if (orig != data)
8846                 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
8847
8848         orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
8849         data |= P_IGNORE_EDB_ERR;
8850         if (orig != data)
8851                 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
8852
8853         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8854         data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
8855         data |= LC_PMI_TO_L1_DIS;
8856         if (!disable_l0s)
8857                 data |= LC_L0S_INACTIVITY(7);
8858
8859         if (!disable_l1) {
8860                 data |= LC_L1_INACTIVITY(7);
8861                 data &= ~LC_PMI_TO_L1_DIS;
8862                 if (orig != data)
8863                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8864
8865                 if (!disable_plloff_in_l1) {
8866                         bool clk_req_support;
8867
8868                         orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
8869                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8870                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8871                         if (orig != data)
8872                                 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
8873
8874                         orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
8875                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8876                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8877                         if (orig != data)
8878                                 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
8879
8880                         orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
8881                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8882                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8883                         if (orig != data)
8884                                 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
8885
8886                         orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
8887                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8888                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8889                         if (orig != data)
8890                                 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
8891
8892                         orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8893                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
8894                         data |= LC_DYN_LANES_PWR_STATE(3);
8895                         if (orig != data)
8896                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
8897
8898                         if (!disable_clkreq) {
8899                                 struct pci_dev *root = rdev->pdev->bus->self;
8900                                 u32 lnkcap;
8901
8902                                 clk_req_support = false;
8903                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
8904                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
8905                                         clk_req_support = true;
8906                         } else {
8907                                 clk_req_support = false;
8908                         }
8909
8910                         if (clk_req_support) {
8911                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
8912                                 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
8913                                 if (orig != data)
8914                                         WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
8915
8916                                 orig = data = RREG32_SMC(THM_CLK_CNTL);
8917                                 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
8918                                 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
8919                                 if (orig != data)
8920                                         WREG32_SMC(THM_CLK_CNTL, data);
8921
8922                                 orig = data = RREG32_SMC(MISC_CLK_CTRL);
8923                                 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
8924                                 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
8925                                 if (orig != data)
8926                                         WREG32_SMC(MISC_CLK_CTRL, data);
8927
8928                                 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
8929                                 data &= ~BCLK_AS_XCLK;
8930                                 if (orig != data)
8931                                         WREG32_SMC(CG_CLKPIN_CNTL, data);
8932
8933                                 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
8934                                 data &= ~FORCE_BIF_REFCLK_EN;
8935                                 if (orig != data)
8936                                         WREG32_SMC(CG_CLKPIN_CNTL_2, data);
8937
8938                                 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
8939                                 data &= ~MPLL_CLKOUT_SEL_MASK;
8940                                 data |= MPLL_CLKOUT_SEL(4);
8941                                 if (orig != data)
8942                                         WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
8943                         }
8944                 }
8945         } else {
8946                 if (orig != data)
8947                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8948         }
8949
8950         orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
8951         data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
8952         if (orig != data)
8953                 WREG32_PCIE_PORT(PCIE_CNTL2, data);
8954
8955         if (!disable_l0s) {
8956                 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8957                 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
8958                         data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8959                         if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
8960                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8961                                 data &= ~LC_L0S_INACTIVITY_MASK;
8962                                 if (orig != data)
8963                                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8964                         }
8965                 }
8966         }
8967 }