drivers/gpu/drm/radeon/si.c

   1 /*
   2  * Copyright 2011 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <drm/drmP.h>
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include <drm/radeon_drm.h>
  31 #include "sid.h"
  32 #include "atom.h"
  33 #include "si_blit_shaders.h"
  34 #include "clearstate_si.h"
  35 #include "radeon_ucode.h"
  36
  37
  38 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
  39 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
  40 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
  41 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
  42 MODULE_FIRMWARE("radeon/TAHITI_mc2.bin");
  43 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
  44 MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
  45 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
  46 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
  47 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
  48 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
  49 MODULE_FIRMWARE("radeon/PITCAIRN_mc2.bin");
  50 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
  51 MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
  52 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
  53 MODULE_FIRMWARE("radeon/VERDE_me.bin");
  54 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
  55 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
  56 MODULE_FIRMWARE("radeon/VERDE_mc2.bin");
  57 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
  58 MODULE_FIRMWARE("radeon/VERDE_smc.bin");
  59 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
  60 MODULE_FIRMWARE("radeon/OLAND_me.bin");
  61 MODULE_FIRMWARE("radeon/OLAND_ce.bin");
  62 MODULE_FIRMWARE("radeon/OLAND_mc.bin");
  63 MODULE_FIRMWARE("radeon/OLAND_mc2.bin");
  64 MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
  65 MODULE_FIRMWARE("radeon/OLAND_smc.bin");
  66 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
  67 MODULE_FIRMWARE("radeon/HAINAN_me.bin");
  68 MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
  69 MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
  70 MODULE_FIRMWARE("radeon/HAINAN_mc2.bin");
  71 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
  72 MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
  73
  74 static void si_pcie_gen3_enable(struct radeon_device *rdev);
  75 static void si_program_aspm(struct radeon_device *rdev);
  76 extern void sumo_rlc_fini(struct radeon_device *rdev);
  77 extern int sumo_rlc_init(struct radeon_device *rdev);
  78 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
  79 extern void r600_ih_ring_fini(struct radeon_device *rdev);
  80 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
  81 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
  82 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
  83 extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
  84 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
  85 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
  86 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
  87                                          bool enable);
  88 static void si_init_pg(struct radeon_device *rdev);
  89 static void si_init_cg(struct radeon_device *rdev);
  90 static void si_fini_pg(struct radeon_device *rdev);
  91 static void si_fini_cg(struct radeon_device *rdev);
  92 static void si_rlc_stop(struct radeon_device *rdev);
  93
  94 static const u32 verde_rlc_save_restore_register_list[] =
  95 {
  96         (0x8000 << 16) | (0x98f4 >> 2),
  97         0x00000000,
  98         (0x8040 << 16) | (0x98f4 >> 2),
  99         0x00000000,
 100         (0x8000 << 16) | (0xe80 >> 2),
 101         0x00000000,
 102         (0x8040 << 16) | (0xe80 >> 2),
 103         0x00000000,
 104         (0x8000 << 16) | (0x89bc >> 2),
 105         0x00000000,
 106         (0x8040 << 16) | (0x89bc >> 2),
 107         0x00000000,
 108         (0x8000 << 16) | (0x8c1c >> 2),
 109         0x00000000,
 110         (0x8040 << 16) | (0x8c1c >> 2),
 111         0x00000000,
 112         (0x9c00 << 16) | (0x98f0 >> 2),
 113         0x00000000,
 114         (0x9c00 << 16) | (0xe7c >> 2),
 115         0x00000000,
 116         (0x8000 << 16) | (0x9148 >> 2),
 117         0x00000000,
 118         (0x8040 << 16) | (0x9148 >> 2),
 119         0x00000000,
 120         (0x9c00 << 16) | (0x9150 >> 2),
 121         0x00000000,
 122         (0x9c00 << 16) | (0x897c >> 2),
 123         0x00000000,
 124         (0x9c00 << 16) | (0x8d8c >> 2),
 125         0x00000000,
 126         (0x9c00 << 16) | (0xac54 >> 2),
 127         0X00000000,
 128         0x3,
 129         (0x9c00 << 16) | (0x98f8 >> 2),
 130         0x00000000,
 131         (0x9c00 << 16) | (0x9910 >> 2),
 132         0x00000000,
 133         (0x9c00 << 16) | (0x9914 >> 2),
 134         0x00000000,
 135         (0x9c00 << 16) | (0x9918 >> 2),
 136         0x00000000,
 137         (0x9c00 << 16) | (0x991c >> 2),
 138         0x00000000,
 139         (0x9c00 << 16) | (0x9920 >> 2),
 140         0x00000000,
 141         (0x9c00 << 16) | (0x9924 >> 2),
 142         0x00000000,
 143         (0x9c00 << 16) | (0x9928 >> 2),
 144         0x00000000,
 145         (0x9c00 << 16) | (0x992c >> 2),
 146         0x00000000,
 147         (0x9c00 << 16) | (0x9930 >> 2),
 148         0x00000000,
 149         (0x9c00 << 16) | (0x9934 >> 2),
 150         0x00000000,
 151         (0x9c00 << 16) | (0x9938 >> 2),
 152         0x00000000,
 153         (0x9c00 << 16) | (0x993c >> 2),
 154         0x00000000,
 155         (0x9c00 << 16) | (0x9940 >> 2),
 156         0x00000000,
 157         (0x9c00 << 16) | (0x9944 >> 2),
 158         0x00000000,
 159         (0x9c00 << 16) | (0x9948 >> 2),
 160         0x00000000,
 161         (0x9c00 << 16) | (0x994c >> 2),
 162         0x00000000,
 163         (0x9c00 << 16) | (0x9950 >> 2),
 164         0x00000000,
 165         (0x9c00 << 16) | (0x9954 >> 2),
 166         0x00000000,
 167         (0x9c00 << 16) | (0x9958 >> 2),
 168         0x00000000,
 169         (0x9c00 << 16) | (0x995c >> 2),
 170         0x00000000,
 171         (0x9c00 << 16) | (0x9960 >> 2),
 172         0x00000000,
 173         (0x9c00 << 16) | (0x9964 >> 2),
 174         0x00000000,
 175         (0x9c00 << 16) | (0x9968 >> 2),
 176         0x00000000,
 177         (0x9c00 << 16) | (0x996c >> 2),
 178         0x00000000,
 179         (0x9c00 << 16) | (0x9970 >> 2),
 180         0x00000000,
 181         (0x9c00 << 16) | (0x9974 >> 2),
 182         0x00000000,
 183         (0x9c00 << 16) | (0x9978 >> 2),
 184         0x00000000,
 185         (0x9c00 << 16) | (0x997c >> 2),
 186         0x00000000,
 187         (0x9c00 << 16) | (0x9980 >> 2),
 188         0x00000000,
 189         (0x9c00 << 16) | (0x9984 >> 2),
 190         0x00000000,
 191         (0x9c00 << 16) | (0x9988 >> 2),
 192         0x00000000,
 193         (0x9c00 << 16) | (0x998c >> 2),
 194         0x00000000,
 195         (0x9c00 << 16) | (0x8c00 >> 2),
 196         0x00000000,
 197         (0x9c00 << 16) | (0x8c14 >> 2),
 198         0x00000000,
 199         (0x9c00 << 16) | (0x8c04 >> 2),
 200         0x00000000,
 201         (0x9c00 << 16) | (0x8c08 >> 2),
 202         0x00000000,
 203         (0x8000 << 16) | (0x9b7c >> 2),
 204         0x00000000,
 205         (0x8040 << 16) | (0x9b7c >> 2),
 206         0x00000000,
 207         (0x8000 << 16) | (0xe84 >> 2),
 208         0x00000000,
 209         (0x8040 << 16) | (0xe84 >> 2),
 210         0x00000000,
 211         (0x8000 << 16) | (0x89c0 >> 2),
 212         0x00000000,
 213         (0x8040 << 16) | (0x89c0 >> 2),
 214         0x00000000,
 215         (0x8000 << 16) | (0x914c >> 2),
 216         0x00000000,
 217         (0x8040 << 16) | (0x914c >> 2),
 218         0x00000000,
 219         (0x8000 << 16) | (0x8c20 >> 2),
 220         0x00000000,
 221         (0x8040 << 16) | (0x8c20 >> 2),
 222         0x00000000,
 223         (0x8000 << 16) | (0x9354 >> 2),
 224         0x00000000,
 225         (0x8040 << 16) | (0x9354 >> 2),
 226         0x00000000,
 227         (0x9c00 << 16) | (0x9060 >> 2),
 228         0x00000000,
 229         (0x9c00 << 16) | (0x9364 >> 2),
 230         0x00000000,
 231         (0x9c00 << 16) | (0x9100 >> 2),
 232         0x00000000,
 233         (0x9c00 << 16) | (0x913c >> 2),
 234         0x00000000,
 235         (0x8000 << 16) | (0x90e0 >> 2),
 236         0x00000000,
 237         (0x8000 << 16) | (0x90e4 >> 2),
 238         0x00000000,
 239         (0x8000 << 16) | (0x90e8 >> 2),
 240         0x00000000,
 241         (0x8040 << 16) | (0x90e0 >> 2),
 242         0x00000000,
 243         (0x8040 << 16) | (0x90e4 >> 2),
 244         0x00000000,
 245         (0x8040 << 16) | (0x90e8 >> 2),
 246         0x00000000,
 247         (0x9c00 << 16) | (0x8bcc >> 2),
 248         0x00000000,
 249         (0x9c00 << 16) | (0x8b24 >> 2),
 250         0x00000000,
 251         (0x9c00 << 16) | (0x88c4 >> 2),
 252         0x00000000,
 253         (0x9c00 << 16) | (0x8e50 >> 2),
 254         0x00000000,
 255         (0x9c00 << 16) | (0x8c0c >> 2),
 256         0x00000000,
 257         (0x9c00 << 16) | (0x8e58 >> 2),
 258         0x00000000,
 259         (0x9c00 << 16) | (0x8e5c >> 2),
 260         0x00000000,
 261         (0x9c00 << 16) | (0x9508 >> 2),
 262         0x00000000,
 263         (0x9c00 << 16) | (0x950c >> 2),
 264         0x00000000,
 265         (0x9c00 << 16) | (0x9494 >> 2),
 266         0x00000000,
 267         (0x9c00 << 16) | (0xac0c >> 2),
 268         0x00000000,
 269         (0x9c00 << 16) | (0xac10 >> 2),
 270         0x00000000,
 271         (0x9c00 << 16) | (0xac14 >> 2),
 272         0x00000000,
 273         (0x9c00 << 16) | (0xae00 >> 2),
 274         0x00000000,
 275         (0x9c00 << 16) | (0xac08 >> 2),
 276         0x00000000,
 277         (0x9c00 << 16) | (0x88d4 >> 2),
 278         0x00000000,
 279         (0x9c00 << 16) | (0x88c8 >> 2),
 280         0x00000000,
 281         (0x9c00 << 16) | (0x88cc >> 2),
 282         0x00000000,
 283         (0x9c00 << 16) | (0x89b0 >> 2),
 284         0x00000000,
 285         (0x9c00 << 16) | (0x8b10 >> 2),
 286         0x00000000,
 287         (0x9c00 << 16) | (0x8a14 >> 2),
 288         0x00000000,
 289         (0x9c00 << 16) | (0x9830 >> 2),
 290         0x00000000,
 291         (0x9c00 << 16) | (0x9834 >> 2),
 292         0x00000000,
 293         (0x9c00 << 16) | (0x9838 >> 2),
 294         0x00000000,
 295         (0x9c00 << 16) | (0x9a10 >> 2),
 296         0x00000000,
 297         (0x8000 << 16) | (0x9870 >> 2),
 298         0x00000000,
 299         (0x8000 << 16) | (0x9874 >> 2),
 300         0x00000000,
 301         (0x8001 << 16) | (0x9870 >> 2),
 302         0x00000000,
 303         (0x8001 << 16) | (0x9874 >> 2),
 304         0x00000000,
 305         (0x8040 << 16) | (0x9870 >> 2),
 306         0x00000000,
 307         (0x8040 << 16) | (0x9874 >> 2),
 308         0x00000000,
 309         (0x8041 << 16) | (0x9870 >> 2),
 310         0x00000000,
 311         (0x8041 << 16) | (0x9874 >> 2),
 312         0x00000000,
 313         0x00000000
 314 };
 315
 316 static const u32 tahiti_golden_rlc_registers[] =
 317 {
 318         0xc424, 0xffffffff, 0x00601005,
 319         0xc47c, 0xffffffff, 0x10104040,
 320         0xc488, 0xffffffff, 0x0100000a,
 321         0xc314, 0xffffffff, 0x00000800,
 322         0xc30c, 0xffffffff, 0x800000f4,
 323         0xf4a8, 0xffffffff, 0x00000000
 324 };
 325
 326 static const u32 tahiti_golden_registers[] =
 327 {
 328         0x9a10, 0x00010000, 0x00018208,
 329         0x9830, 0xffffffff, 0x00000000,
 330         0x9834, 0xf00fffff, 0x00000400,
 331         0x9838, 0x0002021c, 0x00020200,
 332         0xc78, 0x00000080, 0x00000000,
 333         0xd030, 0x000300c0, 0x00800040,
 334         0xd830, 0x000300c0, 0x00800040,
 335         0x5bb0, 0x000000f0, 0x00000070,
 336         0x5bc0, 0x00200000, 0x50100000,
 337         0x7030, 0x31000311, 0x00000011,
 338         0x277c, 0x00000003, 0x000007ff,
 339         0x240c, 0x000007ff, 0x00000000,
 340         0x8a14, 0xf000001f, 0x00000007,
 341         0x8b24, 0xffffffff, 0x00ffffff,
 342         0x8b10, 0x0000ff0f, 0x00000000,
 343         0x28a4c, 0x07ffffff, 0x4e000000,
 344         0x28350, 0x3f3f3fff, 0x2a00126a,
 345         0x30, 0x000000ff, 0x0040,
 346         0x34, 0x00000040, 0x00004040,
 347         0x9100, 0x07ffffff, 0x03000000,
 348         0x8e88, 0x01ff1f3f, 0x00000000,
 349         0x8e84, 0x01ff1f3f, 0x00000000,
 350         0x9060, 0x0000007f, 0x00000020,
 351         0x9508, 0x00010000, 0x00010000,
 352         0xac14, 0x00000200, 0x000002fb,
 353         0xac10, 0xffffffff, 0x0000543b,
 354         0xac0c, 0xffffffff, 0xa9210876,
 355         0x88d0, 0xffffffff, 0x000fff40,
 356         0x88d4, 0x0000001f, 0x00000010,
 357         0x1410, 0x20000000, 0x20fffed8,
 358         0x15c0, 0x000c0fc0, 0x000c0400
 359 };
 360
 361 static const u32 tahiti_golden_registers2[] =
 362 {
 363         0xc64, 0x00000001, 0x00000001
 364 };
 365
 366 static const u32 pitcairn_golden_rlc_registers[] =
 367 {
 368         0xc424, 0xffffffff, 0x00601004,
 369         0xc47c, 0xffffffff, 0x10102020,
 370         0xc488, 0xffffffff, 0x01000020,
 371         0xc314, 0xffffffff, 0x00000800,
 372         0xc30c, 0xffffffff, 0x800000a4
 373 };
 374
 375 static const u32 pitcairn_golden_registers[] =
 376 {
 377         0x9a10, 0x00010000, 0x00018208,
 378         0x9830, 0xffffffff, 0x00000000,
 379         0x9834, 0xf00fffff, 0x00000400,
 380         0x9838, 0x0002021c, 0x00020200,
 381         0xc78, 0x00000080, 0x00000000,
 382         0xd030, 0x000300c0, 0x00800040,
 383         0xd830, 0x000300c0, 0x00800040,
 384         0x5bb0, 0x000000f0, 0x00000070,
 385         0x5bc0, 0x00200000, 0x50100000,
 386         0x7030, 0x31000311, 0x00000011,
 387         0x2ae4, 0x00073ffe, 0x000022a2,
 388         0x240c, 0x000007ff, 0x00000000,
 389         0x8a14, 0xf000001f, 0x00000007,
 390         0x8b24, 0xffffffff, 0x00ffffff,
 391         0x8b10, 0x0000ff0f, 0x00000000,
 392         0x28a4c, 0x07ffffff, 0x4e000000,
 393         0x28350, 0x3f3f3fff, 0x2a00126a,
 394         0x30, 0x000000ff, 0x0040,
 395         0x34, 0x00000040, 0x00004040,
 396         0x9100, 0x07ffffff, 0x03000000,
 397         0x9060, 0x0000007f, 0x00000020,
 398         0x9508, 0x00010000, 0x00010000,
 399         0xac14, 0x000003ff, 0x000000f7,
 400         0xac10, 0xffffffff, 0x00000000,
 401         0xac0c, 0xffffffff, 0x32761054,
 402         0x88d4, 0x0000001f, 0x00000010,
 403         0x15c0, 0x000c0fc0, 0x000c0400
 404 };
 405
 406 static const u32 verde_golden_rlc_registers[] =
 407 {
 408         0xc424, 0xffffffff, 0x033f1005,
 409         0xc47c, 0xffffffff, 0x10808020,
 410         0xc488, 0xffffffff, 0x00800008,
 411         0xc314, 0xffffffff, 0x00001000,
 412         0xc30c, 0xffffffff, 0x80010014
 413 };
 414
 415 static const u32 verde_golden_registers[] =
 416 {
 417         0x9a10, 0x00010000, 0x00018208,
 418         0x9830, 0xffffffff, 0x00000000,
 419         0x9834, 0xf00fffff, 0x00000400,
 420         0x9838, 0x0002021c, 0x00020200,
 421         0xc78, 0x00000080, 0x00000000,
 422         0xd030, 0x000300c0, 0x00800040,
 423         0xd030, 0x000300c0, 0x00800040,
 424         0xd830, 0x000300c0, 0x00800040,
 425         0xd830, 0x000300c0, 0x00800040,
 426         0x5bb0, 0x000000f0, 0x00000070,
 427         0x5bc0, 0x00200000, 0x50100000,
 428         0x7030, 0x31000311, 0x00000011,
 429         0x2ae4, 0x00073ffe, 0x000022a2,
 430         0x2ae4, 0x00073ffe, 0x000022a2,
 431         0x2ae4, 0x00073ffe, 0x000022a2,
 432         0x240c, 0x000007ff, 0x00000000,
 433         0x240c, 0x000007ff, 0x00000000,
 434         0x240c, 0x000007ff, 0x00000000,
 435         0x8a14, 0xf000001f, 0x00000007,
 436         0x8a14, 0xf000001f, 0x00000007,
 437         0x8a14, 0xf000001f, 0x00000007,
 438         0x8b24, 0xffffffff, 0x00ffffff,
 439         0x8b10, 0x0000ff0f, 0x00000000,
 440         0x28a4c, 0x07ffffff, 0x4e000000,
 441         0x28350, 0x3f3f3fff, 0x0000124a,
 442         0x28350, 0x3f3f3fff, 0x0000124a,
 443         0x28350, 0x3f3f3fff, 0x0000124a,
 444         0x30, 0x000000ff, 0x0040,
 445         0x34, 0x00000040, 0x00004040,
 446         0x9100, 0x07ffffff, 0x03000000,
 447         0x9100, 0x07ffffff, 0x03000000,
 448         0x8e88, 0x01ff1f3f, 0x00000000,
 449         0x8e88, 0x01ff1f3f, 0x00000000,
 450         0x8e88, 0x01ff1f3f, 0x00000000,
 451         0x8e84, 0x01ff1f3f, 0x00000000,
 452         0x8e84, 0x01ff1f3f, 0x00000000,
 453         0x8e84, 0x01ff1f3f, 0x00000000,
 454         0x9060, 0x0000007f, 0x00000020,
 455         0x9508, 0x00010000, 0x00010000,
 456         0xac14, 0x000003ff, 0x00000003,
 457         0xac14, 0x000003ff, 0x00000003,
 458         0xac14, 0x000003ff, 0x00000003,
 459         0xac10, 0xffffffff, 0x00000000,
 460         0xac10, 0xffffffff, 0x00000000,
 461         0xac10, 0xffffffff, 0x00000000,
 462         0xac0c, 0xffffffff, 0x00001032,
 463         0xac0c, 0xffffffff, 0x00001032,
 464         0xac0c, 0xffffffff, 0x00001032,
 465         0x88d4, 0x0000001f, 0x00000010,
 466         0x88d4, 0x0000001f, 0x00000010,
 467         0x88d4, 0x0000001f, 0x00000010,
 468         0x15c0, 0x000c0fc0, 0x000c0400
 469 };
 470
 471 static const u32 oland_golden_rlc_registers[] =
 472 {
 473         0xc424, 0xffffffff, 0x00601005,
 474         0xc47c, 0xffffffff, 0x10104040,
 475         0xc488, 0xffffffff, 0x0100000a,
 476         0xc314, 0xffffffff, 0x00000800,
 477         0xc30c, 0xffffffff, 0x800000f4
 478 };
 479
 480 static const u32 oland_golden_registers[] =
 481 {
 482         0x9a10, 0x00010000, 0x00018208,
 483         0x9830, 0xffffffff, 0x00000000,
 484         0x9834, 0xf00fffff, 0x00000400,
 485         0x9838, 0x0002021c, 0x00020200,
 486         0xc78, 0x00000080, 0x00000000,
 487         0xd030, 0x000300c0, 0x00800040,
 488         0xd830, 0x000300c0, 0x00800040,
 489         0x5bb0, 0x000000f0, 0x00000070,
 490         0x5bc0, 0x00200000, 0x50100000,
 491         0x7030, 0x31000311, 0x00000011,
 492         0x2ae4, 0x00073ffe, 0x000022a2,
 493         0x240c, 0x000007ff, 0x00000000,
 494         0x8a14, 0xf000001f, 0x00000007,
 495         0x8b24, 0xffffffff, 0x00ffffff,
 496         0x8b10, 0x0000ff0f, 0x00000000,
 497         0x28a4c, 0x07ffffff, 0x4e000000,
 498         0x28350, 0x3f3f3fff, 0x00000082,
 499         0x30, 0x000000ff, 0x0040,
 500         0x34, 0x00000040, 0x00004040,
 501         0x9100, 0x07ffffff, 0x03000000,
 502         0x9060, 0x0000007f, 0x00000020,
 503         0x9508, 0x00010000, 0x00010000,
 504         0xac14, 0x000003ff, 0x000000f3,
 505         0xac10, 0xffffffff, 0x00000000,
 506         0xac0c, 0xffffffff, 0x00003210,
 507         0x88d4, 0x0000001f, 0x00000010,
 508         0x15c0, 0x000c0fc0, 0x000c0400
 509 };
 510
 511 static const u32 hainan_golden_registers[] =
 512 {
 513         0x9a10, 0x00010000, 0x00018208,
 514         0x9830, 0xffffffff, 0x00000000,
 515         0x9834, 0xf00fffff, 0x00000400,
 516         0x9838, 0x0002021c, 0x00020200,
 517         0xd0c0, 0xff000fff, 0x00000100,
 518         0xd030, 0x000300c0, 0x00800040,
 519         0xd8c0, 0xff000fff, 0x00000100,
 520         0xd830, 0x000300c0, 0x00800040,
 521         0x2ae4, 0x00073ffe, 0x000022a2,
 522         0x240c, 0x000007ff, 0x00000000,
 523         0x8a14, 0xf000001f, 0x00000007,
 524         0x8b24, 0xffffffff, 0x00ffffff,
 525         0x8b10, 0x0000ff0f, 0x00000000,
 526         0x28a4c, 0x07ffffff, 0x4e000000,
 527         0x28350, 0x3f3f3fff, 0x00000000,
 528         0x30, 0x000000ff, 0x0040,
 529         0x34, 0x00000040, 0x00004040,
 530         0x9100, 0x03e00000, 0x03600000,
 531         0x9060, 0x0000007f, 0x00000020,
 532         0x9508, 0x00010000, 0x00010000,
 533         0xac14, 0x000003ff, 0x000000f1,
 534         0xac10, 0xffffffff, 0x00000000,
 535         0xac0c, 0xffffffff, 0x00003210,
 536         0x88d4, 0x0000001f, 0x00000010,
 537         0x15c0, 0x000c0fc0, 0x000c0400
 538 };
 539
 540 static const u32 hainan_golden_registers2[] =
 541 {
 542         0x98f8, 0xffffffff, 0x02010001
 543 };
 544
 545 static const u32 tahiti_mgcg_cgcg_init[] =
 546 {
 547         0xc400, 0xffffffff, 0xfffffffc,
 548         0x802c, 0xffffffff, 0xe0000000,
 549         0x9a60, 0xffffffff, 0x00000100,
 550         0x92a4, 0xffffffff, 0x00000100,
 551         0xc164, 0xffffffff, 0x00000100,
 552         0x9774, 0xffffffff, 0x00000100,
 553         0x8984, 0xffffffff, 0x06000100,
 554         0x8a18, 0xffffffff, 0x00000100,
 555         0x92a0, 0xffffffff, 0x00000100,
 556         0xc380, 0xffffffff, 0x00000100,
 557         0x8b28, 0xffffffff, 0x00000100,
 558         0x9144, 0xffffffff, 0x00000100,
 559         0x8d88, 0xffffffff, 0x00000100,
 560         0x8d8c, 0xffffffff, 0x00000100,
 561         0x9030, 0xffffffff, 0x00000100,
 562         0x9034, 0xffffffff, 0x00000100,
 563         0x9038, 0xffffffff, 0x00000100,
 564         0x903c, 0xffffffff, 0x00000100,
 565         0xad80, 0xffffffff, 0x00000100,
 566         0xac54, 0xffffffff, 0x00000100,
 567         0x897c, 0xffffffff, 0x06000100,
 568         0x9868, 0xffffffff, 0x00000100,
 569         0x9510, 0xffffffff, 0x00000100,
 570         0xaf04, 0xffffffff, 0x00000100,
 571         0xae04, 0xffffffff, 0x00000100,
 572         0x949c, 0xffffffff, 0x00000100,
 573         0x802c, 0xffffffff, 0xe0000000,
 574         0x9160, 0xffffffff, 0x00010000,
 575         0x9164, 0xffffffff, 0x00030002,
 576         0x9168, 0xffffffff, 0x00040007,
 577         0x916c, 0xffffffff, 0x00060005,
 578         0x9170, 0xffffffff, 0x00090008,
 579         0x9174, 0xffffffff, 0x00020001,
 580         0x9178, 0xffffffff, 0x00040003,
 581         0x917c, 0xffffffff, 0x00000007,
 582         0x9180, 0xffffffff, 0x00060005,
 583         0x9184, 0xffffffff, 0x00090008,
 584         0x9188, 0xffffffff, 0x00030002,
 585         0x918c, 0xffffffff, 0x00050004,
 586         0x9190, 0xffffffff, 0x00000008,
 587         0x9194, 0xffffffff, 0x00070006,
 588         0x9198, 0xffffffff, 0x000a0009,
 589         0x919c, 0xffffffff, 0x00040003,
 590         0x91a0, 0xffffffff, 0x00060005,
 591         0x91a4, 0xffffffff, 0x00000009,
 592         0x91a8, 0xffffffff, 0x00080007,
 593         0x91ac, 0xffffffff, 0x000b000a,
 594         0x91b0, 0xffffffff, 0x00050004,
 595         0x91b4, 0xffffffff, 0x00070006,
 596         0x91b8, 0xffffffff, 0x0008000b,
 597         0x91bc, 0xffffffff, 0x000a0009,
 598         0x91c0, 0xffffffff, 0x000d000c,
 599         0x91c4, 0xffffffff, 0x00060005,
 600         0x91c8, 0xffffffff, 0x00080007,
 601         0x91cc, 0xffffffff, 0x0000000b,
 602         0x91d0, 0xffffffff, 0x000a0009,
 603         0x91d4, 0xffffffff, 0x000d000c,
 604         0x91d8, 0xffffffff, 0x00070006,
 605         0x91dc, 0xffffffff, 0x00090008,
 606         0x91e0, 0xffffffff, 0x0000000c,
 607         0x91e4, 0xffffffff, 0x000b000a,
 608         0x91e8, 0xffffffff, 0x000e000d,
 609         0x91ec, 0xffffffff, 0x00080007,
 610         0x91f0, 0xffffffff, 0x000a0009,
 611         0x91f4, 0xffffffff, 0x0000000d,
 612         0x91f8, 0xffffffff, 0x000c000b,
 613         0x91fc, 0xffffffff, 0x000f000e,
 614         0x9200, 0xffffffff, 0x00090008,
 615         0x9204, 0xffffffff, 0x000b000a,
 616         0x9208, 0xffffffff, 0x000c000f,
 617         0x920c, 0xffffffff, 0x000e000d,
 618         0x9210, 0xffffffff, 0x00110010,
 619         0x9214, 0xffffffff, 0x000a0009,
 620         0x9218, 0xffffffff, 0x000c000b,
 621         0x921c, 0xffffffff, 0x0000000f,
 622         0x9220, 0xffffffff, 0x000e000d,
 623         0x9224, 0xffffffff, 0x00110010,
 624         0x9228, 0xffffffff, 0x000b000a,
 625         0x922c, 0xffffffff, 0x000d000c,
 626         0x9230, 0xffffffff, 0x00000010,
 627         0x9234, 0xffffffff, 0x000f000e,
 628         0x9238, 0xffffffff, 0x00120011,
 629         0x923c, 0xffffffff, 0x000c000b,
 630         0x9240, 0xffffffff, 0x000e000d,
 631         0x9244, 0xffffffff, 0x00000011,
 632         0x9248, 0xffffffff, 0x0010000f,
 633         0x924c, 0xffffffff, 0x00130012,
 634         0x9250, 0xffffffff, 0x000d000c,
 635         0x9254, 0xffffffff, 0x000f000e,
 636         0x9258, 0xffffffff, 0x00100013,
 637         0x925c, 0xffffffff, 0x00120011,
 638         0x9260, 0xffffffff, 0x00150014,
 639         0x9264, 0xffffffff, 0x000e000d,
 640         0x9268, 0xffffffff, 0x0010000f,
 641         0x926c, 0xffffffff, 0x00000013,
 642         0x9270, 0xffffffff, 0x00120011,
 643         0x9274, 0xffffffff, 0x00150014,
 644         0x9278, 0xffffffff, 0x000f000e,
 645         0x927c, 0xffffffff, 0x00110010,
 646         0x9280, 0xffffffff, 0x00000014,
 647         0x9284, 0xffffffff, 0x00130012,
 648         0x9288, 0xffffffff, 0x00160015,
 649         0x928c, 0xffffffff, 0x0010000f,
 650         0x9290, 0xffffffff, 0x00120011,
 651         0x9294, 0xffffffff, 0x00000015,
 652         0x9298, 0xffffffff, 0x00140013,
 653         0x929c, 0xffffffff, 0x00170016,
 654         0x9150, 0xffffffff, 0x96940200,
 655         0x8708, 0xffffffff, 0x00900100,
 656         0xc478, 0xffffffff, 0x00000080,
 657         0xc404, 0xffffffff, 0x0020003f,
 658         0x30, 0xffffffff, 0x0000001c,
 659         0x34, 0x000f0000, 0x000f0000,
 660         0x160c, 0xffffffff, 0x00000100,
 661         0x1024, 0xffffffff, 0x00000100,
 662         0x102c, 0x00000101, 0x00000000,
 663         0x20a8, 0xffffffff, 0x00000104,
 664         0x264c, 0x000c0000, 0x000c0000,
 665         0x2648, 0x000c0000, 0x000c0000,
 666         0x55e4, 0xff000fff, 0x00000100,
 667         0x55e8, 0x00000001, 0x00000001,
 668         0x2f50, 0x00000001, 0x00000001,
 669         0x30cc, 0xc0000fff, 0x00000104,
 670         0xc1e4, 0x00000001, 0x00000001,
 671         0xd0c0, 0xfffffff0, 0x00000100,
 672         0xd8c0, 0xfffffff0, 0x00000100
 673 };
 674
 675 static const u32 pitcairn_mgcg_cgcg_init[] =
 676 {
 677         0xc400, 0xffffffff, 0xfffffffc,
 678         0x802c, 0xffffffff, 0xe0000000,
 679         0x9a60, 0xffffffff, 0x00000100,
 680         0x92a4, 0xffffffff, 0x00000100,
 681         0xc164, 0xffffffff, 0x00000100,
 682         0x9774, 0xffffffff, 0x00000100,
 683         0x8984, 0xffffffff, 0x06000100,
 684         0x8a18, 0xffffffff, 0x00000100,
 685         0x92a0, 0xffffffff, 0x00000100,
 686         0xc380, 0xffffffff, 0x00000100,
 687         0x8b28, 0xffffffff, 0x00000100,
 688         0x9144, 0xffffffff, 0x00000100,
 689         0x8d88, 0xffffffff, 0x00000100,
 690         0x8d8c, 0xffffffff, 0x00000100,
 691         0x9030, 0xffffffff, 0x00000100,
 692         0x9034, 0xffffffff, 0x00000100,
 693         0x9038, 0xffffffff, 0x00000100,
 694         0x903c, 0xffffffff, 0x00000100,
 695         0xad80, 0xffffffff, 0x00000100,
 696         0xac54, 0xffffffff, 0x00000100,
 697         0x897c, 0xffffffff, 0x06000100,
 698         0x9868, 0xffffffff, 0x00000100,
 699         0x9510, 0xffffffff, 0x00000100,
 700         0xaf04, 0xffffffff, 0x00000100,
 701         0xae04, 0xffffffff, 0x00000100,
 702         0x949c, 0xffffffff, 0x00000100,
 703         0x802c, 0xffffffff, 0xe0000000,
 704         0x9160, 0xffffffff, 0x00010000,
 705         0x9164, 0xffffffff, 0x00030002,
 706         0x9168, 0xffffffff, 0x00040007,
 707         0x916c, 0xffffffff, 0x00060005,
 708         0x9170, 0xffffffff, 0x00090008,
 709         0x9174, 0xffffffff, 0x00020001,
 710         0x9178, 0xffffffff, 0x00040003,
 711         0x917c, 0xffffffff, 0x00000007,
 712         0x9180, 0xffffffff, 0x00060005,
 713         0x9184, 0xffffffff, 0x00090008,
 714         0x9188, 0xffffffff, 0x00030002,
 715         0x918c, 0xffffffff, 0x00050004,
 716         0x9190, 0xffffffff, 0x00000008,
 717         0x9194, 0xffffffff, 0x00070006,
 718         0x9198, 0xffffffff, 0x000a0009,
 719         0x919c, 0xffffffff, 0x00040003,
 720         0x91a0, 0xffffffff, 0x00060005,
 721         0x91a4, 0xffffffff, 0x00000009,
 722         0x91a8, 0xffffffff, 0x00080007,
 723         0x91ac, 0xffffffff, 0x000b000a,
 724         0x91b0, 0xffffffff, 0x00050004,
 725         0x91b4, 0xffffffff, 0x00070006,
 726         0x91b8, 0xffffffff, 0x0008000b,
 727         0x91bc, 0xffffffff, 0x000a0009,
 728         0x91c0, 0xffffffff, 0x000d000c,
 729         0x9200, 0xffffffff, 0x00090008,
 730         0x9204, 0xffffffff, 0x000b000a,
 731         0x9208, 0xffffffff, 0x000c000f,
 732         0x920c, 0xffffffff, 0x000e000d,
 733         0x9210, 0xffffffff, 0x00110010,
 734         0x9214, 0xffffffff, 0x000a0009,
 735         0x9218, 0xffffffff, 0x000c000b,
 736         0x921c, 0xffffffff, 0x0000000f,
 737         0x9220, 0xffffffff, 0x000e000d,
 738         0x9224, 0xffffffff, 0x00110010,
 739         0x9228, 0xffffffff, 0x000b000a,
 740         0x922c, 0xffffffff, 0x000d000c,
 741         0x9230, 0xffffffff, 0x00000010,
 742         0x9234, 0xffffffff, 0x000f000e,
 743         0x9238, 0xffffffff, 0x00120011,
 744         0x923c, 0xffffffff, 0x000c000b,
 745         0x9240, 0xffffffff, 0x000e000d,
 746         0x9244, 0xffffffff, 0x00000011,
 747         0x9248, 0xffffffff, 0x0010000f,
 748         0x924c, 0xffffffff, 0x00130012,
 749         0x9250, 0xffffffff, 0x000d000c,
 750         0x9254, 0xffffffff, 0x000f000e,
 751         0x9258, 0xffffffff, 0x00100013,
 752         0x925c, 0xffffffff, 0x00120011,
 753         0x9260, 0xffffffff, 0x00150014,
 754         0x9150, 0xffffffff, 0x96940200,
 755         0x8708, 0xffffffff, 0x00900100,
 756         0xc478, 0xffffffff, 0x00000080,
 757         0xc404, 0xffffffff, 0x0020003f,
 758         0x30, 0xffffffff, 0x0000001c,
 759         0x34, 0x000f0000, 0x000f0000,
 760         0x160c, 0xffffffff, 0x00000100,
 761         0x1024, 0xffffffff, 0x00000100,
 762         0x102c, 0x00000101, 0x00000000,
 763         0x20a8, 0xffffffff, 0x00000104,
 764         0x55e4, 0xff000fff, 0x00000100,
 765         0x55e8, 0x00000001, 0x00000001,
 766         0x2f50, 0x00000001, 0x00000001,
 767         0x30cc, 0xc0000fff, 0x00000104,
 768         0xc1e4, 0x00000001, 0x00000001,
 769         0xd0c0, 0xfffffff0, 0x00000100,
 770         0xd8c0, 0xfffffff0, 0x00000100
 771 };
 772
 773 static const u32 verde_mgcg_cgcg_init[] =
 774 {
 775         0xc400, 0xffffffff, 0xfffffffc,
 776         0x802c, 0xffffffff, 0xe0000000,
 777         0x9a60, 0xffffffff, 0x00000100,
 778         0x92a4, 0xffffffff, 0x00000100,
 779         0xc164, 0xffffffff, 0x00000100,
 780         0x9774, 0xffffffff, 0x00000100,
 781         0x8984, 0xffffffff, 0x06000100,
 782         0x8a18, 0xffffffff, 0x00000100,
 783         0x92a0, 0xffffffff, 0x00000100,
 784         0xc380, 0xffffffff, 0x00000100,
 785         0x8b28, 0xffffffff, 0x00000100,
 786         0x9144, 0xffffffff, 0x00000100,
 787         0x8d88, 0xffffffff, 0x00000100,
 788         0x8d8c, 0xffffffff, 0x00000100,
 789         0x9030, 0xffffffff, 0x00000100,
 790         0x9034, 0xffffffff, 0x00000100,
 791         0x9038, 0xffffffff, 0x00000100,
 792         0x903c, 0xffffffff, 0x00000100,
 793         0xad80, 0xffffffff, 0x00000100,
 794         0xac54, 0xffffffff, 0x00000100,
 795         0x897c, 0xffffffff, 0x06000100,
 796         0x9868, 0xffffffff, 0x00000100,
 797         0x9510, 0xffffffff, 0x00000100,
 798         0xaf04, 0xffffffff, 0x00000100,
 799         0xae04, 0xffffffff, 0x00000100,
 800         0x949c, 0xffffffff, 0x00000100,
 801         0x802c, 0xffffffff, 0xe0000000,
 802         0x9160, 0xffffffff, 0x00010000,
 803         0x9164, 0xffffffff, 0x00030002,
 804         0x9168, 0xffffffff, 0x00040007,
 805         0x916c, 0xffffffff, 0x00060005,
 806         0x9170, 0xffffffff, 0x00090008,
 807         0x9174, 0xffffffff, 0x00020001,
 808         0x9178, 0xffffffff, 0x00040003,
 809         0x917c, 0xffffffff, 0x00000007,
 810         0x9180, 0xffffffff, 0x00060005,
 811         0x9184, 0xffffffff, 0x00090008,
 812         0x9188, 0xffffffff, 0x00030002,
 813         0x918c, 0xffffffff, 0x00050004,
 814         0x9190, 0xffffffff, 0x00000008,
 815         0x9194, 0xffffffff, 0x00070006,
 816         0x9198, 0xffffffff, 0x000a0009,
 817         0x919c, 0xffffffff, 0x00040003,
 818         0x91a0, 0xffffffff, 0x00060005,
 819         0x91a4, 0xffffffff, 0x00000009,
 820         0x91a8, 0xffffffff, 0x00080007,
 821         0x91ac, 0xffffffff, 0x000b000a,
 822         0x91b0, 0xffffffff, 0x00050004,
 823         0x91b4, 0xffffffff, 0x00070006,
 824         0x91b8, 0xffffffff, 0x0008000b,
 825         0x91bc, 0xffffffff, 0x000a0009,
 826         0x91c0, 0xffffffff, 0x000d000c,
 827         0x9200, 0xffffffff, 0x00090008,
 828         0x9204, 0xffffffff, 0x000b000a,
 829         0x9208, 0xffffffff, 0x000c000f,
 830         0x920c, 0xffffffff, 0x000e000d,
 831         0x9210, 0xffffffff, 0x00110010,
 832         0x9214, 0xffffffff, 0x000a0009,
 833         0x9218, 0xffffffff, 0x000c000b,
 834         0x921c, 0xffffffff, 0x0000000f,
 835         0x9220, 0xffffffff, 0x000e000d,
 836         0x9224, 0xffffffff, 0x00110010,
 837         0x9228, 0xffffffff, 0x000b000a,
 838         0x922c, 0xffffffff, 0x000d000c,
 839         0x9230, 0xffffffff, 0x00000010,
 840         0x9234, 0xffffffff, 0x000f000e,
 841         0x9238, 0xffffffff, 0x00120011,
 842         0x923c, 0xffffffff, 0x000c000b,
 843         0x9240, 0xffffffff, 0x000e000d,
 844         0x9244, 0xffffffff, 0x00000011,
 845         0x9248, 0xffffffff, 0x0010000f,
 846         0x924c, 0xffffffff, 0x00130012,
 847         0x9250, 0xffffffff, 0x000d000c,
 848         0x9254, 0xffffffff, 0x000f000e,
 849         0x9258, 0xffffffff, 0x00100013,
 850         0x925c, 0xffffffff, 0x00120011,
 851         0x9260, 0xffffffff, 0x00150014,
 852         0x9150, 0xffffffff, 0x96940200,
 853         0x8708, 0xffffffff, 0x00900100,
 854         0xc478, 0xffffffff, 0x00000080,
 855         0xc404, 0xffffffff, 0x0020003f,
 856         0x30, 0xffffffff, 0x0000001c,
 857         0x34, 0x000f0000, 0x000f0000,
 858         0x160c, 0xffffffff, 0x00000100,
 859         0x1024, 0xffffffff, 0x00000100,
 860         0x102c, 0x00000101, 0x00000000,
 861         0x20a8, 0xffffffff, 0x00000104,
 862         0x264c, 0x000c0000, 0x000c0000,
 863         0x2648, 0x000c0000, 0x000c0000,
 864         0x55e4, 0xff000fff, 0x00000100,
 865         0x55e8, 0x00000001, 0x00000001,
 866         0x2f50, 0x00000001, 0x00000001,
 867         0x30cc, 0xc0000fff, 0x00000104,
 868         0xc1e4, 0x00000001, 0x00000001,
 869         0xd0c0, 0xfffffff0, 0x00000100,
 870         0xd8c0, 0xfffffff0, 0x00000100
 871 };
 872
 873 static const u32 oland_mgcg_cgcg_init[] =
 874 {
 875         0xc400, 0xffffffff, 0xfffffffc,
 876         0x802c, 0xffffffff, 0xe0000000,
 877         0x9a60, 0xffffffff, 0x00000100,
 878         0x92a4, 0xffffffff, 0x00000100,
 879         0xc164, 0xffffffff, 0x00000100,
 880         0x9774, 0xffffffff, 0x00000100,
 881         0x8984, 0xffffffff, 0x06000100,
 882         0x8a18, 0xffffffff, 0x00000100,
 883         0x92a0, 0xffffffff, 0x00000100,
 884         0xc380, 0xffffffff, 0x00000100,
 885         0x8b28, 0xffffffff, 0x00000100,
 886         0x9144, 0xffffffff, 0x00000100,
 887         0x8d88, 0xffffffff, 0x00000100,
 888         0x8d8c, 0xffffffff, 0x00000100,
 889         0x9030, 0xffffffff, 0x00000100,
 890         0x9034, 0xffffffff, 0x00000100,
 891         0x9038, 0xffffffff, 0x00000100,
 892         0x903c, 0xffffffff, 0x00000100,
 893         0xad80, 0xffffffff, 0x00000100,
 894         0xac54, 0xffffffff, 0x00000100,
 895         0x897c, 0xffffffff, 0x06000100,
 896         0x9868, 0xffffffff, 0x00000100,
 897         0x9510, 0xffffffff, 0x00000100,
 898         0xaf04, 0xffffffff, 0x00000100,
 899         0xae04, 0xffffffff, 0x00000100,
 900         0x949c, 0xffffffff, 0x00000100,
 901         0x802c, 0xffffffff, 0xe0000000,
 902         0x9160, 0xffffffff, 0x00010000,
 903         0x9164, 0xffffffff, 0x00030002,
 904         0x9168, 0xffffffff, 0x00040007,
 905         0x916c, 0xffffffff, 0x00060005,
 906         0x9170, 0xffffffff, 0x00090008,
 907         0x9174, 0xffffffff, 0x00020001,
 908         0x9178, 0xffffffff, 0x00040003,
 909         0x917c, 0xffffffff, 0x00000007,
 910         0x9180, 0xffffffff, 0x00060005,
 911         0x9184, 0xffffffff, 0x00090008,
 912         0x9188, 0xffffffff, 0x00030002,
 913         0x918c, 0xffffffff, 0x00050004,
 914         0x9190, 0xffffffff, 0x00000008,
 915         0x9194, 0xffffffff, 0x00070006,
 916         0x9198, 0xffffffff, 0x000a0009,
 917         0x919c, 0xffffffff, 0x00040003,
 918         0x91a0, 0xffffffff, 0x00060005,
 919         0x91a4, 0xffffffff, 0x00000009,
 920         0x91a8, 0xffffffff, 0x00080007,
 921         0x91ac, 0xffffffff, 0x000b000a,
 922         0x91b0, 0xffffffff, 0x00050004,
 923         0x91b4, 0xffffffff, 0x00070006,
 924         0x91b8, 0xffffffff, 0x0008000b,
 925         0x91bc, 0xffffffff, 0x000a0009,
 926         0x91c0, 0xffffffff, 0x000d000c,
 927         0x91c4, 0xffffffff, 0x00060005,
 928         0x91c8, 0xffffffff, 0x00080007,
 929         0x91cc, 0xffffffff, 0x0000000b,
 930         0x91d0, 0xffffffff, 0x000a0009,
 931         0x91d4, 0xffffffff, 0x000d000c,
 932         0x9150, 0xffffffff, 0x96940200,
 933         0x8708, 0xffffffff, 0x00900100,
 934         0xc478, 0xffffffff, 0x00000080,
 935         0xc404, 0xffffffff, 0x0020003f,
 936         0x30, 0xffffffff, 0x0000001c,
 937         0x34, 0x000f0000, 0x000f0000,
 938         0x160c, 0xffffffff, 0x00000100,
 939         0x1024, 0xffffffff, 0x00000100,
 940         0x102c, 0x00000101, 0x00000000,
 941         0x20a8, 0xffffffff, 0x00000104,
 942         0x264c, 0x000c0000, 0x000c0000,
 943         0x2648, 0x000c0000, 0x000c0000,
 944         0x55e4, 0xff000fff, 0x00000100,
 945         0x55e8, 0x00000001, 0x00000001,
 946         0x2f50, 0x00000001, 0x00000001,
 947         0x30cc, 0xc0000fff, 0x00000104,
 948         0xc1e4, 0x00000001, 0x00000001,
 949         0xd0c0, 0xfffffff0, 0x00000100,
 950         0xd8c0, 0xfffffff0, 0x00000100
 951 };
 952
 953 static const u32 hainan_mgcg_cgcg_init[] =
 954 {
 955         0xc400, 0xffffffff, 0xfffffffc,
 956         0x802c, 0xffffffff, 0xe0000000,
 957         0x9a60, 0xffffffff, 0x00000100,
 958         0x92a4, 0xffffffff, 0x00000100,
 959         0xc164, 0xffffffff, 0x00000100,
 960         0x9774, 0xffffffff, 0x00000100,
 961         0x8984, 0xffffffff, 0x06000100,
 962         0x8a18, 0xffffffff, 0x00000100,
 963         0x92a0, 0xffffffff, 0x00000100,
 964         0xc380, 0xffffffff, 0x00000100,
 965         0x8b28, 0xffffffff, 0x00000100,
 966         0x9144, 0xffffffff, 0x00000100,
 967         0x8d88, 0xffffffff, 0x00000100,
 968         0x8d8c, 0xffffffff, 0x00000100,
 969         0x9030, 0xffffffff, 0x00000100,
 970         0x9034, 0xffffffff, 0x00000100,
 971         0x9038, 0xffffffff, 0x00000100,
 972         0x903c, 0xffffffff, 0x00000100,
 973         0xad80, 0xffffffff, 0x00000100,
 974         0xac54, 0xffffffff, 0x00000100,
 975         0x897c, 0xffffffff, 0x06000100,
 976         0x9868, 0xffffffff, 0x00000100,
 977         0x9510, 0xffffffff, 0x00000100,
 978         0xaf04, 0xffffffff, 0x00000100,
 979         0xae04, 0xffffffff, 0x00000100,
 980         0x949c, 0xffffffff, 0x00000100,
 981         0x802c, 0xffffffff, 0xe0000000,
 982         0x9160, 0xffffffff, 0x00010000,
 983         0x9164, 0xffffffff, 0x00030002,
 984         0x9168, 0xffffffff, 0x00040007,
 985         0x916c, 0xffffffff, 0x00060005,
 986         0x9170, 0xffffffff, 0x00090008,
 987         0x9174, 0xffffffff, 0x00020001,
 988         0x9178, 0xffffffff, 0x00040003,
 989         0x917c, 0xffffffff, 0x00000007,
 990         0x9180, 0xffffffff, 0x00060005,
 991         0x9184, 0xffffffff, 0x00090008,
 992         0x9188, 0xffffffff, 0x00030002,
 993         0x918c, 0xffffffff, 0x00050004,
 994         0x9190, 0xffffffff, 0x00000008,
 995         0x9194, 0xffffffff, 0x00070006,
 996         0x9198, 0xffffffff, 0x000a0009,
 997         0x919c, 0xffffffff, 0x00040003,
 998         0x91a0, 0xffffffff, 0x00060005,
 999         0x91a4, 0xffffffff, 0x00000009,
1000         0x91a8, 0xffffffff, 0x00080007,
1001         0x91ac, 0xffffffff, 0x000b000a,
1002         0x91b0, 0xffffffff, 0x00050004,
1003         0x91b4, 0xffffffff, 0x00070006,
1004         0x91b8, 0xffffffff, 0x0008000b,
1005         0x91bc, 0xffffffff, 0x000a0009,
1006         0x91c0, 0xffffffff, 0x000d000c,
1007         0x91c4, 0xffffffff, 0x00060005,
1008         0x91c8, 0xffffffff, 0x00080007,
1009         0x91cc, 0xffffffff, 0x0000000b,
1010         0x91d0, 0xffffffff, 0x000a0009,
1011         0x91d4, 0xffffffff, 0x000d000c,
1012         0x9150, 0xffffffff, 0x96940200,
1013         0x8708, 0xffffffff, 0x00900100,
1014         0xc478, 0xffffffff, 0x00000080,
1015         0xc404, 0xffffffff, 0x0020003f,
1016         0x30, 0xffffffff, 0x0000001c,
1017         0x34, 0x000f0000, 0x000f0000,
1018         0x160c, 0xffffffff, 0x00000100,
1019         0x1024, 0xffffffff, 0x00000100,
1020         0x20a8, 0xffffffff, 0x00000104,
1021         0x264c, 0x000c0000, 0x000c0000,
1022         0x2648, 0x000c0000, 0x000c0000,
1023         0x2f50, 0x00000001, 0x00000001,
1024         0x30cc, 0xc0000fff, 0x00000104,
1025         0xc1e4, 0x00000001, 0x00000001,
1026         0xd0c0, 0xfffffff0, 0x00000100,
1027         0xd8c0, 0xfffffff0, 0x00000100
1028 };
1029
1030 static u32 verde_pg_init[] =
1031 {
1032         0x353c, 0xffffffff, 0x40000,
1033         0x3538, 0xffffffff, 0x200010ff,
1034         0x353c, 0xffffffff, 0x0,
1035         0x353c, 0xffffffff, 0x0,
1036         0x353c, 0xffffffff, 0x0,
1037         0x353c, 0xffffffff, 0x0,
1038         0x353c, 0xffffffff, 0x0,
1039         0x353c, 0xffffffff, 0x7007,
1040         0x3538, 0xffffffff, 0x300010ff,
1041         0x353c, 0xffffffff, 0x0,
1042         0x353c, 0xffffffff, 0x0,
1043         0x353c, 0xffffffff, 0x0,
1044         0x353c, 0xffffffff, 0x0,
1045         0x353c, 0xffffffff, 0x0,
1046         0x353c, 0xffffffff, 0x400000,
1047         0x3538, 0xffffffff, 0x100010ff,
1048         0x353c, 0xffffffff, 0x0,
1049         0x353c, 0xffffffff, 0x0,
1050         0x353c, 0xffffffff, 0x0,
1051         0x353c, 0xffffffff, 0x0,
1052         0x353c, 0xffffffff, 0x0,
1053         0x353c, 0xffffffff, 0x120200,
1054         0x3538, 0xffffffff, 0x500010ff,
1055         0x353c, 0xffffffff, 0x0,
1056         0x353c, 0xffffffff, 0x0,
1057         0x353c, 0xffffffff, 0x0,
1058         0x353c, 0xffffffff, 0x0,
1059         0x353c, 0xffffffff, 0x0,
1060         0x353c, 0xffffffff, 0x1e1e16,
1061         0x3538, 0xffffffff, 0x600010ff,
1062         0x353c, 0xffffffff, 0x0,
1063         0x353c, 0xffffffff, 0x0,
1064         0x353c, 0xffffffff, 0x0,
1065         0x353c, 0xffffffff, 0x0,
1066         0x353c, 0xffffffff, 0x0,
1067         0x353c, 0xffffffff, 0x171f1e,
1068         0x3538, 0xffffffff, 0x700010ff,
1069         0x353c, 0xffffffff, 0x0,
1070         0x353c, 0xffffffff, 0x0,
1071         0x353c, 0xffffffff, 0x0,
1072         0x353c, 0xffffffff, 0x0,
1073         0x353c, 0xffffffff, 0x0,
1074         0x353c, 0xffffffff, 0x0,
1075         0x3538, 0xffffffff, 0x9ff,
1076         0x3500, 0xffffffff, 0x0,
1077         0x3504, 0xffffffff, 0x10000800,
1078         0x3504, 0xffffffff, 0xf,
1079         0x3504, 0xffffffff, 0xf,
1080         0x3500, 0xffffffff, 0x4,
1081         0x3504, 0xffffffff, 0x1000051e,
1082         0x3504, 0xffffffff, 0xffff,
1083         0x3504, 0xffffffff, 0xffff,
1084         0x3500, 0xffffffff, 0x8,
1085         0x3504, 0xffffffff, 0x80500,
1086         0x3500, 0xffffffff, 0x12,
1087         0x3504, 0xffffffff, 0x9050c,
1088         0x3500, 0xffffffff, 0x1d,
1089         0x3504, 0xffffffff, 0xb052c,
1090         0x3500, 0xffffffff, 0x2a,
1091         0x3504, 0xffffffff, 0x1053e,
1092         0x3500, 0xffffffff, 0x2d,
1093         0x3504, 0xffffffff, 0x10546,
1094         0x3500, 0xffffffff, 0x30,
1095         0x3504, 0xffffffff, 0xa054e,
1096         0x3500, 0xffffffff, 0x3c,
1097         0x3504, 0xffffffff, 0x1055f,
1098         0x3500, 0xffffffff, 0x3f,
1099         0x3504, 0xffffffff, 0x10567,
1100         0x3500, 0xffffffff, 0x42,
1101         0x3504, 0xffffffff, 0x1056f,
1102         0x3500, 0xffffffff, 0x45,
1103         0x3504, 0xffffffff, 0x10572,
1104         0x3500, 0xffffffff, 0x48,
1105         0x3504, 0xffffffff, 0x20575,
1106         0x3500, 0xffffffff, 0x4c,
1107         0x3504, 0xffffffff, 0x190801,
1108         0x3500, 0xffffffff, 0x67,
1109         0x3504, 0xffffffff, 0x1082a,
1110         0x3500, 0xffffffff, 0x6a,
1111         0x3504, 0xffffffff, 0x1b082d,
1112         0x3500, 0xffffffff, 0x87,
1113         0x3504, 0xffffffff, 0x310851,
1114         0x3500, 0xffffffff, 0xba,
1115         0x3504, 0xffffffff, 0x891,
1116         0x3500, 0xffffffff, 0xbc,
1117         0x3504, 0xffffffff, 0x893,
1118         0x3500, 0xffffffff, 0xbe,
1119         0x3504, 0xffffffff, 0x20895,
1120         0x3500, 0xffffffff, 0xc2,
1121         0x3504, 0xffffffff, 0x20899,
1122         0x3500, 0xffffffff, 0xc6,
1123         0x3504, 0xffffffff, 0x2089d,
1124         0x3500, 0xffffffff, 0xca,
1125         0x3504, 0xffffffff, 0x8a1,
1126         0x3500, 0xffffffff, 0xcc,
1127         0x3504, 0xffffffff, 0x8a3,
1128         0x3500, 0xffffffff, 0xce,
1129         0x3504, 0xffffffff, 0x308a5,
1130         0x3500, 0xffffffff, 0xd3,
1131         0x3504, 0xffffffff, 0x6d08cd,
1132         0x3500, 0xffffffff, 0x142,
1133         0x3504, 0xffffffff, 0x2000095a,
1134         0x3504, 0xffffffff, 0x1,
1135         0x3500, 0xffffffff, 0x144,
1136         0x3504, 0xffffffff, 0x301f095b,
1137         0x3500, 0xffffffff, 0x165,
1138         0x3504, 0xffffffff, 0xc094d,
1139         0x3500, 0xffffffff, 0x173,
1140         0x3504, 0xffffffff, 0xf096d,
1141         0x3500, 0xffffffff, 0x184,
1142         0x3504, 0xffffffff, 0x15097f,
1143         0x3500, 0xffffffff, 0x19b,
1144         0x3504, 0xffffffff, 0xc0998,
1145         0x3500, 0xffffffff, 0x1a9,
1146         0x3504, 0xffffffff, 0x409a7,
1147         0x3500, 0xffffffff, 0x1af,
1148         0x3504, 0xffffffff, 0xcdc,
1149         0x3500, 0xffffffff, 0x1b1,
1150         0x3504, 0xffffffff, 0x800,
1151         0x3508, 0xffffffff, 0x6c9b2000,
1152         0x3510, 0xfc00, 0x2000,
1153         0x3544, 0xffffffff, 0xfc0,
1154         0x28d4, 0x00000100, 0x100
1155 };
1156
1157 static void si_init_golden_registers(struct radeon_device *rdev)
1158 {
1159         switch (rdev->family) {
1160         case CHIP_TAHITI:
1161                 radeon_program_register_sequence(rdev,
1162                                                  tahiti_golden_registers,
1163                                                  (const u32)ARRAY_SIZE(tahiti_golden_registers));
1164                 radeon_program_register_sequence(rdev,
1165                                                  tahiti_golden_rlc_registers,
1166                                                  (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1167                 radeon_program_register_sequence(rdev,
1168                                                  tahiti_mgcg_cgcg_init,
1169                                                  (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1170                 radeon_program_register_sequence(rdev,
1171                                                  tahiti_golden_registers2,
1172                                                  (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1173                 break;
1174         case CHIP_PITCAIRN:
1175                 radeon_program_register_sequence(rdev,
1176                                                  pitcairn_golden_registers,
1177                                                  (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1178                 radeon_program_register_sequence(rdev,
1179                                                  pitcairn_golden_rlc_registers,
1180                                                  (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1181                 radeon_program_register_sequence(rdev,
1182                                                  pitcairn_mgcg_cgcg_init,
1183                                                  (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1184                 break;
1185         case CHIP_VERDE:
1186                 radeon_program_register_sequence(rdev,
1187                                                  verde_golden_registers,
1188                                                  (const u32)ARRAY_SIZE(verde_golden_registers));
1189                 radeon_program_register_sequence(rdev,
1190                                                  verde_golden_rlc_registers,
1191                                                  (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1192                 radeon_program_register_sequence(rdev,
1193                                                  verde_mgcg_cgcg_init,
1194                                                  (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1195                 radeon_program_register_sequence(rdev,
1196                                                  verde_pg_init,
1197                                                  (const u32)ARRAY_SIZE(verde_pg_init));
1198                 break;
1199         case CHIP_OLAND:
1200                 radeon_program_register_sequence(rdev,
1201                                                  oland_golden_registers,
1202                                                  (const u32)ARRAY_SIZE(oland_golden_registers));
1203                 radeon_program_register_sequence(rdev,
1204                                                  oland_golden_rlc_registers,
1205                                                  (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1206                 radeon_program_register_sequence(rdev,
1207                                                  oland_mgcg_cgcg_init,
1208                                                  (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1209                 break;
1210         case CHIP_HAINAN:
1211                 radeon_program_register_sequence(rdev,
1212                                                  hainan_golden_registers,
1213                                                  (const u32)ARRAY_SIZE(hainan_golden_registers));
1214                 radeon_program_register_sequence(rdev,
1215                                                  hainan_golden_registers2,
1216                                                  (const u32)ARRAY_SIZE(hainan_golden_registers2));
1217                 radeon_program_register_sequence(rdev,
1218                                                  hainan_mgcg_cgcg_init,
1219                                                  (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1220                 break;
1221         default:
1222                 break;
1223         }
1224 }
1225
1226 #define PCIE_BUS_CLK                10000
1227 #define TCLK                        (PCIE_BUS_CLK / 10)
1228
1229 /**
1230  * si_get_xclk - get the xclk
1231  *
1232  * @rdev: radeon_device pointer
1233  *
1234  * Returns the reference clock used by the gfx engine
1235  * (SI).
1236  */
1237 u32 si_get_xclk(struct radeon_device *rdev)
1238 {
1239         u32 reference_clock = rdev->clock.spll.reference_freq;
1240         u32 tmp;
1241
1242         tmp = RREG32(CG_CLKPIN_CNTL_2);
1243         if (tmp & MUX_TCLK_TO_XCLK)
1244                 return TCLK;
1245
1246         tmp = RREG32(CG_CLKPIN_CNTL);
1247         if (tmp & XTALIN_DIVIDE)
1248                 return reference_clock / 4;
1249
1250         return reference_clock;
1251 }
1252
1253 /* get temperature in millidegrees */
1254 int si_get_temp(struct radeon_device *rdev)
1255 {
1256         u32 temp;
1257         int actual_temp = 0;
1258
1259         temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1260                 CTF_TEMP_SHIFT;
1261
1262         if (temp & 0x200)
1263                 actual_temp = 255;
1264         else
1265                 actual_temp = temp & 0x1ff;
1266
1267         actual_temp = (actual_temp * 1000);
1268
1269         return actual_temp;
1270 }
1271
1272 #define TAHITI_IO_MC_REGS_SIZE 36
1273
1274 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1275         {0x0000006f, 0x03044000},
1276         {0x00000070, 0x0480c018},
1277         {0x00000071, 0x00000040},
1278         {0x00000072, 0x01000000},
1279         {0x00000074, 0x000000ff},
1280         {0x00000075, 0x00143400},
1281         {0x00000076, 0x08ec0800},
1282         {0x00000077, 0x040000cc},
1283         {0x00000079, 0x00000000},
1284         {0x0000007a, 0x21000409},
1285         {0x0000007c, 0x00000000},
1286         {0x0000007d, 0xe8000000},
1287         {0x0000007e, 0x044408a8},
1288         {0x0000007f, 0x00000003},
1289         {0x00000080, 0x00000000},
1290         {0x00000081, 0x01000000},
1291         {0x00000082, 0x02000000},
1292         {0x00000083, 0x00000000},
1293         {0x00000084, 0xe3f3e4f4},
1294         {0x00000085, 0x00052024},
1295         {0x00000087, 0x00000000},
1296         {0x00000088, 0x66036603},
1297         {0x00000089, 0x01000000},
1298         {0x0000008b, 0x1c0a0000},
1299         {0x0000008c, 0xff010000},
1300         {0x0000008e, 0xffffefff},
1301         {0x0000008f, 0xfff3efff},
1302         {0x00000090, 0xfff3efbf},
1303         {0x00000094, 0x00101101},
1304         {0x00000095, 0x00000fff},
1305         {0x00000096, 0x00116fff},
1306         {0x00000097, 0x60010000},
1307         {0x00000098, 0x10010000},
1308         {0x00000099, 0x00006000},
1309         {0x0000009a, 0x00001000},
1310         {0x0000009f, 0x00a77400}
1311 };
1312
1313 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1314         {0x0000006f, 0x03044000},
1315         {0x00000070, 0x0480c018},
1316         {0x00000071, 0x00000040},
1317         {0x00000072, 0x01000000},
1318         {0x00000074, 0x000000ff},
1319         {0x00000075, 0x00143400},
1320         {0x00000076, 0x08ec0800},
1321         {0x00000077, 0x040000cc},
1322         {0x00000079, 0x00000000},
1323         {0x0000007a, 0x21000409},
1324         {0x0000007c, 0x00000000},
1325         {0x0000007d, 0xe8000000},
1326         {0x0000007e, 0x044408a8},
1327         {0x0000007f, 0x00000003},
1328         {0x00000080, 0x00000000},
1329         {0x00000081, 0x01000000},
1330         {0x00000082, 0x02000000},
1331         {0x00000083, 0x00000000},
1332         {0x00000084, 0xe3f3e4f4},
1333         {0x00000085, 0x00052024},
1334         {0x00000087, 0x00000000},
1335         {0x00000088, 0x66036603},
1336         {0x00000089, 0x01000000},
1337         {0x0000008b, 0x1c0a0000},
1338         {0x0000008c, 0xff010000},
1339         {0x0000008e, 0xffffefff},
1340         {0x0000008f, 0xfff3efff},
1341         {0x00000090, 0xfff3efbf},
1342         {0x00000094, 0x00101101},
1343         {0x00000095, 0x00000fff},
1344         {0x00000096, 0x00116fff},
1345         {0x00000097, 0x60010000},
1346         {0x00000098, 0x10010000},
1347         {0x00000099, 0x00006000},
1348         {0x0000009a, 0x00001000},
1349         {0x0000009f, 0x00a47400}
1350 };
1351
1352 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1353         {0x0000006f, 0x03044000},
1354         {0x00000070, 0x0480c018},
1355         {0x00000071, 0x00000040},
1356         {0x00000072, 0x01000000},
1357         {0x00000074, 0x000000ff},
1358         {0x00000075, 0x00143400},
1359         {0x00000076, 0x08ec0800},
1360         {0x00000077, 0x040000cc},
1361         {0x00000079, 0x00000000},
1362         {0x0000007a, 0x21000409},
1363         {0x0000007c, 0x00000000},
1364         {0x0000007d, 0xe8000000},
1365         {0x0000007e, 0x044408a8},
1366         {0x0000007f, 0x00000003},
1367         {0x00000080, 0x00000000},
1368         {0x00000081, 0x01000000},
1369         {0x00000082, 0x02000000},
1370         {0x00000083, 0x00000000},
1371         {0x00000084, 0xe3f3e4f4},
1372         {0x00000085, 0x00052024},
1373         {0x00000087, 0x00000000},
1374         {0x00000088, 0x66036603},
1375         {0x00000089, 0x01000000},
1376         {0x0000008b, 0x1c0a0000},
1377         {0x0000008c, 0xff010000},
1378         {0x0000008e, 0xffffefff},
1379         {0x0000008f, 0xfff3efff},
1380         {0x00000090, 0xfff3efbf},
1381         {0x00000094, 0x00101101},
1382         {0x00000095, 0x00000fff},
1383         {0x00000096, 0x00116fff},
1384         {0x00000097, 0x60010000},
1385         {0x00000098, 0x10010000},
1386         {0x00000099, 0x00006000},
1387         {0x0000009a, 0x00001000},
1388         {0x0000009f, 0x00a37400}
1389 };
1390
1391 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1392         {0x0000006f, 0x03044000},
1393         {0x00000070, 0x0480c018},
1394         {0x00000071, 0x00000040},
1395         {0x00000072, 0x01000000},
1396         {0x00000074, 0x000000ff},
1397         {0x00000075, 0x00143400},
1398         {0x00000076, 0x08ec0800},
1399         {0x00000077, 0x040000cc},
1400         {0x00000079, 0x00000000},
1401         {0x0000007a, 0x21000409},
1402         {0x0000007c, 0x00000000},
1403         {0x0000007d, 0xe8000000},
1404         {0x0000007e, 0x044408a8},
1405         {0x0000007f, 0x00000003},
1406         {0x00000080, 0x00000000},
1407         {0x00000081, 0x01000000},
1408         {0x00000082, 0x02000000},
1409         {0x00000083, 0x00000000},
1410         {0x00000084, 0xe3f3e4f4},
1411         {0x00000085, 0x00052024},
1412         {0x00000087, 0x00000000},
1413         {0x00000088, 0x66036603},
1414         {0x00000089, 0x01000000},
1415         {0x0000008b, 0x1c0a0000},
1416         {0x0000008c, 0xff010000},
1417         {0x0000008e, 0xffffefff},
1418         {0x0000008f, 0xfff3efff},
1419         {0x00000090, 0xfff3efbf},
1420         {0x00000094, 0x00101101},
1421         {0x00000095, 0x00000fff},
1422         {0x00000096, 0x00116fff},
1423         {0x00000097, 0x60010000},
1424         {0x00000098, 0x10010000},
1425         {0x00000099, 0x00006000},
1426         {0x0000009a, 0x00001000},
1427         {0x0000009f, 0x00a17730}
1428 };
1429
1430 static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1431         {0x0000006f, 0x03044000},
1432         {0x00000070, 0x0480c018},
1433         {0x00000071, 0x00000040},
1434         {0x00000072, 0x01000000},
1435         {0x00000074, 0x000000ff},
1436         {0x00000075, 0x00143400},
1437         {0x00000076, 0x08ec0800},
1438         {0x00000077, 0x040000cc},
1439         {0x00000079, 0x00000000},
1440         {0x0000007a, 0x21000409},
1441         {0x0000007c, 0x00000000},
1442         {0x0000007d, 0xe8000000},
1443         {0x0000007e, 0x044408a8},
1444         {0x0000007f, 0x00000003},
1445         {0x00000080, 0x00000000},
1446         {0x00000081, 0x01000000},
1447         {0x00000082, 0x02000000},
1448         {0x00000083, 0x00000000},
1449         {0x00000084, 0xe3f3e4f4},
1450         {0x00000085, 0x00052024},
1451         {0x00000087, 0x00000000},
1452         {0x00000088, 0x66036603},
1453         {0x00000089, 0x01000000},
1454         {0x0000008b, 0x1c0a0000},
1455         {0x0000008c, 0xff010000},
1456         {0x0000008e, 0xffffefff},
1457         {0x0000008f, 0xfff3efff},
1458         {0x00000090, 0xfff3efbf},
1459         {0x00000094, 0x00101101},
1460         {0x00000095, 0x00000fff},
1461         {0x00000096, 0x00116fff},
1462         {0x00000097, 0x60010000},
1463         {0x00000098, 0x10010000},
1464         {0x00000099, 0x00006000},
1465         {0x0000009a, 0x00001000},
1466         {0x0000009f, 0x00a07730}
1467 };
1468
1469 /* ucode loading */
1470 int si_mc_load_microcode(struct radeon_device *rdev)
1471 {
1472         const __be32 *fw_data;
1473         u32 running, blackout = 0;
1474         u32 *io_mc_regs;
1475         int i, regs_size, ucode_size = rdev->mc_fw->size / 4;
1476
1477         if (!rdev->mc_fw)
1478                 return -EINVAL;
1479
1480         switch (rdev->family) {
1481         case CHIP_TAHITI:
1482                 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1483                 regs_size = TAHITI_IO_MC_REGS_SIZE;
1484                 break;
1485         case CHIP_PITCAIRN:
1486                 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1487                 regs_size = TAHITI_IO_MC_REGS_SIZE;
1488                 break;
1489         case CHIP_VERDE:
1490         default:
1491                 io_mc_regs = (u32 *)&verde_io_mc_regs;
1492                 regs_size = TAHITI_IO_MC_REGS_SIZE;
1493                 break;
1494         case CHIP_OLAND:
1495                 io_mc_regs = (u32 *)&oland_io_mc_regs;
1496                 regs_size = TAHITI_IO_MC_REGS_SIZE;
1497                 break;
1498         case CHIP_HAINAN:
1499                 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1500                 regs_size = TAHITI_IO_MC_REGS_SIZE;
1501                 break;
1502         }
1503
1504         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1505
1506         if (running == 0) {
1507                 if (running) {
1508                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1509                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1510                 }
1511
1512                 /* reset the engine and set to writable */
1513                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1514                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1515
1516                 /* load mc io regs */
1517                 for (i = 0; i < regs_size; i++) {
1518                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1519                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1520                 }
1521                 /* load the MC ucode */
1522                 fw_data = (const __be32 *)rdev->mc_fw->data;
1523                 for (i = 0; i < ucode_size; i++)
1524                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1525
1526                 /* put the engine back into the active state */
1527                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1528                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1529                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1530
1531                 /* wait for training to complete */
1532                 for (i = 0; i < rdev->usec_timeout; i++) {
1533                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1534                                 break;
1535                         udelay(1);
1536                 }
1537                 for (i = 0; i < rdev->usec_timeout; i++) {
1538                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1539                                 break;
1540                         udelay(1);
1541                 }
1542
1543                 if (running)
1544                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1545         }
1546
1547         return 0;
1548 }
1549
1550 static int si_init_microcode(struct radeon_device *rdev)
1551 {
1552         const char *chip_name;
1553         const char *rlc_chip_name;
1554         size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1555         size_t smc_req_size, mc2_req_size;
1556         char fw_name[30];
1557         int err;
1558
1559         DRM_DEBUG("\n");
1560
1561         switch (rdev->family) {
1562         case CHIP_TAHITI:
1563                 chip_name = "TAHITI";
1564                 rlc_chip_name = "TAHITI";
1565                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1566                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1567                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1568                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1569                 mc_req_size = SI_MC_UCODE_SIZE * 4;
1570                 mc2_req_size = TAHITI_MC_UCODE_SIZE * 4;
1571                 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
1572                 break;
1573         case CHIP_PITCAIRN:
1574                 chip_name = "PITCAIRN";
1575                 rlc_chip_name = "PITCAIRN";
1576                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1577                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1578                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1579                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1580                 mc_req_size = SI_MC_UCODE_SIZE * 4;
1581                 mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4;
1582                 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
1583                 break;
1584         case CHIP_VERDE:
1585                 chip_name = "VERDE";
1586                 rlc_chip_name = "VERDE";
1587                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1588                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1589                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1590                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1591                 mc_req_size = SI_MC_UCODE_SIZE * 4;
1592                 mc2_req_size = VERDE_MC_UCODE_SIZE * 4;
1593                 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
1594                 break;
1595         case CHIP_OLAND:
1596                 chip_name = "OLAND";
1597                 rlc_chip_name = "OLAND";
1598                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1599                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1600                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1601                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1602                 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1603                 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
1604                 break;
1605         case CHIP_HAINAN:
1606                 chip_name = "HAINAN";
1607                 rlc_chip_name = "HAINAN";
1608                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1609                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1610                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1611                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1612                 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1613                 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
1614                 break;
1615         default: BUG();
1616         }
1617
1618         DRM_INFO("Loading %s Microcode\n", chip_name);
1619
1620         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1621         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1622         if (err)
1623                 goto out;
1624         if (rdev->pfp_fw->size != pfp_req_size) {
1625                 printk(KERN_ERR
1626                        "si_cp: Bogus length %zu in firmware \"%s\"\n",
1627                        rdev->pfp_fw->size, fw_name);
1628                 err = -EINVAL;
1629                 goto out;
1630         }
1631
1632         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1633         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1634         if (err)
1635                 goto out;
1636         if (rdev->me_fw->size != me_req_size) {
1637                 printk(KERN_ERR
1638                        "si_cp: Bogus length %zu in firmware \"%s\"\n",
1639                        rdev->me_fw->size, fw_name);
1640                 err = -EINVAL;
1641         }
1642
1643         snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1644         err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1645         if (err)
1646                 goto out;
1647         if (rdev->ce_fw->size != ce_req_size) {
1648                 printk(KERN_ERR
1649                        "si_cp: Bogus length %zu in firmware \"%s\"\n",
1650                        rdev->ce_fw->size, fw_name);
1651                 err = -EINVAL;
1652         }
1653
1654         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
1655         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1656         if (err)
1657                 goto out;
1658         if (rdev->rlc_fw->size != rlc_req_size) {
1659                 printk(KERN_ERR
1660                        "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1661                        rdev->rlc_fw->size, fw_name);
1662                 err = -EINVAL;
1663         }
1664
1665         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1666         err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1667         if (err) {
1668                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1669                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1670                 if (err)
1671                         goto out;
1672         }
1673         if ((rdev->mc_fw->size != mc_req_size) &&
1674             (rdev->mc_fw->size != mc2_req_size)) {
1675                 printk(KERN_ERR
1676                        "si_mc: Bogus length %zu in firmware \"%s\"\n",
1677                        rdev->mc_fw->size, fw_name);
1678                 err = -EINVAL;
1679         }
1680         DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1681
1682         snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1683         err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1684         if (err) {
1685                 printk(KERN_ERR
1686                        "smc: error loading firmware \"%s\"\n",
1687                        fw_name);
1688                 release_firmware(rdev->smc_fw);
1689                 rdev->smc_fw = NULL;
1690                 err = 0;
1691         } else if (rdev->smc_fw->size != smc_req_size) {
1692                 printk(KERN_ERR
1693                        "si_smc: Bogus length %zu in firmware \"%s\"\n",
1694                        rdev->smc_fw->size, fw_name);
1695                 err = -EINVAL;
1696         }
1697
1698 out:
1699         if (err) {
1700                 if (err != -EINVAL)
1701                         printk(KERN_ERR
1702                                "si_cp: Failed to load firmware \"%s\"\n",
1703                                fw_name);
1704                 release_firmware(rdev->pfp_fw);
1705                 rdev->pfp_fw = NULL;
1706                 release_firmware(rdev->me_fw);
1707                 rdev->me_fw = NULL;
1708                 release_firmware(rdev->ce_fw);
1709                 rdev->ce_fw = NULL;
1710                 release_firmware(rdev->rlc_fw);
1711                 rdev->rlc_fw = NULL;
1712                 release_firmware(rdev->mc_fw);
1713                 rdev->mc_fw = NULL;
1714                 release_firmware(rdev->smc_fw);
1715                 rdev->smc_fw = NULL;
1716         }
1717         return err;
1718 }
1719
1720 /* watermark setup */
1721 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1722                                    struct radeon_crtc *radeon_crtc,
1723                                    struct drm_display_mode *mode,
1724                                    struct drm_display_mode *other_mode)
1725 {
1726         u32 tmp, buffer_alloc, i;
1727         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1728         /*
1729          * Line Buffer Setup
1730          * There are 3 line buffers, each one shared by 2 display controllers.
1731          * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1732          * the display controllers.  The paritioning is done via one of four
1733          * preset allocations specified in bits 21:20:
1734          *  0 - half lb
1735          *  2 - whole lb, other crtc must be disabled
1736          */
1737         /* this can get tricky if we have two large displays on a paired group
1738          * of crtcs.  Ideally for multiple large displays we'd assign them to
1739          * non-linked crtcs for maximum line buffer allocation.
1740          */
1741         if (radeon_crtc->base.enabled && mode) {
1742                 if (other_mode) {
1743                         tmp = 0; /* 1/2 */
1744                         buffer_alloc = 1;
1745                 } else {
1746                         tmp = 2; /* whole */
1747                         buffer_alloc = 2;
1748                 }
1749         } else {
1750                 tmp = 0;
1751                 buffer_alloc = 0;
1752         }
1753
1754         WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1755                DC_LB_MEMORY_CONFIG(tmp));
1756
1757         WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1758                DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1759         for (i = 0; i < rdev->usec_timeout; i++) {
1760                 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1761                     DMIF_BUFFERS_ALLOCATED_COMPLETED)
1762                         break;
1763                 udelay(1);
1764         }
1765
1766         if (radeon_crtc->base.enabled && mode) {
1767                 switch (tmp) {
1768                 case 0:
1769                 default:
1770                         return 4096 * 2;
1771                 case 2:
1772                         return 8192 * 2;
1773                 }
1774         }
1775
1776         /* controller not enabled, so no lb used */
1777         return 0;
1778 }
1779
1780 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
1781 {
1782         u32 tmp = RREG32(MC_SHARED_CHMAP);
1783
1784         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1785         case 0:
1786         default:
1787                 return 1;
1788         case 1:
1789                 return 2;
1790         case 2:
1791                 return 4;
1792         case 3:
1793                 return 8;
1794         case 4:
1795                 return 3;
1796         case 5:
1797                 return 6;
1798         case 6:
1799                 return 10;
1800         case 7:
1801                 return 12;
1802         case 8:
1803                 return 16;
1804         }
1805 }
1806
1807 struct dce6_wm_params {
1808         u32 dram_channels; /* number of dram channels */
1809         u32 yclk;          /* bandwidth per dram data pin in kHz */
1810         u32 sclk;          /* engine clock in kHz */
1811         u32 disp_clk;      /* display clock in kHz */
1812         u32 src_width;     /* viewport width */
1813         u32 active_time;   /* active display time in ns */
1814         u32 blank_time;    /* blank time in ns */
1815         bool interlaced;    /* mode is interlaced */
1816         fixed20_12 vsc;    /* vertical scale ratio */
1817         u32 num_heads;     /* number of active crtcs */
1818         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1819         u32 lb_size;       /* line buffer allocated to pipe */
1820         u32 vtaps;         /* vertical scaler taps */
1821 };
1822
1823 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1824 {
1825         /* Calculate raw DRAM Bandwidth */
1826         fixed20_12 dram_efficiency; /* 0.7 */
1827         fixed20_12 yclk, dram_channels, bandwidth;
1828         fixed20_12 a;
1829
1830         a.full = dfixed_const(1000);
1831         yclk.full = dfixed_const(wm->yclk);
1832         yclk.full = dfixed_div(yclk, a);
1833         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1834         a.full = dfixed_const(10);
1835         dram_efficiency.full = dfixed_const(7);
1836         dram_efficiency.full = dfixed_div(dram_efficiency, a);
1837         bandwidth.full = dfixed_mul(dram_channels, yclk);
1838         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1839
1840         return dfixed_trunc(bandwidth);
1841 }
1842
1843 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1844 {
1845         /* Calculate DRAM Bandwidth and the part allocated to display. */
1846         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1847         fixed20_12 yclk, dram_channels, bandwidth;
1848         fixed20_12 a;
1849
1850         a.full = dfixed_const(1000);
1851         yclk.full = dfixed_const(wm->yclk);
1852         yclk.full = dfixed_div(yclk, a);
1853         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1854         a.full = dfixed_const(10);
1855         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1856         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1857         bandwidth.full = dfixed_mul(dram_channels, yclk);
1858         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1859
1860         return dfixed_trunc(bandwidth);
1861 }
1862
1863 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1864 {
1865         /* Calculate the display Data return Bandwidth */
1866         fixed20_12 return_efficiency; /* 0.8 */
1867         fixed20_12 sclk, bandwidth;
1868         fixed20_12 a;
1869
1870         a.full = dfixed_const(1000);
1871         sclk.full = dfixed_const(wm->sclk);
1872         sclk.full = dfixed_div(sclk, a);
1873         a.full = dfixed_const(10);
1874         return_efficiency.full = dfixed_const(8);
1875         return_efficiency.full = dfixed_div(return_efficiency, a);
1876         a.full = dfixed_const(32);
1877         bandwidth.full = dfixed_mul(a, sclk);
1878         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1879
1880         return dfixed_trunc(bandwidth);
1881 }
1882
1883 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1884 {
1885         return 32;
1886 }
1887
1888 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1889 {
1890         /* Calculate the DMIF Request Bandwidth */
1891         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1892         fixed20_12 disp_clk, sclk, bandwidth;
1893         fixed20_12 a, b1, b2;
1894         u32 min_bandwidth;
1895
1896         a.full = dfixed_const(1000);
1897         disp_clk.full = dfixed_const(wm->disp_clk);
1898         disp_clk.full = dfixed_div(disp_clk, a);
1899         a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1900         b1.full = dfixed_mul(a, disp_clk);
1901
1902         a.full = dfixed_const(1000);
1903         sclk.full = dfixed_const(wm->sclk);
1904         sclk.full = dfixed_div(sclk, a);
1905         a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1906         b2.full = dfixed_mul(a, sclk);
1907
1908         a.full = dfixed_const(10);
1909         disp_clk_request_efficiency.full = dfixed_const(8);
1910         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1911
1912         min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1913
1914         a.full = dfixed_const(min_bandwidth);
1915         bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1916
1917         return dfixed_trunc(bandwidth);
1918 }
1919
1920 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1921 {
1922         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1923         u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1924         u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1925         u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1926
1927         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1928 }
1929
1930 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1931 {
1932         /* Calculate the display mode Average Bandwidth
1933          * DisplayMode should contain the source and destination dimensions,
1934          * timing, etc.
1935          */
1936         fixed20_12 bpp;
1937         fixed20_12 line_time;
1938         fixed20_12 src_width;
1939         fixed20_12 bandwidth;
1940         fixed20_12 a;
1941
1942         a.full = dfixed_const(1000);
1943         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1944         line_time.full = dfixed_div(line_time, a);
1945         bpp.full = dfixed_const(wm->bytes_per_pixel);
1946         src_width.full = dfixed_const(wm->src_width);
1947         bandwidth.full = dfixed_mul(src_width, bpp);
1948         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1949         bandwidth.full = dfixed_div(bandwidth, line_time);
1950
1951         return dfixed_trunc(bandwidth);
1952 }
1953
1954 static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1955 {
1956         /* First calcualte the latency in ns */
1957         u32 mc_latency = 2000; /* 2000 ns. */
1958         u32 available_bandwidth = dce6_available_bandwidth(wm);
1959         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1960         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1961         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1962         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1963                 (wm->num_heads * cursor_line_pair_return_time);
1964         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1965         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1966         u32 tmp, dmif_size = 12288;
1967         fixed20_12 a, b, c;
1968
1969         if (wm->num_heads == 0)
1970                 return 0;
1971
1972         a.full = dfixed_const(2);
1973         b.full = dfixed_const(1);
1974         if ((wm->vsc.full > a.full) ||
1975             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1976             (wm->vtaps >= 5) ||
1977             ((wm->vsc.full >= a.full) && wm->interlaced))
1978                 max_src_lines_per_dst_line = 4;
1979         else
1980                 max_src_lines_per_dst_line = 2;
1981
1982         a.full = dfixed_const(available_bandwidth);
1983         b.full = dfixed_const(wm->num_heads);
1984         a.full = dfixed_div(a, b);
1985
1986         b.full = dfixed_const(mc_latency + 512);
1987         c.full = dfixed_const(wm->disp_clk);
1988         b.full = dfixed_div(b, c);
1989
1990         c.full = dfixed_const(dmif_size);
1991         b.full = dfixed_div(c, b);
1992
1993         tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1994
1995         b.full = dfixed_const(1000);
1996         c.full = dfixed_const(wm->disp_clk);
1997         b.full = dfixed_div(c, b);
1998         c.full = dfixed_const(wm->bytes_per_pixel);
1999         b.full = dfixed_mul(b, c);
2000
2001         lb_fill_bw = min(tmp, dfixed_trunc(b));
2002
2003         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2004         b.full = dfixed_const(1000);
2005         c.full = dfixed_const(lb_fill_bw);
2006         b.full = dfixed_div(c, b);
2007         a.full = dfixed_div(a, b);
2008         line_fill_time = dfixed_trunc(a);
2009
2010         if (line_fill_time < wm->active_time)
2011                 return latency;
2012         else
2013                 return latency + (line_fill_time - wm->active_time);
2014
2015 }
2016
2017 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2018 {
2019         if (dce6_average_bandwidth(wm) <=
2020             (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2021                 return true;
2022         else
2023                 return false;
2024 };
2025
2026 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2027 {
2028         if (dce6_average_bandwidth(wm) <=
2029             (dce6_available_bandwidth(wm) / wm->num_heads))
2030                 return true;
2031         else
2032                 return false;
2033 };
2034
2035 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2036 {
2037         u32 lb_partitions = wm->lb_size / wm->src_width;
2038         u32 line_time = wm->active_time + wm->blank_time;
2039         u32 latency_tolerant_lines;
2040         u32 latency_hiding;
2041         fixed20_12 a;
2042
2043         a.full = dfixed_const(1);
2044         if (wm->vsc.full > a.full)
2045                 latency_tolerant_lines = 1;
2046         else {
2047                 if (lb_partitions <= (wm->vtaps + 1))
2048                         latency_tolerant_lines = 1;
2049                 else
2050                         latency_tolerant_lines = 2;
2051         }
2052
2053         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2054
2055         if (dce6_latency_watermark(wm) <= latency_hiding)
2056                 return true;
2057         else
2058                 return false;
2059 }
2060
2061 static void dce6_program_watermarks(struct radeon_device *rdev,
2062                                          struct radeon_crtc *radeon_crtc,
2063                                          u32 lb_size, u32 num_heads)
2064 {
2065         struct drm_display_mode *mode = &radeon_crtc->base.mode;
2066         struct dce6_wm_params wm_low, wm_high;
2067         u32 dram_channels;
2068         u32 pixel_period;
2069         u32 line_time = 0;
2070         u32 latency_watermark_a = 0, latency_watermark_b = 0;
2071         u32 priority_a_mark = 0, priority_b_mark = 0;
2072         u32 priority_a_cnt = PRIORITY_OFF;
2073         u32 priority_b_cnt = PRIORITY_OFF;
2074         u32 tmp, arb_control3;
2075         fixed20_12 a, b, c;
2076
2077         if (radeon_crtc->base.enabled && num_heads && mode) {
2078                 pixel_period = 1000000 / (u32)mode->clock;
2079                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2080                 priority_a_cnt = 0;
2081                 priority_b_cnt = 0;
2082
2083                 if (rdev->family == CHIP_ARUBA)
2084                         dram_channels = evergreen_get_number_of_dram_channels(rdev);
2085                 else
2086                         dram_channels = si_get_number_of_dram_channels(rdev);
2087
2088                 /* watermark for high clocks */
2089                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2090                         wm_high.yclk =
2091                                 radeon_dpm_get_mclk(rdev, false) * 10;
2092                         wm_high.sclk =
2093                                 radeon_dpm_get_sclk(rdev, false) * 10;
2094                 } else {
2095                         wm_high.yclk = rdev->pm.current_mclk * 10;
2096                         wm_high.sclk = rdev->pm.current_sclk * 10;
2097                 }
2098
2099                 wm_high.disp_clk = mode->clock;
2100                 wm_high.src_width = mode->crtc_hdisplay;
2101                 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2102                 wm_high.blank_time = line_time - wm_high.active_time;
2103                 wm_high.interlaced = false;
2104                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2105                         wm_high.interlaced = true;
2106                 wm_high.vsc = radeon_crtc->vsc;
2107                 wm_high.vtaps = 1;
2108                 if (radeon_crtc->rmx_type != RMX_OFF)
2109                         wm_high.vtaps = 2;
2110                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2111                 wm_high.lb_size = lb_size;
2112                 wm_high.dram_channels = dram_channels;
2113                 wm_high.num_heads = num_heads;
2114
2115                 /* watermark for low clocks */
2116                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2117                         wm_low.yclk =
2118                                 radeon_dpm_get_mclk(rdev, true) * 10;
2119                         wm_low.sclk =
2120                                 radeon_dpm_get_sclk(rdev, true) * 10;
2121                 } else {
2122                         wm_low.yclk = rdev->pm.current_mclk * 10;
2123                         wm_low.sclk = rdev->pm.current_sclk * 10;
2124                 }
2125
2126                 wm_low.disp_clk = mode->clock;
2127                 wm_low.src_width = mode->crtc_hdisplay;
2128                 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2129                 wm_low.blank_time = line_time - wm_low.active_time;
2130                 wm_low.interlaced = false;
2131                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2132                         wm_low.interlaced = true;
2133                 wm_low.vsc = radeon_crtc->vsc;
2134                 wm_low.vtaps = 1;
2135                 if (radeon_crtc->rmx_type != RMX_OFF)
2136                         wm_low.vtaps = 2;
2137                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2138                 wm_low.lb_size = lb_size;
2139                 wm_low.dram_channels = dram_channels;
2140                 wm_low.num_heads = num_heads;
2141
2142                 /* set for high clocks */
2143                 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2144                 /* set for low clocks */
2145                 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2146
2147                 /* possibly force display priority to high */
2148                 /* should really do this at mode validation time... */
2149                 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2150                     !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2151                     !dce6_check_latency_hiding(&wm_high) ||
2152                     (rdev->disp_priority == 2)) {
2153                         DRM_DEBUG_KMS("force priority to high\n");
2154                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2155                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2156                 }
2157                 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2158                     !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2159                     !dce6_check_latency_hiding(&wm_low) ||
2160                     (rdev->disp_priority == 2)) {
2161                         DRM_DEBUG_KMS("force priority to high\n");
2162                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2163                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2164                 }
2165
2166                 a.full = dfixed_const(1000);
2167                 b.full = dfixed_const(mode->clock);
2168                 b.full = dfixed_div(b, a);
2169                 c.full = dfixed_const(latency_watermark_a);
2170                 c.full = dfixed_mul(c, b);
2171                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2172                 c.full = dfixed_div(c, a);
2173                 a.full = dfixed_const(16);
2174                 c.full = dfixed_div(c, a);
2175                 priority_a_mark = dfixed_trunc(c);
2176                 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2177
2178                 a.full = dfixed_const(1000);
2179                 b.full = dfixed_const(mode->clock);
2180                 b.full = dfixed_div(b, a);
2181                 c.full = dfixed_const(latency_watermark_b);
2182                 c.full = dfixed_mul(c, b);
2183                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2184                 c.full = dfixed_div(c, a);
2185                 a.full = dfixed_const(16);
2186                 c.full = dfixed_div(c, a);
2187                 priority_b_mark = dfixed_trunc(c);
2188                 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2189         }
2190
2191         /* select wm A */
2192         arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2193         tmp = arb_control3;
2194         tmp &= ~LATENCY_WATERMARK_MASK(3);
2195         tmp |= LATENCY_WATERMARK_MASK(1);
2196         WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2197         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2198                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2199                 LATENCY_HIGH_WATERMARK(line_time)));
2200         /* select wm B */
2201         tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2202         tmp &= ~LATENCY_WATERMARK_MASK(3);
2203         tmp |= LATENCY_WATERMARK_MASK(2);
2204         WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2205         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2206                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2207                 LATENCY_HIGH_WATERMARK(line_time)));
2208         /* restore original selection */
2209         WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2210
2211         /* write the priority marks */
2212         WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2213         WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2214
2215         /* save values for DPM */
2216         radeon_crtc->line_time = line_time;
2217         radeon_crtc->wm_high = latency_watermark_a;
2218         radeon_crtc->wm_low = latency_watermark_b;
2219 }
2220
2221 void dce6_bandwidth_update(struct radeon_device *rdev)
2222 {
2223         struct drm_display_mode *mode0 = NULL;
2224         struct drm_display_mode *mode1 = NULL;
2225         u32 num_heads = 0, lb_size;
2226         int i;
2227
2228         radeon_update_display_priority(rdev);
2229
2230         for (i = 0; i < rdev->num_crtc; i++) {
2231                 if (rdev->mode_info.crtcs[i]->base.enabled)
2232                         num_heads++;
2233         }
2234         for (i = 0; i < rdev->num_crtc; i += 2) {
2235                 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2236                 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2237                 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2238                 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2239                 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2240                 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2241         }
2242 }
2243
2244 /*
2245  * Core functions
2246  */
2247 static void si_tiling_mode_table_init(struct radeon_device *rdev)
2248 {
2249         const u32 num_tile_mode_states = 32;
2250         u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2251
2252         switch (rdev->config.si.mem_row_size_in_kb) {
2253         case 1:
2254                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2255                 break;
2256         case 2:
2257         default:
2258                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2259                 break;
2260         case 4:
2261                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2262                 break;
2263         }
2264
2265         if ((rdev->family == CHIP_TAHITI) ||
2266             (rdev->family == CHIP_PITCAIRN)) {
2267                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2268                         switch (reg_offset) {
2269                         case 0:  /* non-AA compressed depth or any compressed stencil */
2270                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2271                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2272                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2273                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2274                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2275                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2276                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2277                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2278                                 break;
2279                         case 1:  /* 2xAA/4xAA compressed depth only */
2280                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2281                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2282                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2283                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2284                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2285                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2286                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2287                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2288                                 break;
2289                         case 2:  /* 8xAA compressed depth only */
2290                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2291                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2292                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2293                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2294                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2295                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2296                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2297                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2298                                 break;
2299                         case 3:  /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2300                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2301                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2302                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2303                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2304                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2305                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2306                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2307                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2308                                 break;
2309                         case 4:  /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2310                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2311                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2312                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2313                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2314                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2315                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2316                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2317                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2318                                 break;
2319                         case 5:  /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2320                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2321                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2322                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2323                                                  TILE_SPLIT(split_equal_to_row_size) |
2324                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2325                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2326                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2327                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2328                                 break;
2329                         case 6:  /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2330                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2331                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2332                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2333                                                  TILE_SPLIT(split_equal_to_row_size) |
2334                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2335                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2336                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2337                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2338                                 break;
2339                         case 7:  /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2340                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2341                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2342                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2343                                                  TILE_SPLIT(split_equal_to_row_size) |
2344                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2345                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2346                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2347                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2348                                 break;
2349                         case 8:  /* 1D and 1D Array Surfaces */
2350                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2351                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2352                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2353                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2354                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2355                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2356                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2357                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2358                                 break;
2359                         case 9:  /* Displayable maps. */
2360                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2361                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2362                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2363                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2364                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2365                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2366                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2367                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2368                                 break;
2369                         case 10:  /* Display 8bpp. */
2370                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2371                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2372                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2373                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2374                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2375                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2376                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2377                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2378                                 break;
2379                         case 11:  /* Display 16bpp. */
2380                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2381                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2382                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2383                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2384                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2385                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2386                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2387                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2388                                 break;
2389                         case 12:  /* Display 32bpp. */
2390                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2391                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2392                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2393                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2394                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2395                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2396                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2397                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2398                                 break;
2399                         case 13:  /* Thin. */
2400                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2401                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2402                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2403                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2404                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2405                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2406                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2407                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2408                                 break;
2409                         case 14:  /* Thin 8 bpp. */
2410                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2411                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2412                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2413                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2414                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2415                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2416                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2417                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2418                                 break;
2419                         case 15:  /* Thin 16 bpp. */
2420                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2421                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2422                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2423                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2424                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2425                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2426                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2427                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2428                                 break;
2429                         case 16:  /* Thin 32 bpp. */
2430                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2431                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2432                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2433                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2434                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2435                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2436                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2437                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2438                                 break;
2439                         case 17:  /* Thin 64 bpp. */
2440                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2441                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2442                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2443                                                  TILE_SPLIT(split_equal_to_row_size) |
2444                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2445                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2446                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2447                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2448                                 break;
2449                         case 21:  /* 8 bpp PRT. */
2450                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2451                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2452                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2453                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2454                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2455                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2456                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2457                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2458                                 break;
2459                         case 22:  /* 16 bpp PRT */
2460                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2461                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2462                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2463                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2464                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2465                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2466                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2467                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2468                                 break;
2469                         case 23:  /* 32 bpp PRT */
2470                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2471                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2472                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2473                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2474                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2475                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2476                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2477                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2478                                 break;
2479                         case 24:  /* 64 bpp PRT */
2480                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2481                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2482                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2483                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2484                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2485                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2486                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2487                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2488                                 break;
2489                         case 25:  /* 128 bpp PRT */
2490                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2491                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2492                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2493                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2494                                                  NUM_BANKS(ADDR_SURF_8_BANK) |
2495                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2496                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2497                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2498                                 break;
2499                         default:
2500                                 gb_tile_moden = 0;
2501                                 break;
2502                         }
2503                         rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2504                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2505                 }
2506         } else if ((rdev->family == CHIP_VERDE) ||
2507                    (rdev->family == CHIP_OLAND) ||
2508                    (rdev->family == CHIP_HAINAN)) {
2509                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2510                         switch (reg_offset) {
2511                         case 0:  /* non-AA compressed depth or any compressed stencil */
2512                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2513                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2514                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2515                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2516                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2517                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2518                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2519                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2520                                 break;
2521                         case 1:  /* 2xAA/4xAA compressed depth only */
2522                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2523                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2524                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2525                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2526                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2527                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2528                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2529                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2530                                 break;
2531                         case 2:  /* 8xAA compressed depth only */
2532                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2533                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2534                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2535                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2536                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2537                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2538                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2539                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2540                                 break;
2541                         case 3:  /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2542                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2543                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2544                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2545                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2546                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2547                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2548                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2549                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2550                                 break;
2551                         case 4:  /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2552                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2553                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2554                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2555                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2556                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2557                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2558                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2559                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2560                                 break;
2561                         case 5:  /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2562                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2563                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2564                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2565                                                  TILE_SPLIT(split_equal_to_row_size) |
2566                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2567                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2568                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2569                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2570                                 break;
2571                         case 6:  /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2572                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2573                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2574                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2575                                                  TILE_SPLIT(split_equal_to_row_size) |
2576                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2577                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2578                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2579                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2580                                 break;
2581                         case 7:  /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2582                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2583                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2584                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2585                                                  TILE_SPLIT(split_equal_to_row_size) |
2586                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2587                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2588                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2589                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2590                                 break;
2591                         case 8:  /* 1D and 1D Array Surfaces */
2592                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2593                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2594                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2595                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2596                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2597                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2598                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2599                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2600                                 break;
2601                         case 9:  /* Displayable maps. */
2602                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2603                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2604                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2605                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2606                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2607                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2608                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2609                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2610                                 break;
2611                         case 10:  /* Display 8bpp. */
2612                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2613                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2614                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2615                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2616                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2617                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2618                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2619                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2620                                 break;
2621                         case 11:  /* Display 16bpp. */
2622                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2623                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2624                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2625                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2626                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2627                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2628                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2629                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2630                                 break;
2631                         case 12:  /* Display 32bpp. */
2632                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2633                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2634                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2635                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2636                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2637                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2638                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2639                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2640                                 break;
2641                         case 13:  /* Thin. */
2642                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2643                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2644                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2645                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2646                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2647                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2648                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2649                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2650                                 break;
2651                         case 14:  /* Thin 8 bpp. */
2652                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2653                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2654                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2655                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2656                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2657                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2658                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2659                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2660                                 break;
2661                         case 15:  /* Thin 16 bpp. */
2662                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2663                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2664                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2665                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2666                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2667                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2668                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2669                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2670                                 break;
2671                         case 16:  /* Thin 32 bpp. */
2672                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2673                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2674                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2675                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2676                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2677                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2678                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2679                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2680                                 break;
2681                         case 17:  /* Thin 64 bpp. */
2682                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2683                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2684                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2685                                                  TILE_SPLIT(split_equal_to_row_size) |
2686                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2687                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2688                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2689                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2690                                 break;
2691                         case 21:  /* 8 bpp PRT. */
2692                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2693                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2694                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2695                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2696                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2697                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2698                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2699                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2700                                 break;
2701                         case 22:  /* 16 bpp PRT */
2702                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2703                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2704                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2705                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2706                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2707                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2708                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2709                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2710                                 break;
2711                         case 23:  /* 32 bpp PRT */
2712                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2713                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2714                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2715                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2716                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2717                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2718                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2719                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2720                                 break;
2721                         case 24:  /* 64 bpp PRT */
2722                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2723                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2724                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2725                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2726                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2727                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2728                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2729                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2730                                 break;
2731                         case 25:  /* 128 bpp PRT */
2732                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2733                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2734                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2735                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2736                                                  NUM_BANKS(ADDR_SURF_8_BANK) |
2737                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2738                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2739                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2740                                 break;
2741                         default:
2742                                 gb_tile_moden = 0;
2743                                 break;
2744                         }
2745                         rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2746                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2747                 }
2748         } else
2749                 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2750 }
2751
2752 static void si_select_se_sh(struct radeon_device *rdev,
2753                             u32 se_num, u32 sh_num)
2754 {
2755         u32 data = INSTANCE_BROADCAST_WRITES;
2756
2757         if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2758                 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2759         else if (se_num == 0xffffffff)
2760                 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2761         else if (sh_num == 0xffffffff)
2762                 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2763         else
2764                 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2765         WREG32(GRBM_GFX_INDEX, data);
2766 }
2767
2768 static u32 si_create_bitmask(u32 bit_width)
2769 {
2770         u32 i, mask = 0;
2771
2772         for (i = 0; i < bit_width; i++) {
2773                 mask <<= 1;
2774                 mask |= 1;
2775         }
2776         return mask;
2777 }
2778
2779 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2780 {
2781         u32 data, mask;
2782
2783         data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2784         if (data & 1)
2785                 data &= INACTIVE_CUS_MASK;
2786         else
2787                 data = 0;
2788         data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2789
2790         data >>= INACTIVE_CUS_SHIFT;
2791
2792         mask = si_create_bitmask(cu_per_sh);
2793
2794         return ~data & mask;
2795 }
2796
2797 static void si_setup_spi(struct radeon_device *rdev,
2798                          u32 se_num, u32 sh_per_se,
2799                          u32 cu_per_sh)
2800 {
2801         int i, j, k;
2802         u32 data, mask, active_cu;
2803
2804         for (i = 0; i < se_num; i++) {
2805                 for (j = 0; j < sh_per_se; j++) {
2806                         si_select_se_sh(rdev, i, j);
2807                         data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2808                         active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2809
2810                         mask = 1;
2811                         for (k = 0; k < 16; k++) {
2812                                 mask <<= k;
2813                                 if (active_cu & mask) {
2814                                         data &= ~mask;
2815                                         WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2816                                         break;
2817                                 }
2818                         }
2819                 }
2820         }
2821         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2822 }
2823
2824 static u32 si_get_rb_disabled(struct radeon_device *rdev,
2825                               u32 max_rb_num_per_se,
2826                               u32 sh_per_se)
2827 {
2828         u32 data, mask;
2829
2830         data = RREG32(CC_RB_BACKEND_DISABLE);
2831         if (data & 1)
2832                 data &= BACKEND_DISABLE_MASK;
2833         else
2834                 data = 0;
2835         data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2836
2837         data >>= BACKEND_DISABLE_SHIFT;
2838
2839         mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
2840
2841         return data & mask;
2842 }
2843
2844 static void si_setup_rb(struct radeon_device *rdev,
2845                         u32 se_num, u32 sh_per_se,
2846                         u32 max_rb_num_per_se)
2847 {
2848         int i, j;
2849         u32 data, mask;
2850         u32 disabled_rbs = 0;
2851         u32 enabled_rbs = 0;
2852
2853         for (i = 0; i < se_num; i++) {
2854                 for (j = 0; j < sh_per_se; j++) {
2855                         si_select_se_sh(rdev, i, j);
2856                         data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
2857                         disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2858                 }
2859         }
2860         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2861
2862         mask = 1;
2863         for (i = 0; i < max_rb_num_per_se * se_num; i++) {
2864                 if (!(disabled_rbs & mask))
2865                         enabled_rbs |= mask;
2866                 mask <<= 1;
2867         }
2868
2869         rdev->config.si.backend_enable_mask = enabled_rbs;
2870
2871         for (i = 0; i < se_num; i++) {
2872                 si_select_se_sh(rdev, i, 0xffffffff);
2873                 data = 0;
2874                 for (j = 0; j < sh_per_se; j++) {
2875                         switch (enabled_rbs & 3) {
2876                         case 1:
2877                                 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2878                                 break;
2879                         case 2:
2880                                 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2881                                 break;
2882                         case 3:
2883                         default:
2884                                 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2885                                 break;
2886                         }
2887                         enabled_rbs >>= 2;
2888                 }
2889                 WREG32(PA_SC_RASTER_CONFIG, data);
2890         }
2891         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2892 }
2893
2894 static void si_gpu_init(struct radeon_device *rdev)
2895 {
2896         u32 gb_addr_config = 0;
2897         u32 mc_shared_chmap, mc_arb_ramcfg;
2898         u32 sx_debug_1;
2899         u32 hdp_host_path_cntl;
2900         u32 tmp;
2901         int i, j;
2902
2903         switch (rdev->family) {
2904         case CHIP_TAHITI:
2905                 rdev->config.si.max_shader_engines = 2;
2906                 rdev->config.si.max_tile_pipes = 12;
2907                 rdev->config.si.max_cu_per_sh = 8;
2908                 rdev->config.si.max_sh_per_se = 2;
2909                 rdev->config.si.max_backends_per_se = 4;
2910                 rdev->config.si.max_texture_channel_caches = 12;
2911                 rdev->config.si.max_gprs = 256;
2912                 rdev->config.si.max_gs_threads = 32;
2913                 rdev->config.si.max_hw_contexts = 8;
2914
2915                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2916                 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2917                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2918                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2919                 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2920                 break;
2921         case CHIP_PITCAIRN:
2922                 rdev->config.si.max_shader_engines = 2;
2923                 rdev->config.si.max_tile_pipes = 8;
2924                 rdev->config.si.max_cu_per_sh = 5;
2925                 rdev->config.si.max_sh_per_se = 2;
2926                 rdev->config.si.max_backends_per_se = 4;
2927                 rdev->config.si.max_texture_channel_caches = 8;
2928                 rdev->config.si.max_gprs = 256;
2929                 rdev->config.si.max_gs_threads = 32;
2930                 rdev->config.si.max_hw_contexts = 8;
2931
2932                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2933                 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2934                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2935                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2936                 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2937                 break;
2938         case CHIP_VERDE:
2939         default:
2940                 rdev->config.si.max_shader_engines = 1;
2941                 rdev->config.si.max_tile_pipes = 4;
2942                 rdev->config.si.max_cu_per_sh = 5;
2943                 rdev->config.si.max_sh_per_se = 2;
2944                 rdev->config.si.max_backends_per_se = 4;
2945                 rdev->config.si.max_texture_channel_caches = 4;
2946                 rdev->config.si.max_gprs = 256;
2947                 rdev->config.si.max_gs_threads = 32;
2948                 rdev->config.si.max_hw_contexts = 8;
2949
2950                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2951                 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2952                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2953                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2954                 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2955                 break;
2956         case CHIP_OLAND:
2957                 rdev->config.si.max_shader_engines = 1;
2958                 rdev->config.si.max_tile_pipes = 4;
2959                 rdev->config.si.max_cu_per_sh = 6;
2960                 rdev->config.si.max_sh_per_se = 1;
2961                 rdev->config.si.max_backends_per_se = 2;
2962                 rdev->config.si.max_texture_channel_caches = 4;
2963                 rdev->config.si.max_gprs = 256;
2964                 rdev->config.si.max_gs_threads = 16;
2965                 rdev->config.si.max_hw_contexts = 8;
2966
2967                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2968                 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2969                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2970                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2971                 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2972                 break;
2973         case CHIP_HAINAN:
2974                 rdev->config.si.max_shader_engines = 1;
2975                 rdev->config.si.max_tile_pipes = 4;
2976                 rdev->config.si.max_cu_per_sh = 5;
2977                 rdev->config.si.max_sh_per_se = 1;
2978                 rdev->config.si.max_backends_per_se = 1;
2979                 rdev->config.si.max_texture_channel_caches = 2;
2980                 rdev->config.si.max_gprs = 256;
2981                 rdev->config.si.max_gs_threads = 16;
2982                 rdev->config.si.max_hw_contexts = 8;
2983
2984                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2985                 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2986                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2987                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2988                 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
2989                 break;
2990         }
2991
2992         /* Initialize HDP */
2993         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2994                 WREG32((0x2c14 + j), 0x00000000);
2995                 WREG32((0x2c18 + j), 0x00000000);
2996                 WREG32((0x2c1c + j), 0x00000000);
2997                 WREG32((0x2c20 + j), 0x00000000);
2998                 WREG32((0x2c24 + j), 0x00000000);
2999         }
3000
3001         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3002
3003         evergreen_fix_pci_max_read_req_size(rdev);
3004
3005         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3006
3007         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3008         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3009
3010         rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
3011         rdev->config.si.mem_max_burst_length_bytes = 256;
3012         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3013         rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3014         if (rdev->config.si.mem_row_size_in_kb > 4)
3015                 rdev->config.si.mem_row_size_in_kb = 4;
3016         /* XXX use MC settings? */
3017         rdev->config.si.shader_engine_tile_size = 32;
3018         rdev->config.si.num_gpus = 1;
3019         rdev->config.si.multi_gpu_tile_size = 64;
3020
3021         /* fix up row size */
3022         gb_addr_config &= ~ROW_SIZE_MASK;
3023         switch (rdev->config.si.mem_row_size_in_kb) {
3024         case 1:
3025         default:
3026                 gb_addr_config |= ROW_SIZE(0);
3027                 break;
3028         case 2:
3029                 gb_addr_config |= ROW_SIZE(1);
3030                 break;
3031         case 4:
3032                 gb_addr_config |= ROW_SIZE(2);
3033                 break;
3034         }
3035
3036         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3037          * not have bank info, so create a custom tiling dword.
3038          * bits 3:0   num_pipes
3039          * bits 7:4   num_banks
3040          * bits 11:8  group_size
3041          * bits 15:12 row_size
3042          */
3043         rdev->config.si.tile_config = 0;
3044         switch (rdev->config.si.num_tile_pipes) {
3045         case 1:
3046                 rdev->config.si.tile_config |= (0 << 0);
3047                 break;
3048         case 2:
3049                 rdev->config.si.tile_config |= (1 << 0);
3050                 break;
3051         case 4:
3052                 rdev->config.si.tile_config |= (2 << 0);
3053                 break;
3054         case 8:
3055         default:
3056                 /* XXX what about 12? */
3057                 rdev->config.si.tile_config |= (3 << 0);
3058                 break;
3059         }
3060         switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3061         case 0: /* four banks */
3062                 rdev->config.si.tile_config |= 0 << 4;
3063                 break;
3064         case 1: /* eight banks */
3065                 rdev->config.si.tile_config |= 1 << 4;
3066                 break;
3067         case 2: /* sixteen banks */
3068         default:
3069                 rdev->config.si.tile_config |= 2 << 4;
3070                 break;
3071         }
3072         rdev->config.si.tile_config |=
3073                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3074         rdev->config.si.tile_config |=
3075                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3076
3077         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3078         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3079         WREG32(DMIF_ADDR_CALC, gb_addr_config);
3080         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3081         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3082         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3083         if (rdev->has_uvd) {
3084                 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3085                 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3086                 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3087         }
3088
3089         si_tiling_mode_table_init(rdev);
3090
3091         si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3092                     rdev->config.si.max_sh_per_se,
3093                     rdev->config.si.max_backends_per_se);
3094
3095         si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3096                      rdev->config.si.max_sh_per_se,
3097                      rdev->config.si.max_cu_per_sh);
3098
3099
3100         /* set HW defaults for 3D engine */
3101         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3102                                      ROQ_IB2_START(0x2b)));
3103         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3104
3105         sx_debug_1 = RREG32(SX_DEBUG_1);
3106         WREG32(SX_DEBUG_1, sx_debug_1);
3107
3108         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3109
3110         WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3111                                  SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3112                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3113                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3114
3115         WREG32(VGT_NUM_INSTANCES, 1);
3116
3117         WREG32(CP_PERFMON_CNTL, 0);
3118
3119         WREG32(SQ_CONFIG, 0);
3120
3121         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3122                                           FORCE_EOV_MAX_REZ_CNT(255)));
3123
3124         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3125                AUTO_INVLD_EN(ES_AND_GS_AUTO));
3126
3127         WREG32(VGT_GS_VERTEX_REUSE, 16);
3128         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3129
3130         WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3131         WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3132         WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3133         WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3134         WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3135         WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3136         WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3137         WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3138
3139         tmp = RREG32(HDP_MISC_CNTL);
3140         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3141         WREG32(HDP_MISC_CNTL, tmp);
3142
3143         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3144         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3145
3146         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3147
3148         udelay(50);
3149 }
3150
3151 /*
3152  * GPU scratch registers helpers function.
3153  */
3154 static void si_scratch_init(struct radeon_device *rdev)
3155 {
3156         int i;
3157
3158         rdev->scratch.num_reg = 7;
3159         rdev->scratch.reg_base = SCRATCH_REG0;
3160         for (i = 0; i < rdev->scratch.num_reg; i++) {
3161                 rdev->scratch.free[i] = true;
3162                 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3163         }
3164 }
3165
3166 void si_fence_ring_emit(struct radeon_device *rdev,
3167                         struct radeon_fence *fence)
3168 {
3169         struct radeon_ring *ring = &rdev->ring[fence->ring];
3170         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3171
3172         /* flush read cache over gart */
3173         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3174         radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3175         radeon_ring_write(ring, 0);
3176         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3177         radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3178                           PACKET3_TC_ACTION_ENA |
3179                           PACKET3_SH_KCACHE_ACTION_ENA |
3180                           PACKET3_SH_ICACHE_ACTION_ENA);
3181         radeon_ring_write(ring, 0xFFFFFFFF);
3182         radeon_ring_write(ring, 0);
3183         radeon_ring_write(ring, 10); /* poll interval */
3184         /* EVENT_WRITE_EOP - flush caches, send int */
3185         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3186         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3187         radeon_ring_write(ring, addr & 0xffffffff);
3188         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3189         radeon_ring_write(ring, fence->seq);
3190         radeon_ring_write(ring, 0);
3191 }
3192
3193 /*
3194  * IB stuff
3195  */
3196 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3197 {
3198         struct radeon_ring *ring = &rdev->ring[ib->ring];
3199         u32 header;
3200
3201         if (ib->is_const_ib) {
3202                 /* set switch buffer packet before const IB */
3203                 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3204                 radeon_ring_write(ring, 0);
3205
3206                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3207         } else {
3208                 u32 next_rptr;
3209                 if (ring->rptr_save_reg) {
3210                         next_rptr = ring->wptr + 3 + 4 + 8;
3211                         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3212                         radeon_ring_write(ring, ((ring->rptr_save_reg -
3213                                                   PACKET3_SET_CONFIG_REG_START) >> 2));
3214                         radeon_ring_write(ring, next_rptr);
3215                 } else if (rdev->wb.enabled) {
3216                         next_rptr = ring->wptr + 5 + 4 + 8;
3217                         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3218                         radeon_ring_write(ring, (1 << 8));
3219                         radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3220                         radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3221                         radeon_ring_write(ring, next_rptr);
3222                 }
3223
3224                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3225         }
3226
3227         radeon_ring_write(ring, header);
3228         radeon_ring_write(ring,
3229 #ifdef __BIG_ENDIAN
3230                           (2 << 0) |
3231 #endif
3232                           (ib->gpu_addr & 0xFFFFFFFC));
3233         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3234         radeon_ring_write(ring, ib->length_dw |
3235                           (ib->vm ? (ib->vm->id << 24) : 0));
3236
3237         if (!ib->is_const_ib) {
3238                 /* flush read cache over gart for this vmid */
3239                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3240                 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3241                 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
3242                 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3243                 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3244                                   PACKET3_TC_ACTION_ENA |
3245                                   PACKET3_SH_KCACHE_ACTION_ENA |
3246                                   PACKET3_SH_ICACHE_ACTION_ENA);
3247                 radeon_ring_write(ring, 0xFFFFFFFF);
3248                 radeon_ring_write(ring, 0);
3249                 radeon_ring_write(ring, 10); /* poll interval */
3250         }
3251 }
3252
3253 /*
3254  * CP.
3255  */
3256 static void si_cp_enable(struct radeon_device *rdev, bool enable)
3257 {
3258         if (enable)
3259                 WREG32(CP_ME_CNTL, 0);
3260         else {
3261                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3262                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3263                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3264                 WREG32(SCRATCH_UMSK, 0);
3265                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3266                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3267                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3268         }
3269         udelay(50);
3270 }
3271
3272 static int si_cp_load_microcode(struct radeon_device *rdev)
3273 {
3274         const __be32 *fw_data;
3275         int i;
3276
3277         if (!rdev->me_fw || !rdev->pfp_fw)
3278                 return -EINVAL;
3279
3280         si_cp_enable(rdev, false);
3281
3282         /* PFP */
3283         fw_data = (const __be32 *)rdev->pfp_fw->data;
3284         WREG32(CP_PFP_UCODE_ADDR, 0);
3285         for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3286                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3287         WREG32(CP_PFP_UCODE_ADDR, 0);
3288
3289         /* CE */
3290         fw_data = (const __be32 *)rdev->ce_fw->data;
3291         WREG32(CP_CE_UCODE_ADDR, 0);
3292         for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3293                 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3294         WREG32(CP_CE_UCODE_ADDR, 0);
3295
3296         /* ME */
3297         fw_data = (const __be32 *)rdev->me_fw->data;
3298         WREG32(CP_ME_RAM_WADDR, 0);
3299         for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3300                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3301         WREG32(CP_ME_RAM_WADDR, 0);
3302
3303         WREG32(CP_PFP_UCODE_ADDR, 0);
3304         WREG32(CP_CE_UCODE_ADDR, 0);
3305         WREG32(CP_ME_RAM_WADDR, 0);
3306         WREG32(CP_ME_RAM_RADDR, 0);
3307         return 0;
3308 }
3309
3310 static int si_cp_start(struct radeon_device *rdev)
3311 {
3312         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3313         int r, i;
3314
3315         r = radeon_ring_lock(rdev, ring, 7 + 4);
3316         if (r) {
3317                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3318                 return r;
3319         }
3320         /* init the CP */
3321         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3322         radeon_ring_write(ring, 0x1);
3323         radeon_ring_write(ring, 0x0);
3324         radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3325         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3326         radeon_ring_write(ring, 0);
3327         radeon_ring_write(ring, 0);
3328
3329         /* init the CE partitions */
3330         radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3331         radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3332         radeon_ring_write(ring, 0xc000);
3333         radeon_ring_write(ring, 0xe000);
3334         radeon_ring_unlock_commit(rdev, ring);
3335
3336         si_cp_enable(rdev, true);
3337
3338         r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3339         if (r) {
3340                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3341                 return r;
3342         }
3343
3344         /* setup clear context state */
3345         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3346         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3347
3348         for (i = 0; i < si_default_size; i++)
3349                 radeon_ring_write(ring, si_default_state[i]);
3350
3351         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3352         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3353
3354         /* set clear context state */
3355         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3356         radeon_ring_write(ring, 0);
3357
3358         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3359         radeon_ring_write(ring, 0x00000316);
3360         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3361         radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3362
3363         radeon_ring_unlock_commit(rdev, ring);
3364
3365         for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3366                 ring = &rdev->ring[i];
3367                 r = radeon_ring_lock(rdev, ring, 2);
3368
3369                 /* clear the compute context state */
3370                 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3371                 radeon_ring_write(ring, 0);
3372
3373                 radeon_ring_unlock_commit(rdev, ring);
3374         }
3375
3376         return 0;
3377 }
3378
3379 static void si_cp_fini(struct radeon_device *rdev)
3380 {
3381         struct radeon_ring *ring;
3382         si_cp_enable(rdev, false);
3383
3384         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3385         radeon_ring_fini(rdev, ring);
3386         radeon_scratch_free(rdev, ring->rptr_save_reg);
3387
3388         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3389         radeon_ring_fini(rdev, ring);
3390         radeon_scratch_free(rdev, ring->rptr_save_reg);
3391
3392         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3393         radeon_ring_fini(rdev, ring);
3394         radeon_scratch_free(rdev, ring->rptr_save_reg);
3395 }
3396
3397 static int si_cp_resume(struct radeon_device *rdev)
3398 {
3399         struct radeon_ring *ring;
3400         u32 tmp;
3401         u32 rb_bufsz;
3402         int r;
3403
3404         si_enable_gui_idle_interrupt(rdev, false);
3405
3406         WREG32(CP_SEM_WAIT_TIMER, 0x0);
3407         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3408
3409         /* Set the write pointer delay */
3410         WREG32(CP_RB_WPTR_DELAY, 0);
3411
3412         WREG32(CP_DEBUG, 0);
3413         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3414
3415         /* ring 0 - compute and gfx */
3416         /* Set ring buffer size */
3417         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3418         rb_bufsz = order_base_2(ring->ring_size / 8);
3419         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3420 #ifdef __BIG_ENDIAN
3421         tmp |= BUF_SWAP_32BIT;
3422 #endif
3423         WREG32(CP_RB0_CNTL, tmp);
3424
3425         /* Initialize the ring buffer's read and write pointers */
3426         WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3427         ring->wptr = 0;
3428         WREG32(CP_RB0_WPTR, ring->wptr);
3429
3430         /* set the wb address whether it's enabled or not */
3431         WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3432         WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3433
3434         if (rdev->wb.enabled)
3435                 WREG32(SCRATCH_UMSK, 0xff);
3436         else {
3437                 tmp |= RB_NO_UPDATE;
3438                 WREG32(SCRATCH_UMSK, 0);
3439         }
3440
3441         mdelay(1);
3442         WREG32(CP_RB0_CNTL, tmp);
3443
3444         WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3445
3446         /* ring1  - compute only */
3447         /* Set ring buffer size */
3448         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3449         rb_bufsz = order_base_2(ring->ring_size / 8);
3450         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3451 #ifdef __BIG_ENDIAN
3452         tmp |= BUF_SWAP_32BIT;
3453 #endif
3454         WREG32(CP_RB1_CNTL, tmp);
3455
3456         /* Initialize the ring buffer's read and write pointers */
3457         WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3458         ring->wptr = 0;
3459         WREG32(CP_RB1_WPTR, ring->wptr);
3460
3461         /* set the wb address whether it's enabled or not */
3462         WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3463         WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3464
3465         mdelay(1);
3466         WREG32(CP_RB1_CNTL, tmp);
3467
3468         WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3469
3470         /* ring2 - compute only */
3471         /* Set ring buffer size */
3472         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3473         rb_bufsz = order_base_2(ring->ring_size / 8);
3474         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3475 #ifdef __BIG_ENDIAN
3476         tmp |= BUF_SWAP_32BIT;
3477 #endif
3478         WREG32(CP_RB2_CNTL, tmp);
3479
3480         /* Initialize the ring buffer's read and write pointers */
3481         WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3482         ring->wptr = 0;
3483         WREG32(CP_RB2_WPTR, ring->wptr);
3484
3485         /* set the wb address whether it's enabled or not */
3486         WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3487         WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3488
3489         mdelay(1);
3490         WREG32(CP_RB2_CNTL, tmp);
3491
3492         WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3493
3494         /* start the rings */
3495         si_cp_start(rdev);
3496         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3497         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3498         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3499         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3500         if (r) {
3501                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3502                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3503                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3504                 return r;
3505         }
3506         r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3507         if (r) {
3508                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3509         }
3510         r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3511         if (r) {
3512                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3513         }
3514
3515         si_enable_gui_idle_interrupt(rdev, true);
3516
3517         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3518                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3519
3520         return 0;
3521 }
3522
3523 u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3524 {
3525         u32 reset_mask = 0;
3526         u32 tmp;
3527
3528         /* GRBM_STATUS */
3529         tmp = RREG32(GRBM_STATUS);
3530         if (tmp & (PA_BUSY | SC_BUSY |
3531                    BCI_BUSY | SX_BUSY |
3532                    TA_BUSY | VGT_BUSY |
3533                    DB_BUSY | CB_BUSY |
3534                    GDS_BUSY | SPI_BUSY |
3535                    IA_BUSY | IA_BUSY_NO_DMA))
3536                 reset_mask |= RADEON_RESET_GFX;
3537
3538         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3539                    CP_BUSY | CP_COHERENCY_BUSY))
3540                 reset_mask |= RADEON_RESET_CP;
3541
3542         if (tmp & GRBM_EE_BUSY)
3543                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3544
3545         /* GRBM_STATUS2 */
3546         tmp = RREG32(GRBM_STATUS2);
3547         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3548                 reset_mask |= RADEON_RESET_RLC;
3549
3550         /* DMA_STATUS_REG 0 */
3551         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3552         if (!(tmp & DMA_IDLE))
3553                 reset_mask |= RADEON_RESET_DMA;
3554
3555         /* DMA_STATUS_REG 1 */
3556         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3557         if (!(tmp & DMA_IDLE))
3558                 reset_mask |= RADEON_RESET_DMA1;
3559
3560         /* SRBM_STATUS2 */
3561         tmp = RREG32(SRBM_STATUS2);
3562         if (tmp & DMA_BUSY)
3563                 reset_mask |= RADEON_RESET_DMA;
3564
3565         if (tmp & DMA1_BUSY)
3566                 reset_mask |= RADEON_RESET_DMA1;
3567
3568         /* SRBM_STATUS */
3569         tmp = RREG32(SRBM_STATUS);
3570
3571         if (tmp & IH_BUSY)
3572                 reset_mask |= RADEON_RESET_IH;
3573
3574         if (tmp & SEM_BUSY)
3575                 reset_mask |= RADEON_RESET_SEM;
3576
3577         if (tmp & GRBM_RQ_PENDING)
3578                 reset_mask |= RADEON_RESET_GRBM;
3579
3580         if (tmp & VMC_BUSY)
3581                 reset_mask |= RADEON_RESET_VMC;
3582
3583         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3584                    MCC_BUSY | MCD_BUSY))
3585                 reset_mask |= RADEON_RESET_MC;
3586
3587         if (evergreen_is_display_hung(rdev))
3588                 reset_mask |= RADEON_RESET_DISPLAY;
3589
3590         /* VM_L2_STATUS */
3591         tmp = RREG32(VM_L2_STATUS);
3592         if (tmp & L2_BUSY)
3593                 reset_mask |= RADEON_RESET_VMC;
3594
3595         /* Skip MC reset as it's mostly likely not hung, just busy */
3596         if (reset_mask & RADEON_RESET_MC) {
3597                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3598                 reset_mask &= ~RADEON_RESET_MC;
3599         }
3600
3601         return reset_mask;
3602 }
3603
3604 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3605 {
3606         struct evergreen_mc_save save;
3607         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3608         u32 tmp;
3609
3610         if (reset_mask == 0)
3611                 return;
3612
3613         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3614
3615         evergreen_print_gpu_status_regs(rdev);
3616         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
3617                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3618         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3619                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3620
3621         /* disable PG/CG */
3622         si_fini_pg(rdev);
3623         si_fini_cg(rdev);
3624
3625         /* stop the rlc */
3626         si_rlc_stop(rdev);
3627
3628         /* Disable CP parsing/prefetching */
3629         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3630
3631         if (reset_mask & RADEON_RESET_DMA) {
3632                 /* dma0 */
3633                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3634                 tmp &= ~DMA_RB_ENABLE;
3635                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3636         }
3637         if (reset_mask & RADEON_RESET_DMA1) {
3638                 /* dma1 */
3639                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3640                 tmp &= ~DMA_RB_ENABLE;
3641                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3642         }
3643
3644         udelay(50);
3645
3646         evergreen_mc_stop(rdev, &save);
3647         if (evergreen_mc_wait_for_idle(rdev)) {
3648                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3649         }
3650
3651         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3652                 grbm_soft_reset = SOFT_RESET_CB |
3653                         SOFT_RESET_DB |
3654                         SOFT_RESET_GDS |
3655                         SOFT_RESET_PA |
3656                         SOFT_RESET_SC |
3657                         SOFT_RESET_BCI |
3658                         SOFT_RESET_SPI |
3659                         SOFT_RESET_SX |
3660                         SOFT_RESET_TC |
3661                         SOFT_RESET_TA |
3662                         SOFT_RESET_VGT |
3663                         SOFT_RESET_IA;
3664         }
3665
3666         if (reset_mask & RADEON_RESET_CP) {
3667                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3668
3669                 srbm_soft_reset |= SOFT_RESET_GRBM;
3670         }
3671
3672         if (reset_mask & RADEON_RESET_DMA)
3673                 srbm_soft_reset |= SOFT_RESET_DMA;
3674
3675         if (reset_mask & RADEON_RESET_DMA1)
3676                 srbm_soft_reset |= SOFT_RESET_DMA1;
3677
3678         if (reset_mask & RADEON_RESET_DISPLAY)
3679                 srbm_soft_reset |= SOFT_RESET_DC;
3680
3681         if (reset_mask & RADEON_RESET_RLC)
3682                 grbm_soft_reset |= SOFT_RESET_RLC;
3683
3684         if (reset_mask & RADEON_RESET_SEM)
3685                 srbm_soft_reset |= SOFT_RESET_SEM;
3686
3687         if (reset_mask & RADEON_RESET_IH)
3688                 srbm_soft_reset |= SOFT_RESET_IH;
3689
3690         if (reset_mask & RADEON_RESET_GRBM)
3691                 srbm_soft_reset |= SOFT_RESET_GRBM;
3692
3693         if (reset_mask & RADEON_RESET_VMC)
3694                 srbm_soft_reset |= SOFT_RESET_VMC;
3695
3696         if (reset_mask & RADEON_RESET_MC)
3697                 srbm_soft_reset |= SOFT_RESET_MC;
3698
3699         if (grbm_soft_reset) {
3700                 tmp = RREG32(GRBM_SOFT_RESET);
3701                 tmp |= grbm_soft_reset;
3702                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3703                 WREG32(GRBM_SOFT_RESET, tmp);
3704                 tmp = RREG32(GRBM_SOFT_RESET);
3705
3706                 udelay(50);
3707
3708                 tmp &= ~grbm_soft_reset;
3709                 WREG32(GRBM_SOFT_RESET, tmp);
3710                 tmp = RREG32(GRBM_SOFT_RESET);
3711         }
3712
3713         if (srbm_soft_reset) {
3714                 tmp = RREG32(SRBM_SOFT_RESET);
3715                 tmp |= srbm_soft_reset;
3716                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3717                 WREG32(SRBM_SOFT_RESET, tmp);
3718                 tmp = RREG32(SRBM_SOFT_RESET);
3719
3720                 udelay(50);
3721
3722                 tmp &= ~srbm_soft_reset;
3723                 WREG32(SRBM_SOFT_RESET, tmp);
3724                 tmp = RREG32(SRBM_SOFT_RESET);
3725         }
3726
3727         /* Wait a little for things to settle down */
3728         udelay(50);
3729
3730         evergreen_mc_resume(rdev, &save);
3731         udelay(50);
3732
3733         evergreen_print_gpu_status_regs(rdev);
3734 }
3735
3736 static void si_set_clk_bypass_mode(struct radeon_device *rdev)
3737 {
3738         u32 tmp, i;
3739
3740         tmp = RREG32(CG_SPLL_FUNC_CNTL);
3741         tmp |= SPLL_BYPASS_EN;
3742         WREG32(CG_SPLL_FUNC_CNTL, tmp);
3743
3744         tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3745         tmp |= SPLL_CTLREQ_CHG;
3746         WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3747
3748         for (i = 0; i < rdev->usec_timeout; i++) {
3749                 if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
3750                         break;
3751                 udelay(1);
3752         }
3753
3754         tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3755         tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);
3756         WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3757
3758         tmp = RREG32(MPLL_CNTL_MODE);
3759         tmp &= ~MPLL_MCLK_SEL;
3760         WREG32(MPLL_CNTL_MODE, tmp);
3761 }
3762
3763 static void si_spll_powerdown(struct radeon_device *rdev)
3764 {
3765         u32 tmp;
3766
3767         tmp = RREG32(SPLL_CNTL_MODE);
3768         tmp |= SPLL_SW_DIR_CONTROL;
3769         WREG32(SPLL_CNTL_MODE, tmp);
3770
3771         tmp = RREG32(CG_SPLL_FUNC_CNTL);
3772         tmp |= SPLL_RESET;
3773         WREG32(CG_SPLL_FUNC_CNTL, tmp);
3774
3775         tmp = RREG32(CG_SPLL_FUNC_CNTL);
3776         tmp |= SPLL_SLEEP;
3777         WREG32(CG_SPLL_FUNC_CNTL, tmp);
3778
3779         tmp = RREG32(SPLL_CNTL_MODE);
3780         tmp &= ~SPLL_SW_DIR_CONTROL;
3781         WREG32(SPLL_CNTL_MODE, tmp);
3782 }
3783
3784 static void si_gpu_pci_config_reset(struct radeon_device *rdev)
3785 {
3786         struct evergreen_mc_save save;
3787         u32 tmp, i;
3788
3789         dev_info(rdev->dev, "GPU pci config reset\n");
3790
3791         /* disable dpm? */
3792
3793         /* disable cg/pg */
3794         si_fini_pg(rdev);
3795         si_fini_cg(rdev);
3796
3797         /* Disable CP parsing/prefetching */
3798         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3799         /* dma0 */
3800         tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3801         tmp &= ~DMA_RB_ENABLE;
3802         WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3803         /* dma1 */
3804         tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3805         tmp &= ~DMA_RB_ENABLE;
3806         WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3807         /* XXX other engines? */
3808
3809         /* halt the rlc, disable cp internal ints */
3810         si_rlc_stop(rdev);
3811
3812         udelay(50);
3813
3814         /* disable mem access */
3815         evergreen_mc_stop(rdev, &save);
3816         if (evergreen_mc_wait_for_idle(rdev)) {
3817                 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3818         }
3819
3820         /* set mclk/sclk to bypass */
3821         si_set_clk_bypass_mode(rdev);
3822         /* powerdown spll */
3823         si_spll_powerdown(rdev);
3824         /* disable BM */
3825         pci_clear_master(rdev->pdev);
3826         /* reset */
3827         radeon_pci_config_reset(rdev);
3828         /* wait for asic to come out of reset */
3829         for (i = 0; i < rdev->usec_timeout; i++) {
3830                 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
3831                         break;
3832                 udelay(1);
3833         }
3834 }
3835
3836 int si_asic_reset(struct radeon_device *rdev)
3837 {
3838         u32 reset_mask;
3839
3840         reset_mask = si_gpu_check_soft_reset(rdev);
3841
3842         if (reset_mask)
3843                 r600_set_bios_scratch_engine_hung(rdev, true);
3844
3845         /* try soft reset */
3846         si_gpu_soft_reset(rdev, reset_mask);
3847
3848         reset_mask = si_gpu_check_soft_reset(rdev);
3849
3850         /* try pci config reset */
3851         if (reset_mask && radeon_hard_reset)
3852                 si_gpu_pci_config_reset(rdev);
3853
3854         reset_mask = si_gpu_check_soft_reset(rdev);
3855
3856         if (!reset_mask)
3857                 r600_set_bios_scratch_engine_hung(rdev, false);
3858
3859         return 0;
3860 }
3861
3862 /**
3863  * si_gfx_is_lockup - Check if the GFX engine is locked up
3864  *
3865  * @rdev: radeon_device pointer
3866  * @ring: radeon_ring structure holding ring information
3867  *
3868  * Check if the GFX engine is locked up.
3869  * Returns true if the engine appears to be locked up, false if not.
3870  */
3871 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3872 {
3873         u32 reset_mask = si_gpu_check_soft_reset(rdev);
3874
3875         if (!(reset_mask & (RADEON_RESET_GFX |
3876                             RADEON_RESET_COMPUTE |
3877                             RADEON_RESET_CP))) {
3878                 radeon_ring_lockup_update(rdev, ring);
3879                 return false;
3880         }
3881         return radeon_ring_test_lockup(rdev, ring);
3882 }
3883
3884 /* MC */
3885 static void si_mc_program(struct radeon_device *rdev)
3886 {
3887         struct evergreen_mc_save save;
3888         u32 tmp;
3889         int i, j;
3890
3891         /* Initialize HDP */
3892         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3893                 WREG32((0x2c14 + j), 0x00000000);
3894                 WREG32((0x2c18 + j), 0x00000000);
3895                 WREG32((0x2c1c + j), 0x00000000);
3896                 WREG32((0x2c20 + j), 0x00000000);
3897                 WREG32((0x2c24 + j), 0x00000000);
3898         }
3899         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3900
3901         evergreen_mc_stop(rdev, &save);
3902         if (radeon_mc_wait_for_idle(rdev)) {
3903                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3904         }
3905         if (!ASIC_IS_NODCE(rdev))
3906                 /* Lockout access through VGA aperture*/
3907                 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
3908         /* Update configuration */
3909         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3910                rdev->mc.vram_start >> 12);
3911         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3912                rdev->mc.vram_end >> 12);
3913         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3914                rdev->vram_scratch.gpu_addr >> 12);
3915         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3916         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3917         WREG32(MC_VM_FB_LOCATION, tmp);
3918         /* XXX double check these! */
3919         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3920         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3921         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3922         WREG32(MC_VM_AGP_BASE, 0);
3923         WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3924         WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3925         if (radeon_mc_wait_for_idle(rdev)) {
3926                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3927         }
3928         evergreen_mc_resume(rdev, &save);
3929         if (!ASIC_IS_NODCE(rdev)) {
3930                 /* we need to own VRAM, so turn off the VGA renderer here
3931                  * to stop it overwriting our objects */
3932                 rv515_vga_render_disable(rdev);
3933         }
3934 }
3935
3936 void si_vram_gtt_location(struct radeon_device *rdev,
3937                           struct radeon_mc *mc)
3938 {
3939         if (mc->mc_vram_size > 0xFFC0000000ULL) {
3940                 /* leave room for at least 1024M GTT */
3941                 dev_warn(rdev->dev, "limiting VRAM\n");
3942                 mc->real_vram_size = 0xFFC0000000ULL;
3943                 mc->mc_vram_size = 0xFFC0000000ULL;
3944         }
3945         radeon_vram_location(rdev, &rdev->mc, 0);
3946         rdev->mc.gtt_base_align = 0;
3947         radeon_gtt_location(rdev, mc);
3948 }
3949
3950 static int si_mc_init(struct radeon_device *rdev)
3951 {
3952         u32 tmp;
3953         int chansize, numchan;
3954
3955         /* Get VRAM informations */
3956         rdev->mc.vram_is_ddr = true;
3957         tmp = RREG32(MC_ARB_RAMCFG);
3958         if (tmp & CHANSIZE_OVERRIDE) {
3959                 chansize = 16;
3960         } else if (tmp & CHANSIZE_MASK) {
3961                 chansize = 64;
3962         } else {
3963                 chansize = 32;
3964         }
3965         tmp = RREG32(MC_SHARED_CHMAP);
3966         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3967         case 0:
3968         default:
3969                 numchan = 1;
3970                 break;
3971         case 1:
3972                 numchan = 2;
3973                 break;
3974         case 2:
3975                 numchan = 4;
3976                 break;
3977         case 3:
3978                 numchan = 8;
3979                 break;
3980         case 4:
3981                 numchan = 3;
3982                 break;
3983         case 5:
3984                 numchan = 6;
3985                 break;
3986         case 6:
3987                 numchan = 10;
3988                 break;
3989         case 7:
3990                 numchan = 12;
3991                 break;
3992         case 8:
3993                 numchan = 16;
3994                 break;
3995         }
3996         rdev->mc.vram_width = numchan * chansize;
3997         /* Could aper size report 0 ? */
3998         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3999         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4000         /* size in MB on si */
4001         tmp = RREG32(CONFIG_MEMSIZE);
4002         /* some boards may have garbage in the upper 16 bits */
4003         if (tmp & 0xffff0000) {
4004                 DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
4005                 if (tmp & 0xffff)
4006                         tmp &= 0xffff;
4007         }
4008         rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
4009         rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
4010         rdev->mc.visible_vram_size = rdev->mc.aper_size;
4011         si_vram_gtt_location(rdev, &rdev->mc);
4012         radeon_update_bandwidth_info(rdev);
4013
4014         return 0;
4015 }
4016
4017 /*
4018  * GART
4019  */
4020 void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
4021 {
4022         /* flush hdp cache */
4023         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4024
4025         /* bits 0-15 are the VM contexts0-15 */
4026         WREG32(VM_INVALIDATE_REQUEST, 1);
4027 }
4028
4029 static int si_pcie_gart_enable(struct radeon_device *rdev)
4030 {
4031         int r, i;
4032
4033         if (rdev->gart.robj == NULL) {
4034                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4035                 return -EINVAL;
4036         }
4037         r = radeon_gart_table_vram_pin(rdev);
4038         if (r)
4039                 return r;
4040         radeon_gart_restore(rdev);
4041         /* Setup TLB control */
4042         WREG32(MC_VM_MX_L1_TLB_CNTL,
4043                (0xA << 7) |
4044                ENABLE_L1_TLB |
4045                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4046                ENABLE_ADVANCED_DRIVER_MODEL |
4047                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4048         /* Setup L2 cache */
4049         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4050                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4051                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4052                EFFECTIVE_L2_QUEUE_SIZE(7) |
4053                CONTEXT1_IDENTITY_ACCESS_MODE(1));
4054         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4055         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4056                L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4057         /* setup context0 */
4058         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4059         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4060         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4061         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4062                         (u32)(rdev->dummy_page.addr >> 12));
4063         WREG32(VM_CONTEXT0_CNTL2, 0);
4064         WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4065                                   RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4066
4067         WREG32(0x15D4, 0);
4068         WREG32(0x15D8, 0);
4069         WREG32(0x15DC, 0);
4070
4071         /* empty context1-15 */
4072         /* set vm size, must be a multiple of 4 */
4073         WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
4074         WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
4075         /* Assign the pt base to something valid for now; the pts used for
4076          * the VMs are determined by the application and setup and assigned
4077          * on the fly in the vm part of radeon_gart.c
4078          */
4079         for (i = 1; i < 16; i++) {
4080                 if (i < 8)
4081                         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4082                                rdev->gart.table_addr >> 12);
4083                 else
4084                         WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4085                                rdev->gart.table_addr >> 12);
4086         }
4087
4088         /* enable context1-15 */
4089         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4090                (u32)(rdev->dummy_page.addr >> 12));
4091         WREG32(VM_CONTEXT1_CNTL2, 4);
4092         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4093                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4094                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4095                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4096                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4097                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4098                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4099                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4100                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4101                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4102                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4103                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4104                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
4105
4106         si_pcie_gart_tlb_flush(rdev);
4107         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4108                  (unsigned)(rdev->mc.gtt_size >> 20),
4109                  (unsigned long long)rdev->gart.table_addr);
4110         rdev->gart.ready = true;
4111         return 0;
4112 }
4113
4114 static void si_pcie_gart_disable(struct radeon_device *rdev)
4115 {
4116         /* Disable all tables */
4117         WREG32(VM_CONTEXT0_CNTL, 0);
4118         WREG32(VM_CONTEXT1_CNTL, 0);
4119         /* Setup TLB control */
4120         WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4121                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4122         /* Setup L2 cache */
4123         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4124                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4125                EFFECTIVE_L2_QUEUE_SIZE(7) |
4126                CONTEXT1_IDENTITY_ACCESS_MODE(1));
4127         WREG32(VM_L2_CNTL2, 0);
4128         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4129                L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4130         radeon_gart_table_vram_unpin(rdev);
4131 }
4132
4133 static void si_pcie_gart_fini(struct radeon_device *rdev)
4134 {
4135         si_pcie_gart_disable(rdev);
4136         radeon_gart_table_vram_free(rdev);
4137         radeon_gart_fini(rdev);
4138 }
4139
4140 /* vm parser */
4141 static bool si_vm_reg_valid(u32 reg)
4142 {
4143         /* context regs are fine */
4144         if (reg >= 0x28000)
4145                 return true;
4146
4147         /* check config regs */
4148         switch (reg) {
4149         case GRBM_GFX_INDEX:
4150         case CP_STRMOUT_CNTL:
4151         case VGT_VTX_VECT_EJECT_REG:
4152         case VGT_CACHE_INVALIDATION:
4153         case VGT_ESGS_RING_SIZE:
4154         case VGT_GSVS_RING_SIZE:
4155         case VGT_GS_VERTEX_REUSE:
4156         case VGT_PRIMITIVE_TYPE:
4157         case VGT_INDEX_TYPE:
4158         case VGT_NUM_INDICES:
4159         case VGT_NUM_INSTANCES:
4160         case VGT_TF_RING_SIZE:
4161         case VGT_HS_OFFCHIP_PARAM:
4162         case VGT_TF_MEMORY_BASE:
4163         case PA_CL_ENHANCE:
4164         case PA_SU_LINE_STIPPLE_VALUE:
4165         case PA_SC_LINE_STIPPLE_STATE:
4166         case PA_SC_ENHANCE:
4167         case SQC_CACHES:
4168         case SPI_STATIC_THREAD_MGMT_1:
4169         case SPI_STATIC_THREAD_MGMT_2:
4170         case SPI_STATIC_THREAD_MGMT_3:
4171         case SPI_PS_MAX_WAVE_ID:
4172         case SPI_CONFIG_CNTL:
4173         case SPI_CONFIG_CNTL_1:
4174         case TA_CNTL_AUX:
4175                 return true;
4176         default:
4177                 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4178                 return false;
4179         }
4180 }
4181
4182 static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4183                                   u32 *ib, struct radeon_cs_packet *pkt)
4184 {
4185         switch (pkt->opcode) {
4186         case PACKET3_NOP:
4187         case PACKET3_SET_BASE:
4188         case PACKET3_SET_CE_DE_COUNTERS:
4189         case PACKET3_LOAD_CONST_RAM:
4190         case PACKET3_WRITE_CONST_RAM:
4191         case PACKET3_WRITE_CONST_RAM_OFFSET:
4192         case PACKET3_DUMP_CONST_RAM:
4193         case PACKET3_INCREMENT_CE_COUNTER:
4194         case PACKET3_WAIT_ON_DE_COUNTER:
4195         case PACKET3_CE_WRITE:
4196                 break;
4197         default:
4198                 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4199                 return -EINVAL;
4200         }
4201         return 0;
4202 }
4203
4204 static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4205 {
4206         u32 start_reg, reg, i;
4207         u32 command = ib[idx + 4];
4208         u32 info = ib[idx + 1];
4209         u32 idx_value = ib[idx];
4210         if (command & PACKET3_CP_DMA_CMD_SAS) {
4211                 /* src address space is register */
4212                 if (((info & 0x60000000) >> 29) == 0) {
4213                         start_reg = idx_value << 2;
4214                         if (command & PACKET3_CP_DMA_CMD_SAIC) {
4215                                 reg = start_reg;
4216                                 if (!si_vm_reg_valid(reg)) {
4217                                         DRM_ERROR("CP DMA Bad SRC register\n");
4218                                         return -EINVAL;
4219                                 }
4220                         } else {
4221                                 for (i = 0; i < (command & 0x1fffff); i++) {
4222                                         reg = start_reg + (4 * i);
4223                                         if (!si_vm_reg_valid(reg)) {
4224                                                 DRM_ERROR("CP DMA Bad SRC register\n");
4225                                                 return -EINVAL;
4226                                         }
4227                                 }
4228                         }
4229                 }
4230         }
4231         if (command & PACKET3_CP_DMA_CMD_DAS) {
4232                 /* dst address space is register */
4233                 if (((info & 0x00300000) >> 20) == 0) {
4234                         start_reg = ib[idx + 2];
4235                         if (command & PACKET3_CP_DMA_CMD_DAIC) {
4236                                 reg = start_reg;
4237                                 if (!si_vm_reg_valid(reg)) {
4238                                         DRM_ERROR("CP DMA Bad DST register\n");
4239                                         return -EINVAL;
4240                                 }
4241                         } else {
4242                                 for (i = 0; i < (command & 0x1fffff); i++) {
4243                                         reg = start_reg + (4 * i);
4244                                 if (!si_vm_reg_valid(reg)) {
4245                                                 DRM_ERROR("CP DMA Bad DST register\n");
4246                                                 return -EINVAL;
4247                                         }
4248                                 }
4249                         }
4250                 }
4251         }
4252         return 0;
4253 }
4254
4255 static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4256                                    u32 *ib, struct radeon_cs_packet *pkt)
4257 {
4258         int r;
4259         u32 idx = pkt->idx + 1;
4260         u32 idx_value = ib[idx];
4261         u32 start_reg, end_reg, reg, i;
4262
4263         switch (pkt->opcode) {
4264         case PACKET3_NOP:
4265         case PACKET3_SET_BASE:
4266         case PACKET3_CLEAR_STATE:
4267         case PACKET3_INDEX_BUFFER_SIZE:
4268         case PACKET3_DISPATCH_DIRECT:
4269         case PACKET3_DISPATCH_INDIRECT:
4270         case PACKET3_ALLOC_GDS:
4271         case PACKET3_WRITE_GDS_RAM:
4272         case PACKET3_ATOMIC_GDS:
4273         case PACKET3_ATOMIC:
4274         case PACKET3_OCCLUSION_QUERY:
4275         case PACKET3_SET_PREDICATION:
4276         case PACKET3_COND_EXEC:
4277         case PACKET3_PRED_EXEC:
4278         case PACKET3_DRAW_INDIRECT:
4279         case PACKET3_DRAW_INDEX_INDIRECT:
4280         case PACKET3_INDEX_BASE:
4281         case PACKET3_DRAW_INDEX_2:
4282         case PACKET3_CONTEXT_CONTROL:
4283         case PACKET3_INDEX_TYPE:
4284         case PACKET3_DRAW_INDIRECT_MULTI:
4285         case PACKET3_DRAW_INDEX_AUTO:
4286         case PACKET3_DRAW_INDEX_IMMD:
4287         case PACKET3_NUM_INSTANCES:
4288         case PACKET3_DRAW_INDEX_MULTI_AUTO:
4289         case PACKET3_STRMOUT_BUFFER_UPDATE:
4290         case PACKET3_DRAW_INDEX_OFFSET_2:
4291         case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4292         case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4293         case PACKET3_MPEG_INDEX:
4294         case PACKET3_WAIT_REG_MEM:
4295         case PACKET3_MEM_WRITE:
4296         case PACKET3_PFP_SYNC_ME:
4297         case PACKET3_SURFACE_SYNC:
4298         case PACKET3_EVENT_WRITE:
4299         case PACKET3_EVENT_WRITE_EOP:
4300         case PACKET3_EVENT_WRITE_EOS:
4301         case PACKET3_SET_CONTEXT_REG:
4302         case PACKET3_SET_CONTEXT_REG_INDIRECT:
4303         case PACKET3_SET_SH_REG:
4304         case PACKET3_SET_SH_REG_OFFSET:
4305         case PACKET3_INCREMENT_DE_COUNTER:
4306         case PACKET3_WAIT_ON_CE_COUNTER:
4307         case PACKET3_WAIT_ON_AVAIL_BUFFER:
4308         case PACKET3_ME_WRITE:
4309                 break;
4310         case PACKET3_COPY_DATA:
4311                 if ((idx_value & 0xf00) == 0) {
4312                         reg = ib[idx + 3] * 4;
4313                         if (!si_vm_reg_valid(reg))
4314                                 return -EINVAL;
4315                 }
4316                 break;
4317         case PACKET3_WRITE_DATA:
4318                 if ((idx_value & 0xf00) == 0) {
4319                         start_reg = ib[idx + 1] * 4;
4320                         if (idx_value & 0x10000) {
4321                                 if (!si_vm_reg_valid(start_reg))
4322                                         return -EINVAL;
4323                         } else {
4324                                 for (i = 0; i < (pkt->count - 2); i++) {
4325                                         reg = start_reg + (4 * i);
4326                                         if (!si_vm_reg_valid(reg))
4327                                                 return -EINVAL;
4328                                 }
4329                         }
4330                 }
4331                 break;
4332         case PACKET3_COND_WRITE:
4333                 if (idx_value & 0x100) {
4334                         reg = ib[idx + 5] * 4;
4335                         if (!si_vm_reg_valid(reg))
4336                                 return -EINVAL;
4337                 }
4338                 break;
4339         case PACKET3_COPY_DW:
4340                 if (idx_value & 0x2) {
4341                         reg = ib[idx + 3] * 4;
4342                         if (!si_vm_reg_valid(reg))
4343                                 return -EINVAL;
4344                 }
4345                 break;
4346         case PACKET3_SET_CONFIG_REG:
4347                 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4348                 end_reg = 4 * pkt->count + start_reg - 4;
4349                 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4350                     (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4351                     (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4352                         DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4353                         return -EINVAL;
4354                 }
4355                 for (i = 0; i < pkt->count; i++) {
4356                         reg = start_reg + (4 * i);
4357                         if (!si_vm_reg_valid(reg))
4358                                 return -EINVAL;
4359                 }
4360                 break;
4361         case PACKET3_CP_DMA:
4362                 r = si_vm_packet3_cp_dma_check(ib, idx);
4363                 if (r)
4364                         return r;
4365                 break;
4366         default:
4367                 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4368                 return -EINVAL;
4369         }
4370         return 0;
4371 }
4372
4373 static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4374                                        u32 *ib, struct radeon_cs_packet *pkt)
4375 {
4376         int r;
4377         u32 idx = pkt->idx + 1;
4378         u32 idx_value = ib[idx];
4379         u32 start_reg, reg, i;
4380
4381         switch (pkt->opcode) {
4382         case PACKET3_NOP:
4383         case PACKET3_SET_BASE:
4384         case PACKET3_CLEAR_STATE:
4385         case PACKET3_DISPATCH_DIRECT:
4386         case PACKET3_DISPATCH_INDIRECT:
4387         case PACKET3_ALLOC_GDS:
4388         case PACKET3_WRITE_GDS_RAM:
4389         case PACKET3_ATOMIC_GDS:
4390         case PACKET3_ATOMIC:
4391         case PACKET3_OCCLUSION_QUERY:
4392         case PACKET3_SET_PREDICATION:
4393         case PACKET3_COND_EXEC:
4394         case PACKET3_PRED_EXEC:
4395         case PACKET3_CONTEXT_CONTROL:
4396         case PACKET3_STRMOUT_BUFFER_UPDATE:
4397         case PACKET3_WAIT_REG_MEM:
4398         case PACKET3_MEM_WRITE:
4399         case PACKET3_PFP_SYNC_ME:
4400         case PACKET3_SURFACE_SYNC:
4401         case PACKET3_EVENT_WRITE:
4402         case PACKET3_EVENT_WRITE_EOP:
4403         case PACKET3_EVENT_WRITE_EOS:
4404         case PACKET3_SET_CONTEXT_REG:
4405         case PACKET3_SET_CONTEXT_REG_INDIRECT:
4406         case PACKET3_SET_SH_REG:
4407         case PACKET3_SET_SH_REG_OFFSET:
4408         case PACKET3_INCREMENT_DE_COUNTER:
4409         case PACKET3_WAIT_ON_CE_COUNTER:
4410         case PACKET3_WAIT_ON_AVAIL_BUFFER:
4411         case PACKET3_ME_WRITE:
4412                 break;
4413         case PACKET3_COPY_DATA:
4414                 if ((idx_value & 0xf00) == 0) {
4415                         reg = ib[idx + 3] * 4;
4416                         if (!si_vm_reg_valid(reg))
4417                                 return -EINVAL;
4418                 }
4419                 break;
4420         case PACKET3_WRITE_DATA:
4421                 if ((idx_value & 0xf00) == 0) {
4422                         start_reg = ib[idx + 1] * 4;
4423                         if (idx_value & 0x10000) {
4424                                 if (!si_vm_reg_valid(start_reg))
4425                                         return -EINVAL;
4426                         } else {
4427                                 for (i = 0; i < (pkt->count - 2); i++) {
4428                                         reg = start_reg + (4 * i);
4429                                         if (!si_vm_reg_valid(reg))
4430                                                 return -EINVAL;
4431                                 }
4432                         }
4433                 }
4434                 break;
4435         case PACKET3_COND_WRITE:
4436                 if (idx_value & 0x100) {
4437                         reg = ib[idx + 5] * 4;
4438                         if (!si_vm_reg_valid(reg))
4439                                 return -EINVAL;
4440                 }
4441                 break;
4442         case PACKET3_COPY_DW:
4443                 if (idx_value & 0x2) {
4444                         reg = ib[idx + 3] * 4;
4445                         if (!si_vm_reg_valid(reg))
4446                                 return -EINVAL;
4447                 }
4448                 break;
4449         case PACKET3_CP_DMA:
4450                 r = si_vm_packet3_cp_dma_check(ib, idx);
4451                 if (r)
4452                         return r;
4453                 break;
4454         default:
4455                 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4456                 return -EINVAL;
4457         }
4458         return 0;
4459 }
4460
4461 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4462 {
4463         int ret = 0;
4464         u32 idx = 0;
4465         struct radeon_cs_packet pkt;
4466
4467         do {
4468                 pkt.idx = idx;
4469                 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4470                 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
4471                 pkt.one_reg_wr = 0;
4472                 switch (pkt.type) {
4473                 case RADEON_PACKET_TYPE0:
4474                         dev_err(rdev->dev, "Packet0 not allowed!\n");
4475                         ret = -EINVAL;
4476                         break;
4477                 case RADEON_PACKET_TYPE2:
4478                         idx += 1;
4479                         break;
4480                 case RADEON_PACKET_TYPE3:
4481                         pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
4482                         if (ib->is_const_ib)
4483                                 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4484                         else {
4485                                 switch (ib->ring) {
4486                                 case RADEON_RING_TYPE_GFX_INDEX:
4487                                         ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4488                                         break;
4489                                 case CAYMAN_RING_TYPE_CP1_INDEX:
4490                                 case CAYMAN_RING_TYPE_CP2_INDEX:
4491                                         ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4492                                         break;
4493                                 default:
4494                                         dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
4495                                         ret = -EINVAL;
4496                                         break;
4497                                 }
4498                         }
4499                         idx += pkt.count + 2;
4500                         break;
4501                 default:
4502                         dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4503                         ret = -EINVAL;
4504                         break;
4505                 }
4506                 if (ret)
4507                         break;
4508         } while (idx < ib->length_dw);
4509
4510         return ret;
4511 }
4512
4513 /*
4514  * vm
4515  */
4516 int si_vm_init(struct radeon_device *rdev)
4517 {
4518         /* number of VMs */
4519         rdev->vm_manager.nvm = 16;
4520         /* base offset of vram pages */
4521         rdev->vm_manager.vram_base_offset = 0;
4522
4523         return 0;
4524 }
4525
4526 void si_vm_fini(struct radeon_device *rdev)
4527 {
4528 }
4529
4530 /**
4531  * si_vm_decode_fault - print human readable fault info
4532  *
4533  * @rdev: radeon_device pointer
4534  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4535  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4536  *
4537  * Print human readable fault information (SI).
4538  */
4539 static void si_vm_decode_fault(struct radeon_device *rdev,
4540                                u32 status, u32 addr)
4541 {
4542         u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4543         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4544         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4545         char *block;
4546
4547         if (rdev->family == CHIP_TAHITI) {
4548                 switch (mc_id) {
4549                 case 160:
4550                 case 144:
4551                 case 96:
4552                 case 80:
4553                 case 224:
4554                 case 208:
4555                 case 32:
4556                 case 16:
4557                         block = "CB";
4558                         break;
4559                 case 161:
4560                 case 145:
4561                 case 97:
4562                 case 81:
4563                 case 225:
4564                 case 209:
4565                 case 33:
4566                 case 17:
4567                         block = "CB_FMASK";
4568                         break;
4569                 case 162:
4570                 case 146:
4571                 case 98:
4572                 case 82:
4573                 case 226:
4574                 case 210:
4575                 case 34:
4576                 case 18:
4577                         block = "CB_CMASK";
4578                         break;
4579                 case 163:
4580                 case 147:
4581                 case 99:
4582                 case 83:
4583                 case 227:
4584                 case 211:
4585                 case 35:
4586                 case 19:
4587                         block = "CB_IMMED";
4588                         break;
4589                 case 164:
4590                 case 148:
4591                 case 100:
4592                 case 84:
4593                 case 228:
4594                 case 212:
4595                 case 36:
4596                 case 20:
4597                         block = "DB";
4598                         break;
4599                 case 165:
4600                 case 149:
4601                 case 101:
4602                 case 85:
4603                 case 229:
4604                 case 213:
4605                 case 37:
4606                 case 21:
4607                         block = "DB_HTILE";
4608                         break;
4609                 case 167:
4610                 case 151:
4611                 case 103:
4612                 case 87:
4613                 case 231:
4614                 case 215:
4615                 case 39:
4616                 case 23:
4617                         block = "DB_STEN";
4618                         break;
4619                 case 72:
4620                 case 68:
4621                 case 64:
4622                 case 8:
4623                 case 4:
4624                 case 0:
4625                 case 136:
4626                 case 132:
4627                 case 128:
4628                 case 200:
4629                 case 196:
4630                 case 192:
4631                         block = "TC";
4632                         break;
4633                 case 112:
4634                 case 48:
4635                         block = "CP";
4636                         break;
4637                 case 49:
4638                 case 177:
4639                 case 50:
4640                 case 178:
4641                         block = "SH";
4642                         break;
4643                 case 53:
4644                 case 190:
4645                         block = "VGT";
4646                         break;
4647                 case 117:
4648                         block = "IH";
4649                         break;
4650                 case 51:
4651                 case 115:
4652                         block = "RLC";
4653                         break;
4654                 case 119:
4655                 case 183:
4656                         block = "DMA0";
4657                         break;
4658                 case 61:
4659                         block = "DMA1";
4660                         break;
4661                 case 248:
4662                 case 120:
4663                         block = "HDP";
4664                         break;
4665                 default:
4666                         block = "unknown";
4667                         break;
4668                 }
4669         } else {
4670                 switch (mc_id) {
4671                 case 32:
4672                 case 16:
4673                 case 96:
4674                 case 80:
4675                 case 160:
4676                 case 144:
4677                 case 224:
4678                 case 208:
4679                         block = "CB";
4680                         break;
4681                 case 33:
4682                 case 17:
4683                 case 97:
4684                 case 81:
4685                 case 161:
4686                 case 145:
4687                 case 225:
4688                 case 209:
4689                         block = "CB_FMASK";
4690                         break;
4691                 case 34:
4692                 case 18:
4693                 case 98:
4694                 case 82:
4695                 case 162:
4696                 case 146:
4697                 case 226:
4698                 case 210:
4699                         block = "CB_CMASK";
4700                         break;
4701                 case 35:
4702                 case 19:
4703                 case 99:
4704                 case 83:
4705                 case 163:
4706                 case 147:
4707                 case 227:
4708                 case 211:
4709                         block = "CB_IMMED";
4710                         break;
4711                 case 36:
4712                 case 20:
4713                 case 100:
4714                 case 84:
4715                 case 164:
4716                 case 148:
4717                 case 228:
4718                 case 212:
4719                         block = "DB";
4720                         break;
4721                 case 37:
4722                 case 21:
4723                 case 101:
4724                 case 85:
4725                 case 165:
4726                 case 149:
4727                 case 229:
4728                 case 213:
4729                         block = "DB_HTILE";
4730                         break;
4731                 case 39:
4732                 case 23:
4733                 case 103:
4734                 case 87:
4735                 case 167:
4736                 case 151:
4737                 case 231:
4738                 case 215:
4739                         block = "DB_STEN";
4740                         break;
4741                 case 72:
4742                 case 68:
4743                 case 8:
4744                 case 4:
4745                 case 136:
4746                 case 132:
4747                 case 200:
4748                 case 196:
4749                         block = "TC";
4750                         break;
4751                 case 112:
4752                 case 48:
4753                         block = "CP";
4754                         break;
4755                 case 49:
4756                 case 177:
4757                 case 50:
4758                 case 178:
4759                         block = "SH";
4760                         break;
4761                 case 53:
4762                         block = "VGT";
4763                         break;
4764                 case 117:
4765                         block = "IH";
4766                         break;
4767                 case 51:
4768                 case 115:
4769                         block = "RLC";
4770                         break;
4771                 case 119:
4772                 case 183:
4773                         block = "DMA0";
4774                         break;
4775                 case 61:
4776                         block = "DMA1";
4777                         break;
4778                 case 248:
4779                 case 120:
4780                         block = "HDP";
4781                         break;
4782                 default:
4783                         block = "unknown";
4784                         break;
4785                 }
4786         }
4787
4788         printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4789                protections, vmid, addr,
4790                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
4791                block, mc_id);
4792 }
4793
4794 void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4795 {
4796         struct radeon_ring *ring = &rdev->ring[ridx];
4797
4798         if (vm == NULL)
4799                 return;
4800
4801         /* write new base address */
4802         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4803         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4804                                  WRITE_DATA_DST_SEL(0)));
4805
4806         if (vm->id < 8) {
4807                 radeon_ring_write(ring,
4808                                   (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
4809         } else {
4810                 radeon_ring_write(ring,
4811                                   (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
4812         }
4813         radeon_ring_write(ring, 0);
4814         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4815
4816         /* flush hdp cache */
4817         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4818         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4819                                  WRITE_DATA_DST_SEL(0)));
4820         radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4821         radeon_ring_write(ring, 0);
4822         radeon_ring_write(ring, 0x1);
4823
4824         /* bits 0-15 are the VM contexts0-15 */
4825         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4826         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4827                                  WRITE_DATA_DST_SEL(0)));
4828         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4829         radeon_ring_write(ring, 0);
4830         radeon_ring_write(ring, 1 << vm->id);
4831
4832         /* sync PFP to ME, otherwise we might get invalid PFP reads */
4833         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4834         radeon_ring_write(ring, 0x0);
4835 }
4836
4837 /*
4838  *  Power and clock gating
4839  */
4840 static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
4841 {
4842         int i;
4843
4844         for (i = 0; i < rdev->usec_timeout; i++) {
4845                 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
4846                         break;
4847                 udelay(1);
4848         }
4849
4850         for (i = 0; i < rdev->usec_timeout; i++) {
4851                 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
4852                         break;
4853                 udelay(1);
4854         }
4855 }
4856
4857 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
4858                                          bool enable)
4859 {
4860         u32 tmp = RREG32(CP_INT_CNTL_RING0);
4861         u32 mask;
4862         int i;
4863
4864         if (enable)
4865                 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4866         else
4867                 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4868         WREG32(CP_INT_CNTL_RING0, tmp);
4869
4870         if (!enable) {
4871                 /* read a gfx register */
4872                 tmp = RREG32(DB_DEPTH_INFO);
4873
4874                 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
4875                 for (i = 0; i < rdev->usec_timeout; i++) {
4876                         if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
4877                                 break;
4878                         udelay(1);
4879                 }
4880         }
4881 }
4882
4883 static void si_set_uvd_dcm(struct radeon_device *rdev,
4884                            bool sw_mode)
4885 {
4886         u32 tmp, tmp2;
4887
4888         tmp = RREG32(UVD_CGC_CTRL);
4889         tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
4890         tmp |= DCM | CG_DT(1) | CLK_OD(4);
4891
4892         if (sw_mode) {
4893                 tmp &= ~0x7ffff800;
4894                 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
4895         } else {
4896                 tmp |= 0x7ffff800;
4897                 tmp2 = 0;
4898         }
4899
4900         WREG32(UVD_CGC_CTRL, tmp);
4901         WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
4902 }
4903
4904 void si_init_uvd_internal_cg(struct radeon_device *rdev)
4905 {
4906         bool hw_mode = true;
4907
4908         if (hw_mode) {
4909                 si_set_uvd_dcm(rdev, false);
4910         } else {
4911                 u32 tmp = RREG32(UVD_CGC_CTRL);
4912                 tmp &= ~DCM;
4913                 WREG32(UVD_CGC_CTRL, tmp);
4914         }
4915 }
4916
4917 static u32 si_halt_rlc(struct radeon_device *rdev)
4918 {
4919         u32 data, orig;
4920
4921         orig = data = RREG32(RLC_CNTL);
4922
4923         if (data & RLC_ENABLE) {
4924                 data &= ~RLC_ENABLE;
4925                 WREG32(RLC_CNTL, data);
4926
4927                 si_wait_for_rlc_serdes(rdev);
4928         }
4929
4930         return orig;
4931 }
4932
4933 static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
4934 {
4935         u32 tmp;
4936
4937         tmp = RREG32(RLC_CNTL);
4938         if (tmp != rlc)
4939                 WREG32(RLC_CNTL, rlc);
4940 }
4941
4942 static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
4943 {
4944         u32 data, orig;
4945
4946         orig = data = RREG32(DMA_PG);
4947         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
4948                 data |= PG_CNTL_ENABLE;
4949         else
4950                 data &= ~PG_CNTL_ENABLE;
4951         if (orig != data)
4952                 WREG32(DMA_PG, data);
4953 }
4954
4955 static void si_init_dma_pg(struct radeon_device *rdev)
4956 {
4957         u32 tmp;
4958
4959         WREG32(DMA_PGFSM_WRITE,  0x00002000);
4960         WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
4961
4962         for (tmp = 0; tmp < 5; tmp++)
4963                 WREG32(DMA_PGFSM_WRITE, 0);
4964 }
4965
4966 static void si_enable_gfx_cgpg(struct radeon_device *rdev,
4967                                bool enable)
4968 {
4969         u32 tmp;
4970
4971         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
4972                 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
4973                 WREG32(RLC_TTOP_D, tmp);
4974
4975                 tmp = RREG32(RLC_PG_CNTL);
4976                 tmp |= GFX_PG_ENABLE;
4977                 WREG32(RLC_PG_CNTL, tmp);
4978
4979                 tmp = RREG32(RLC_AUTO_PG_CTRL);
4980                 tmp |= AUTO_PG_EN;
4981                 WREG32(RLC_AUTO_PG_CTRL, tmp);
4982         } else {
4983                 tmp = RREG32(RLC_AUTO_PG_CTRL);
4984                 tmp &= ~AUTO_PG_EN;
4985                 WREG32(RLC_AUTO_PG_CTRL, tmp);
4986
4987                 tmp = RREG32(DB_RENDER_CONTROL);
4988         }
4989 }
4990
4991 static void si_init_gfx_cgpg(struct radeon_device *rdev)
4992 {
4993         u32 tmp;
4994
4995         WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4996
4997         tmp = RREG32(RLC_PG_CNTL);
4998         tmp |= GFX_PG_SRC;
4999         WREG32(RLC_PG_CNTL, tmp);
5000
5001         WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5002
5003         tmp = RREG32(RLC_AUTO_PG_CTRL);
5004
5005         tmp &= ~GRBM_REG_SGIT_MASK;
5006         tmp |= GRBM_REG_SGIT(0x700);
5007         tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
5008         WREG32(RLC_AUTO_PG_CTRL, tmp);
5009 }
5010
5011 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
5012 {
5013         u32 mask = 0, tmp, tmp1;
5014         int i;
5015
5016         si_select_se_sh(rdev, se, sh);
5017         tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5018         tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5019         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5020
5021         tmp &= 0xffff0000;
5022
5023         tmp |= tmp1;
5024         tmp >>= 16;
5025
5026         for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5027                 mask <<= 1;
5028                 mask |= 1;
5029         }
5030
5031         return (~tmp) & mask;
5032 }
5033
5034 static void si_init_ao_cu_mask(struct radeon_device *rdev)
5035 {
5036         u32 i, j, k, active_cu_number = 0;
5037         u32 mask, counter, cu_bitmap;
5038         u32 tmp = 0;
5039
5040         for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5041                 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5042                         mask = 1;
5043                         cu_bitmap = 0;
5044                         counter  = 0;
5045                         for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
5046                                 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
5047                                         if (counter < 2)
5048                                                 cu_bitmap |= mask;
5049                                         counter++;
5050                                 }
5051                                 mask <<= 1;
5052                         }
5053
5054                         active_cu_number += counter;
5055                         tmp |= (cu_bitmap << (i * 16 + j * 8));
5056                 }
5057         }
5058
5059         WREG32(RLC_PG_AO_CU_MASK, tmp);
5060
5061         tmp = RREG32(RLC_MAX_PG_CU);
5062         tmp &= ~MAX_PU_CU_MASK;
5063         tmp |= MAX_PU_CU(active_cu_number);
5064         WREG32(RLC_MAX_PG_CU, tmp);
5065 }
5066
5067 static void si_enable_cgcg(struct radeon_device *rdev,
5068                            bool enable)
5069 {
5070         u32 data, orig, tmp;
5071
5072         orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5073
5074         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5075                 si_enable_gui_idle_interrupt(rdev, true);
5076
5077                 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5078
5079                 tmp = si_halt_rlc(rdev);
5080
5081                 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5082                 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5083                 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5084
5085                 si_wait_for_rlc_serdes(rdev);
5086
5087                 si_update_rlc(rdev, tmp);
5088
5089                 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5090
5091                 data |= CGCG_EN | CGLS_EN;
5092         } else {
5093                 si_enable_gui_idle_interrupt(rdev, false);
5094
5095                 RREG32(CB_CGTT_SCLK_CTRL);
5096                 RREG32(CB_CGTT_SCLK_CTRL);
5097                 RREG32(CB_CGTT_SCLK_CTRL);
5098                 RREG32(CB_CGTT_SCLK_CTRL);
5099
5100                 data &= ~(CGCG_EN | CGLS_EN);
5101         }
5102
5103         if (orig != data)
5104                 WREG32(RLC_CGCG_CGLS_CTRL, data);
5105 }
5106
5107 static void si_enable_mgcg(struct radeon_device *rdev,
5108                            bool enable)
5109 {
5110         u32 data, orig, tmp = 0;
5111
5112         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5113                 orig = data = RREG32(CGTS_SM_CTRL_REG);
5114                 data = 0x96940200;
5115                 if (orig != data)
5116                         WREG32(CGTS_SM_CTRL_REG, data);
5117
5118                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5119                         orig = data = RREG32(CP_MEM_SLP_CNTL);
5120                         data |= CP_MEM_LS_EN;
5121                         if (orig != data)
5122                                 WREG32(CP_MEM_SLP_CNTL, data);
5123                 }
5124
5125                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5126                 data &= 0xffffffc0;
5127                 if (orig != data)
5128                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5129
5130                 tmp = si_halt_rlc(rdev);
5131
5132                 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5133                 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5134                 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5135
5136                 si_update_rlc(rdev, tmp);
5137         } else {
5138                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5139                 data |= 0x00000003;
5140                 if (orig != data)
5141                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5142
5143                 data = RREG32(CP_MEM_SLP_CNTL);
5144                 if (data & CP_MEM_LS_EN) {
5145                         data &= ~CP_MEM_LS_EN;
5146                         WREG32(CP_MEM_SLP_CNTL, data);
5147                 }
5148                 orig = data = RREG32(CGTS_SM_CTRL_REG);
5149                 data |= LS_OVERRIDE | OVERRIDE;
5150                 if (orig != data)
5151                         WREG32(CGTS_SM_CTRL_REG, data);
5152
5153                 tmp = si_halt_rlc(rdev);
5154
5155                 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5156                 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5157                 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5158
5159                 si_update_rlc(rdev, tmp);
5160         }
5161 }
5162
5163 static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5164                                bool enable)
5165 {
5166         u32 orig, data, tmp;
5167
5168         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5169                 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5170                 tmp |= 0x3fff;
5171                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5172
5173                 orig = data = RREG32(UVD_CGC_CTRL);
5174                 data |= DCM;
5175                 if (orig != data)
5176                         WREG32(UVD_CGC_CTRL, data);
5177
5178                 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5179                 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5180         } else {
5181                 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5182                 tmp &= ~0x3fff;
5183                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5184
5185                 orig = data = RREG32(UVD_CGC_CTRL);
5186                 data &= ~DCM;
5187                 if (orig != data)
5188                         WREG32(UVD_CGC_CTRL, data);
5189
5190                 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5191                 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5192         }
5193 }
5194
5195 static const u32 mc_cg_registers[] =
5196 {
5197         MC_HUB_MISC_HUB_CG,
5198         MC_HUB_MISC_SIP_CG,
5199         MC_HUB_MISC_VM_CG,
5200         MC_XPB_CLK_GAT,
5201         ATC_MISC_CG,
5202         MC_CITF_MISC_WR_CG,
5203         MC_CITF_MISC_RD_CG,
5204         MC_CITF_MISC_VM_CG,
5205         VM_L2_CG,
5206 };
5207
5208 static void si_enable_mc_ls(struct radeon_device *rdev,
5209                             bool enable)
5210 {
5211         int i;
5212         u32 orig, data;
5213
5214         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5215                 orig = data = RREG32(mc_cg_registers[i]);
5216                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5217                         data |= MC_LS_ENABLE;
5218                 else
5219                         data &= ~MC_LS_ENABLE;
5220                 if (data != orig)
5221                         WREG32(mc_cg_registers[i], data);
5222         }
5223 }
5224
5225 static void si_enable_mc_mgcg(struct radeon_device *rdev,
5226                                bool enable)
5227 {
5228         int i;
5229         u32 orig, data;
5230
5231         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5232                 orig = data = RREG32(mc_cg_registers[i]);
5233                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5234                         data |= MC_CG_ENABLE;
5235                 else
5236                         data &= ~MC_CG_ENABLE;
5237                 if (data != orig)
5238                         WREG32(mc_cg_registers[i], data);
5239         }
5240 }
5241
5242 static void si_enable_dma_mgcg(struct radeon_device *rdev,
5243                                bool enable)
5244 {
5245         u32 orig, data, offset;
5246         int i;
5247
5248         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5249                 for (i = 0; i < 2; i++) {
5250                         if (i == 0)
5251                                 offset = DMA0_REGISTER_OFFSET;
5252                         else
5253                                 offset = DMA1_REGISTER_OFFSET;
5254                         orig = data = RREG32(DMA_POWER_CNTL + offset);
5255                         data &= ~MEM_POWER_OVERRIDE;
5256                         if (data != orig)
5257                                 WREG32(DMA_POWER_CNTL + offset, data);
5258                         WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5259                 }
5260         } else {
5261                 for (i = 0; i < 2; i++) {
5262                         if (i == 0)
5263                                 offset = DMA0_REGISTER_OFFSET;
5264                         else
5265                                 offset = DMA1_REGISTER_OFFSET;
5266                         orig = data = RREG32(DMA_POWER_CNTL + offset);
5267                         data |= MEM_POWER_OVERRIDE;
5268                         if (data != orig)
5269                                 WREG32(DMA_POWER_CNTL + offset, data);
5270
5271                         orig = data = RREG32(DMA_CLK_CTRL + offset);
5272                         data = 0xff000000;
5273                         if (data != orig)
5274                                 WREG32(DMA_CLK_CTRL + offset, data);
5275                 }
5276         }
5277 }
5278
5279 static void si_enable_bif_mgls(struct radeon_device *rdev,
5280                                bool enable)
5281 {
5282         u32 orig, data;
5283
5284         orig = data = RREG32_PCIE(PCIE_CNTL2);
5285
5286         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5287                 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5288                         REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5289         else
5290                 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5291                           REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5292
5293         if (orig != data)
5294                 WREG32_PCIE(PCIE_CNTL2, data);
5295 }
5296
5297 static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5298                                bool enable)
5299 {
5300         u32 orig, data;
5301
5302         orig = data = RREG32(HDP_HOST_PATH_CNTL);
5303
5304         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5305                 data &= ~CLOCK_GATING_DIS;
5306         else
5307                 data |= CLOCK_GATING_DIS;
5308
5309         if (orig != data)
5310                 WREG32(HDP_HOST_PATH_CNTL, data);
5311 }
5312
5313 static void si_enable_hdp_ls(struct radeon_device *rdev,
5314                              bool enable)
5315 {
5316         u32 orig, data;
5317
5318         orig = data = RREG32(HDP_MEM_POWER_LS);
5319
5320         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5321                 data |= HDP_LS_ENABLE;
5322         else
5323                 data &= ~HDP_LS_ENABLE;
5324
5325         if (orig != data)
5326                 WREG32(HDP_MEM_POWER_LS, data);
5327 }
5328
5329 static void si_update_cg(struct radeon_device *rdev,
5330                          u32 block, bool enable)
5331 {
5332         if (block & RADEON_CG_BLOCK_GFX) {
5333                 si_enable_gui_idle_interrupt(rdev, false);
5334                 /* order matters! */
5335                 if (enable) {
5336                         si_enable_mgcg(rdev, true);
5337                         si_enable_cgcg(rdev, true);
5338                 } else {
5339                         si_enable_cgcg(rdev, false);
5340                         si_enable_mgcg(rdev, false);
5341                 }
5342                 si_enable_gui_idle_interrupt(rdev, true);
5343         }
5344
5345         if (block & RADEON_CG_BLOCK_MC) {
5346                 si_enable_mc_mgcg(rdev, enable);
5347                 si_enable_mc_ls(rdev, enable);
5348         }
5349
5350         if (block & RADEON_CG_BLOCK_SDMA) {
5351                 si_enable_dma_mgcg(rdev, enable);
5352         }
5353
5354         if (block & RADEON_CG_BLOCK_BIF) {
5355                 si_enable_bif_mgls(rdev, enable);
5356         }
5357
5358         if (block & RADEON_CG_BLOCK_UVD) {
5359                 if (rdev->has_uvd) {
5360                         si_enable_uvd_mgcg(rdev, enable);
5361                 }
5362         }
5363
5364         if (block & RADEON_CG_BLOCK_HDP) {
5365                 si_enable_hdp_mgcg(rdev, enable);
5366                 si_enable_hdp_ls(rdev, enable);
5367         }
5368 }
5369
5370 static void si_init_cg(struct radeon_device *rdev)
5371 {
5372         si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5373                             RADEON_CG_BLOCK_MC |
5374                             RADEON_CG_BLOCK_SDMA |
5375                             RADEON_CG_BLOCK_BIF |
5376                             RADEON_CG_BLOCK_HDP), true);
5377         if (rdev->has_uvd) {
5378                 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
5379                 si_init_uvd_internal_cg(rdev);
5380         }
5381 }
5382
5383 static void si_fini_cg(struct radeon_device *rdev)
5384 {
5385         if (rdev->has_uvd) {
5386                 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
5387         }
5388         si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5389                             RADEON_CG_BLOCK_MC |
5390                             RADEON_CG_BLOCK_SDMA |
5391                             RADEON_CG_BLOCK_BIF |
5392                             RADEON_CG_BLOCK_HDP), false);
5393 }
5394
5395 u32 si_get_csb_size(struct radeon_device *rdev)
5396 {
5397         u32 count = 0;
5398         const struct cs_section_def *sect = NULL;
5399         const struct cs_extent_def *ext = NULL;
5400
5401         if (rdev->rlc.cs_data == NULL)
5402                 return 0;
5403
5404         /* begin clear state */
5405         count += 2;
5406         /* context control state */
5407         count += 3;
5408
5409         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5410                 for (ext = sect->section; ext->extent != NULL; ++ext) {
5411                         if (sect->id == SECT_CONTEXT)
5412                                 count += 2 + ext->reg_count;
5413                         else
5414                                 return 0;
5415                 }
5416         }
5417         /* pa_sc_raster_config */
5418         count += 3;
5419         /* end clear state */
5420         count += 2;
5421         /* clear state */
5422         count += 2;
5423
5424         return count;
5425 }
5426
5427 void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5428 {
5429         u32 count = 0, i;
5430         const struct cs_section_def *sect = NULL;
5431         const struct cs_extent_def *ext = NULL;
5432
5433         if (rdev->rlc.cs_data == NULL)
5434                 return;
5435         if (buffer == NULL)
5436                 return;
5437
5438         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5439         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
5440
5441         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5442         buffer[count++] = cpu_to_le32(0x80000000);
5443         buffer[count++] = cpu_to_le32(0x80000000);
5444
5445         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5446                 for (ext = sect->section; ext->extent != NULL; ++ext) {
5447                         if (sect->id == SECT_CONTEXT) {
5448                                 buffer[count++] =
5449                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5450                                 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
5451                                 for (i = 0; i < ext->reg_count; i++)
5452                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
5453                         } else {
5454                                 return;
5455                         }
5456                 }
5457         }
5458
5459         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5460         buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
5461         switch (rdev->family) {
5462         case CHIP_TAHITI:
5463         case CHIP_PITCAIRN:
5464                 buffer[count++] = cpu_to_le32(0x2a00126a);
5465                 break;
5466         case CHIP_VERDE:
5467                 buffer[count++] = cpu_to_le32(0x0000124a);
5468                 break;
5469         case CHIP_OLAND:
5470                 buffer[count++] = cpu_to_le32(0x00000082);
5471                 break;
5472         case CHIP_HAINAN:
5473                 buffer[count++] = cpu_to_le32(0x00000000);
5474                 break;
5475         default:
5476                 buffer[count++] = cpu_to_le32(0x00000000);
5477                 break;
5478         }
5479
5480         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5481         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
5482
5483         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5484         buffer[count++] = cpu_to_le32(0);
5485 }
5486
5487 static void si_init_pg(struct radeon_device *rdev)
5488 {
5489         if (rdev->pg_flags) {
5490                 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5491                         si_init_dma_pg(rdev);
5492                 }
5493                 si_init_ao_cu_mask(rdev);
5494                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5495                         si_init_gfx_cgpg(rdev);
5496                 } else {
5497                         WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5498                         WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5499                 }
5500                 si_enable_dma_pg(rdev, true);
5501                 si_enable_gfx_cgpg(rdev, true);
5502         } else {
5503                 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5504                 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5505         }
5506 }
5507
5508 static void si_fini_pg(struct radeon_device *rdev)
5509 {
5510         if (rdev->pg_flags) {
5511                 si_enable_dma_pg(rdev, false);
5512                 si_enable_gfx_cgpg(rdev, false);
5513         }
5514 }
5515
5516 /*
5517  * RLC
5518  */
5519 void si_rlc_reset(struct radeon_device *rdev)
5520 {
5521         u32 tmp = RREG32(GRBM_SOFT_RESET);
5522
5523         tmp |= SOFT_RESET_RLC;
5524         WREG32(GRBM_SOFT_RESET, tmp);
5525         udelay(50);
5526         tmp &= ~SOFT_RESET_RLC;
5527         WREG32(GRBM_SOFT_RESET, tmp);
5528         udelay(50);
5529 }
5530
5531 static void si_rlc_stop(struct radeon_device *rdev)
5532 {
5533         WREG32(RLC_CNTL, 0);
5534
5535         si_enable_gui_idle_interrupt(rdev, false);
5536
5537         si_wait_for_rlc_serdes(rdev);
5538 }
5539
5540 static void si_rlc_start(struct radeon_device *rdev)
5541 {
5542         WREG32(RLC_CNTL, RLC_ENABLE);
5543
5544         si_enable_gui_idle_interrupt(rdev, true);
5545
5546         udelay(50);
5547 }
5548
5549 static bool si_lbpw_supported(struct radeon_device *rdev)
5550 {
5551         u32 tmp;
5552
5553         /* Enable LBPW only for DDR3 */
5554         tmp = RREG32(MC_SEQ_MISC0);
5555         if ((tmp & 0xF0000000) == 0xB0000000)
5556                 return true;
5557         return false;
5558 }
5559
5560 static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5561 {
5562         u32 tmp;
5563
5564         tmp = RREG32(RLC_LB_CNTL);
5565         if (enable)
5566                 tmp |= LOAD_BALANCE_ENABLE;
5567         else
5568                 tmp &= ~LOAD_BALANCE_ENABLE;
5569         WREG32(RLC_LB_CNTL, tmp);
5570
5571         if (!enable) {
5572                 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5573                 WREG32(SPI_LB_CU_MASK, 0x00ff);
5574         }
5575 }
5576
5577 static int si_rlc_resume(struct radeon_device *rdev)
5578 {
5579         u32 i;
5580         const __be32 *fw_data;
5581
5582         if (!rdev->rlc_fw)
5583                 return -EINVAL;
5584
5585         si_rlc_stop(rdev);
5586
5587         si_rlc_reset(rdev);
5588
5589         si_init_pg(rdev);
5590
5591         si_init_cg(rdev);
5592
5593         WREG32(RLC_RL_BASE, 0);
5594         WREG32(RLC_RL_SIZE, 0);
5595         WREG32(RLC_LB_CNTL, 0);
5596         WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5597         WREG32(RLC_LB_CNTR_INIT, 0);
5598         WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5599
5600         WREG32(RLC_MC_CNTL, 0);
5601         WREG32(RLC_UCODE_CNTL, 0);
5602
5603         fw_data = (const __be32 *)rdev->rlc_fw->data;
5604         for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5605                 WREG32(RLC_UCODE_ADDR, i);
5606                 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5607         }
5608         WREG32(RLC_UCODE_ADDR, 0);
5609
5610         si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5611
5612         si_rlc_start(rdev);
5613
5614         return 0;
5615 }
5616
5617 static void si_enable_interrupts(struct radeon_device *rdev)
5618 {
5619         u32 ih_cntl = RREG32(IH_CNTL);
5620         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5621
5622         ih_cntl |= ENABLE_INTR;
5623         ih_rb_cntl |= IH_RB_ENABLE;
5624         WREG32(IH_CNTL, ih_cntl);
5625         WREG32(IH_RB_CNTL, ih_rb_cntl);
5626         rdev->ih.enabled = true;
5627 }
5628
5629 static void si_disable_interrupts(struct radeon_device *rdev)
5630 {
5631         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5632         u32 ih_cntl = RREG32(IH_CNTL);
5633
5634         ih_rb_cntl &= ~IH_RB_ENABLE;
5635         ih_cntl &= ~ENABLE_INTR;
5636         WREG32(IH_RB_CNTL, ih_rb_cntl);
5637         WREG32(IH_CNTL, ih_cntl);
5638         /* set rptr, wptr to 0 */
5639         WREG32(IH_RB_RPTR, 0);
5640         WREG32(IH_RB_WPTR, 0);
5641         rdev->ih.enabled = false;
5642         rdev->ih.rptr = 0;
5643 }
5644
5645 static void si_disable_interrupt_state(struct radeon_device *rdev)
5646 {
5647         u32 tmp;
5648
5649         tmp = RREG32(CP_INT_CNTL_RING0) &
5650                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5651         WREG32(CP_INT_CNTL_RING0, tmp);
5652         WREG32(CP_INT_CNTL_RING1, 0);
5653         WREG32(CP_INT_CNTL_RING2, 0);
5654         tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5655         WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5656         tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5657         WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
5658         WREG32(GRBM_INT_CNTL, 0);
5659         if (rdev->num_crtc >= 2) {
5660                 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5661                 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5662         }
5663         if (rdev->num_crtc >= 4) {
5664                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5665                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5666         }
5667         if (rdev->num_crtc >= 6) {
5668                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5669                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5670         }
5671
5672         if (rdev->num_crtc >= 2) {
5673                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5674                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5675         }
5676         if (rdev->num_crtc >= 4) {
5677                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5678                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5679         }
5680         if (rdev->num_crtc >= 6) {
5681                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5682                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5683         }
5684
5685         if (!ASIC_IS_NODCE(rdev)) {
5686                 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5687
5688                 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5689                 WREG32(DC_HPD1_INT_CONTROL, tmp);
5690                 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5691                 WREG32(DC_HPD2_INT_CONTROL, tmp);
5692                 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5693                 WREG32(DC_HPD3_INT_CONTROL, tmp);
5694                 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5695                 WREG32(DC_HPD4_INT_CONTROL, tmp);
5696                 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5697                 WREG32(DC_HPD5_INT_CONTROL, tmp);
5698                 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5699                 WREG32(DC_HPD6_INT_CONTROL, tmp);
5700         }
5701 }
5702
5703 static int si_irq_init(struct radeon_device *rdev)
5704 {
5705         int ret = 0;
5706         int rb_bufsz;
5707         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5708
5709         /* allocate ring */
5710         ret = r600_ih_ring_alloc(rdev);
5711         if (ret)
5712                 return ret;
5713
5714         /* disable irqs */
5715         si_disable_interrupts(rdev);
5716
5717         /* init rlc */
5718         ret = si_rlc_resume(rdev);
5719         if (ret) {
5720                 r600_ih_ring_fini(rdev);
5721                 return ret;
5722         }
5723
5724         /* setup interrupt control */
5725         /* set dummy read address to ring address */
5726         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5727         interrupt_cntl = RREG32(INTERRUPT_CNTL);
5728         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5729          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5730          */
5731         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5732         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5733         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5734         WREG32(INTERRUPT_CNTL, interrupt_cntl);
5735
5736         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5737         rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
5738
5739         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5740                       IH_WPTR_OVERFLOW_CLEAR |
5741                       (rb_bufsz << 1));
5742
5743         if (rdev->wb.enabled)
5744                 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5745
5746         /* set the writeback address whether it's enabled or not */
5747         WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5748         WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5749
5750         WREG32(IH_RB_CNTL, ih_rb_cntl);
5751
5752         /* set rptr, wptr to 0 */
5753         WREG32(IH_RB_RPTR, 0);
5754         WREG32(IH_RB_WPTR, 0);
5755
5756         /* Default settings for IH_CNTL (disabled at first) */
5757         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5758         /* RPTR_REARM only works if msi's are enabled */
5759         if (rdev->msi_enabled)
5760                 ih_cntl |= RPTR_REARM;
5761         WREG32(IH_CNTL, ih_cntl);
5762
5763         /* force the active interrupt state to all disabled */
5764         si_disable_interrupt_state(rdev);
5765
5766         pci_set_master(rdev->pdev);
5767
5768         /* enable irqs */
5769         si_enable_interrupts(rdev);
5770
5771         return ret;
5772 }
5773
5774 int si_irq_set(struct radeon_device *rdev)
5775 {
5776         u32 cp_int_cntl;
5777         u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
5778         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5779         u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
5780         u32 grbm_int_cntl = 0;
5781         u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
5782         u32 dma_cntl, dma_cntl1;
5783         u32 thermal_int = 0;
5784
5785         if (!rdev->irq.installed) {
5786                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5787                 return -EINVAL;
5788         }
5789         /* don't enable anything if the ih is disabled */
5790         if (!rdev->ih.enabled) {
5791                 si_disable_interrupts(rdev);
5792                 /* force the active interrupt state to all disabled */
5793                 si_disable_interrupt_state(rdev);
5794                 return 0;
5795         }
5796
5797         cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
5798                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5799
5800         if (!ASIC_IS_NODCE(rdev)) {
5801                 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
5802                 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
5803                 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
5804                 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
5805                 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
5806                 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
5807         }
5808
5809         dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5810         dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5811
5812         thermal_int = RREG32(CG_THERMAL_INT) &
5813                 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
5814
5815         /* enable CP interrupts on all rings */
5816         if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
5817                 DRM_DEBUG("si_irq_set: sw int gfx\n");
5818                 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
5819         }
5820         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
5821                 DRM_DEBUG("si_irq_set: sw int cp1\n");
5822                 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
5823         }
5824         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
5825                 DRM_DEBUG("si_irq_set: sw int cp2\n");
5826                 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
5827         }
5828         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
5829                 DRM_DEBUG("si_irq_set: sw int dma\n");
5830                 dma_cntl |= TRAP_ENABLE;
5831         }
5832
5833         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
5834                 DRM_DEBUG("si_irq_set: sw int dma1\n");
5835                 dma_cntl1 |= TRAP_ENABLE;
5836         }
5837         if (rdev->irq.crtc_vblank_int[0] ||
5838             atomic_read(&rdev->irq.pflip[0])) {
5839                 DRM_DEBUG("si_irq_set: vblank 0\n");
5840                 crtc1 |= VBLANK_INT_MASK;
5841         }
5842         if (rdev->irq.crtc_vblank_int[1] ||
5843             atomic_read(&rdev->irq.pflip[1])) {
5844                 DRM_DEBUG("si_irq_set: vblank 1\n");
5845                 crtc2 |= VBLANK_INT_MASK;
5846         }
5847         if (rdev->irq.crtc_vblank_int[2] ||
5848             atomic_read(&rdev->irq.pflip[2])) {
5849                 DRM_DEBUG("si_irq_set: vblank 2\n");
5850                 crtc3 |= VBLANK_INT_MASK;
5851         }
5852         if (rdev->irq.crtc_vblank_int[3] ||
5853             atomic_read(&rdev->irq.pflip[3])) {
5854                 DRM_DEBUG("si_irq_set: vblank 3\n");
5855                 crtc4 |= VBLANK_INT_MASK;
5856         }
5857         if (rdev->irq.crtc_vblank_int[4] ||
5858             atomic_read(&rdev->irq.pflip[4])) {
5859                 DRM_DEBUG("si_irq_set: vblank 4\n");
5860                 crtc5 |= VBLANK_INT_MASK;
5861         }
5862         if (rdev->irq.crtc_vblank_int[5] ||
5863             atomic_read(&rdev->irq.pflip[5])) {
5864                 DRM_DEBUG("si_irq_set: vblank 5\n");
5865                 crtc6 |= VBLANK_INT_MASK;
5866         }
5867         if (rdev->irq.hpd[0]) {
5868                 DRM_DEBUG("si_irq_set: hpd 1\n");
5869                 hpd1 |= DC_HPDx_INT_EN;
5870         }
5871         if (rdev->irq.hpd[1]) {
5872                 DRM_DEBUG("si_irq_set: hpd 2\n");
5873                 hpd2 |= DC_HPDx_INT_EN;
5874         }
5875         if (rdev->irq.hpd[2]) {
5876                 DRM_DEBUG("si_irq_set: hpd 3\n");
5877                 hpd3 |= DC_HPDx_INT_EN;
5878         }
5879         if (rdev->irq.hpd[3]) {
5880                 DRM_DEBUG("si_irq_set: hpd 4\n");
5881                 hpd4 |= DC_HPDx_INT_EN;
5882         }
5883         if (rdev->irq.hpd[4]) {
5884                 DRM_DEBUG("si_irq_set: hpd 5\n");
5885                 hpd5 |= DC_HPDx_INT_EN;
5886         }
5887         if (rdev->irq.hpd[5]) {
5888                 DRM_DEBUG("si_irq_set: hpd 6\n");
5889                 hpd6 |= DC_HPDx_INT_EN;
5890         }
5891
5892         WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
5893         WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
5894         WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
5895
5896         WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
5897         WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
5898
5899         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
5900
5901         if (rdev->irq.dpm_thermal) {
5902                 DRM_DEBUG("dpm thermal\n");
5903                 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
5904         }
5905
5906         if (rdev->num_crtc >= 2) {
5907                 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
5908                 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
5909         }
5910         if (rdev->num_crtc >= 4) {
5911                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
5912                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
5913         }
5914         if (rdev->num_crtc >= 6) {
5915                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
5916                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
5917         }
5918
5919         if (rdev->num_crtc >= 2) {
5920                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
5921                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
5922         }
5923         if (rdev->num_crtc >= 4) {
5924                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
5925                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
5926         }
5927         if (rdev->num_crtc >= 6) {
5928                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
5929                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
5930         }
5931
5932         if (!ASIC_IS_NODCE(rdev)) {
5933                 WREG32(DC_HPD1_INT_CONTROL, hpd1);
5934                 WREG32(DC_HPD2_INT_CONTROL, hpd2);
5935                 WREG32(DC_HPD3_INT_CONTROL, hpd3);
5936                 WREG32(DC_HPD4_INT_CONTROL, hpd4);
5937                 WREG32(DC_HPD5_INT_CONTROL, hpd5);
5938                 WREG32(DC_HPD6_INT_CONTROL, hpd6);
5939         }
5940
5941         WREG32(CG_THERMAL_INT, thermal_int);
5942
5943         return 0;
5944 }
5945
5946 static inline void si_irq_ack(struct radeon_device *rdev)
5947 {
5948         u32 tmp;
5949
5950         if (ASIC_IS_NODCE(rdev))
5951                 return;
5952
5953         rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
5954         rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
5955         rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
5956         rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
5957         rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
5958         rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
5959         rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
5960         rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
5961         if (rdev->num_crtc >= 4) {
5962                 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
5963                 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
5964         }
5965         if (rdev->num_crtc >= 6) {
5966                 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
5967                 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
5968         }
5969
5970         if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
5971                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5972         if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
5973                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5974         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
5975                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
5976         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
5977                 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
5978         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
5979                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
5980         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
5981                 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
5982
5983         if (rdev->num_crtc >= 4) {
5984                 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
5985                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5986                 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
5987                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5988                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
5989                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
5990                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
5991                         WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
5992                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
5993                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
5994                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
5995                         WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
5996         }
5997
5998         if (rdev->num_crtc >= 6) {
5999                 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
6000                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6001                 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
6002                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6003                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6004                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6005                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6006                         WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6007                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6008                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6009                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6010                         WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6011         }
6012
6013         if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6014                 tmp = RREG32(DC_HPD1_INT_CONTROL);
6015                 tmp |= DC_HPDx_INT_ACK;
6016                 WREG32(DC_HPD1_INT_CONTROL, tmp);
6017         }
6018         if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6019                 tmp = RREG32(DC_HPD2_INT_CONTROL);
6020                 tmp |= DC_HPDx_INT_ACK;
6021                 WREG32(DC_HPD2_INT_CONTROL, tmp);
6022         }
6023         if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6024                 tmp = RREG32(DC_HPD3_INT_CONTROL);
6025                 tmp |= DC_HPDx_INT_ACK;
6026                 WREG32(DC_HPD3_INT_CONTROL, tmp);
6027         }
6028         if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6029                 tmp = RREG32(DC_HPD4_INT_CONTROL);
6030                 tmp |= DC_HPDx_INT_ACK;
6031                 WREG32(DC_HPD4_INT_CONTROL, tmp);
6032         }
6033         if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6034                 tmp = RREG32(DC_HPD5_INT_CONTROL);
6035                 tmp |= DC_HPDx_INT_ACK;
6036                 WREG32(DC_HPD5_INT_CONTROL, tmp);
6037         }
6038         if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6039                 tmp = RREG32(DC_HPD5_INT_CONTROL);
6040                 tmp |= DC_HPDx_INT_ACK;
6041                 WREG32(DC_HPD6_INT_CONTROL, tmp);
6042         }
6043 }
6044
6045 static void si_irq_disable(struct radeon_device *rdev)
6046 {
6047         si_disable_interrupts(rdev);
6048         /* Wait and acknowledge irq */
6049         mdelay(1);
6050         si_irq_ack(rdev);
6051         si_disable_interrupt_state(rdev);
6052 }
6053
6054 static void si_irq_suspend(struct radeon_device *rdev)
6055 {
6056         si_irq_disable(rdev);
6057         si_rlc_stop(rdev);
6058 }
6059
6060 static void si_irq_fini(struct radeon_device *rdev)
6061 {
6062         si_irq_suspend(rdev);
6063         r600_ih_ring_fini(rdev);
6064 }
6065
6066 static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6067 {
6068         u32 wptr, tmp;
6069
6070         if (rdev->wb.enabled)
6071                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6072         else
6073                 wptr = RREG32(IH_RB_WPTR);
6074
6075         if (wptr & RB_OVERFLOW) {
6076                 /* When a ring buffer overflow happen start parsing interrupt
6077                  * from the last not overwritten vector (wptr + 16). Hopefully
6078                  * this should allow us to catchup.
6079                  */
6080                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6081                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6082                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6083                 tmp = RREG32(IH_RB_CNTL);
6084                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6085                 WREG32(IH_RB_CNTL, tmp);
6086         }
6087         return (wptr & rdev->ih.ptr_mask);
6088 }
6089
6090 /*        SI IV Ring
6091  * Each IV ring entry is 128 bits:
6092  * [7:0]    - interrupt source id
6093  * [31:8]   - reserved
6094  * [59:32]  - interrupt source data
6095  * [63:60]  - reserved
6096  * [71:64]  - RINGID
6097  * [79:72]  - VMID
6098  * [127:80] - reserved
6099  */
6100 int si_irq_process(struct radeon_device *rdev)
6101 {
6102         u32 wptr;
6103         u32 rptr;
6104         u32 src_id, src_data, ring_id;
6105         u32 ring_index;
6106         bool queue_hotplug = false;
6107         bool queue_thermal = false;
6108         u32 status, addr;
6109
6110         if (!rdev->ih.enabled || rdev->shutdown)
6111                 return IRQ_NONE;
6112
6113         wptr = si_get_ih_wptr(rdev);
6114
6115 restart_ih:
6116         /* is somebody else already processing irqs? */
6117         if (atomic_xchg(&rdev->ih.lock, 1))
6118                 return IRQ_NONE;
6119
6120         rptr = rdev->ih.rptr;
6121         DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6122
6123         /* Order reading of wptr vs. reading of IH ring data */
6124         rmb();
6125
6126         /* display interrupts */
6127         si_irq_ack(rdev);
6128
6129         while (rptr != wptr) {
6130                 /* wptr/rptr are in bytes! */
6131                 ring_index = rptr / 4;
6132                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6133                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6134                 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6135
6136                 switch (src_id) {
6137                 case 1: /* D1 vblank/vline */
6138                         switch (src_data) {
6139                         case 0: /* D1 vblank */
6140                                 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6141                                         if (rdev->irq.crtc_vblank_int[0]) {
6142                                                 drm_handle_vblank(rdev->ddev, 0);
6143                                                 rdev->pm.vblank_sync = true;
6144                                                 wake_up(&rdev->irq.vblank_queue);
6145                                         }
6146                                         if (atomic_read(&rdev->irq.pflip[0]))
6147                                                 radeon_crtc_handle_flip(rdev, 0);
6148                                         rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6149                                         DRM_DEBUG("IH: D1 vblank\n");
6150                                 }
6151                                 break;
6152                         case 1: /* D1 vline */
6153                                 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
6154                                         rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6155                                         DRM_DEBUG("IH: D1 vline\n");
6156                                 }
6157                                 break;
6158                         default:
6159                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6160                                 break;
6161                         }
6162                         break;
6163                 case 2: /* D2 vblank/vline */
6164                         switch (src_data) {
6165                         case 0: /* D2 vblank */
6166                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6167                                         if (rdev->irq.crtc_vblank_int[1]) {
6168                                                 drm_handle_vblank(rdev->ddev, 1);
6169                                                 rdev->pm.vblank_sync = true;
6170                                                 wake_up(&rdev->irq.vblank_queue);
6171                                         }
6172                                         if (atomic_read(&rdev->irq.pflip[1]))
6173                                                 radeon_crtc_handle_flip(rdev, 1);
6174                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6175                                         DRM_DEBUG("IH: D2 vblank\n");
6176                                 }
6177                                 break;
6178                         case 1: /* D2 vline */
6179                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6180                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6181                                         DRM_DEBUG("IH: D2 vline\n");
6182                                 }
6183                                 break;
6184                         default:
6185                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6186                                 break;
6187                         }
6188                         break;
6189                 case 3: /* D3 vblank/vline */
6190                         switch (src_data) {
6191                         case 0: /* D3 vblank */
6192                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6193                                         if (rdev->irq.crtc_vblank_int[2]) {
6194                                                 drm_handle_vblank(rdev->ddev, 2);
6195                                                 rdev->pm.vblank_sync = true;
6196                                                 wake_up(&rdev->irq.vblank_queue);
6197                                         }
6198                                         if (atomic_read(&rdev->irq.pflip[2]))
6199                                                 radeon_crtc_handle_flip(rdev, 2);
6200                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6201                                         DRM_DEBUG("IH: D3 vblank\n");
6202                                 }
6203                                 break;
6204                         case 1: /* D3 vline */
6205                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6206                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6207                                         DRM_DEBUG("IH: D3 vline\n");
6208                                 }
6209                                 break;
6210                         default:
6211                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6212                                 break;
6213                         }
6214                         break;
6215                 case 4: /* D4 vblank/vline */
6216                         switch (src_data) {
6217                         case 0: /* D4 vblank */
6218                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6219                                         if (rdev->irq.crtc_vblank_int[3]) {
6220                                                 drm_handle_vblank(rdev->ddev, 3);
6221                                                 rdev->pm.vblank_sync = true;
6222                                                 wake_up(&rdev->irq.vblank_queue);
6223                                         }
6224                                         if (atomic_read(&rdev->irq.pflip[3]))
6225                                                 radeon_crtc_handle_flip(rdev, 3);
6226                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6227                                         DRM_DEBUG("IH: D4 vblank\n");
6228                                 }
6229                                 break;
6230                         case 1: /* D4 vline */
6231                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6232                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6233                                         DRM_DEBUG("IH: D4 vline\n");
6234                                 }
6235                                 break;
6236                         default:
6237                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6238                                 break;
6239                         }
6240                         break;
6241                 case 5: /* D5 vblank/vline */
6242                         switch (src_data) {
6243                         case 0: /* D5 vblank */
6244                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6245                                         if (rdev->irq.crtc_vblank_int[4]) {
6246                                                 drm_handle_vblank(rdev->ddev, 4);
6247                                                 rdev->pm.vblank_sync = true;
6248                                                 wake_up(&rdev->irq.vblank_queue);
6249                                         }
6250                                         if (atomic_read(&rdev->irq.pflip[4]))
6251                                                 radeon_crtc_handle_flip(rdev, 4);
6252                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6253                                         DRM_DEBUG("IH: D5 vblank\n");
6254                                 }
6255                                 break;
6256                         case 1: /* D5 vline */
6257                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6258                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6259                                         DRM_DEBUG("IH: D5 vline\n");
6260                                 }
6261                                 break;
6262                         default:
6263                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6264                                 break;
6265                         }
6266                         break;
6267                 case 6: /* D6 vblank/vline */
6268                         switch (src_data) {
6269                         case 0: /* D6 vblank */
6270                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6271                                         if (rdev->irq.crtc_vblank_int[5]) {
6272                                                 drm_handle_vblank(rdev->ddev, 5);
6273                                                 rdev->pm.vblank_sync = true;
6274                                                 wake_up(&rdev->irq.vblank_queue);
6275                                         }
6276                                         if (atomic_read(&rdev->irq.pflip[5]))
6277                                                 radeon_crtc_handle_flip(rdev, 5);
6278                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6279                                         DRM_DEBUG("IH: D6 vblank\n");
6280                                 }
6281                                 break;
6282                         case 1: /* D6 vline */
6283                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6284                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6285                                         DRM_DEBUG("IH: D6 vline\n");
6286                                 }
6287                                 break;
6288                         default:
6289                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6290                                 break;
6291                         }
6292                         break;
6293                 case 42: /* HPD hotplug */
6294                         switch (src_data) {
6295                         case 0:
6296                                 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6297                                         rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6298                                         queue_hotplug = true;
6299                                         DRM_DEBUG("IH: HPD1\n");
6300                                 }
6301                                 break;
6302                         case 1:
6303                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6304                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6305                                         queue_hotplug = true;
6306                                         DRM_DEBUG("IH: HPD2\n");
6307                                 }
6308                                 break;
6309                         case 2:
6310                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6311                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6312                                         queue_hotplug = true;
6313                                         DRM_DEBUG("IH: HPD3\n");
6314                                 }
6315                                 break;
6316                         case 3:
6317                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6318                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6319                                         queue_hotplug = true;
6320                                         DRM_DEBUG("IH: HPD4\n");
6321                                 }
6322                                 break;
6323                         case 4:
6324                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6325                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6326                                         queue_hotplug = true;
6327                                         DRM_DEBUG("IH: HPD5\n");
6328                                 }
6329                                 break;
6330                         case 5:
6331                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6332                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6333                                         queue_hotplug = true;
6334                                         DRM_DEBUG("IH: HPD6\n");
6335                                 }
6336                                 break;
6337                         default:
6338                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6339                                 break;
6340                         }
6341                         break;
6342                 case 124: /* UVD */
6343                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6344                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6345                         break;
6346                 case 146:
6347                 case 147:
6348                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6349                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6350                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6351                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
6352                                 addr);
6353                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6354                                 status);
6355                         si_vm_decode_fault(rdev, status, addr);
6356                         /* reset addr and status */
6357                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6358                         break;
6359                 case 176: /* RINGID0 CP_INT */
6360                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6361                         break;
6362                 case 177: /* RINGID1 CP_INT */
6363                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6364                         break;
6365                 case 178: /* RINGID2 CP_INT */
6366                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6367                         break;
6368                 case 181: /* CP EOP event */
6369                         DRM_DEBUG("IH: CP EOP\n");
6370                         switch (ring_id) {
6371                         case 0:
6372                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6373                                 break;
6374                         case 1:
6375                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6376                                 break;
6377                         case 2:
6378                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6379                                 break;
6380                         }
6381                         break;
6382                 case 224: /* DMA trap event */
6383                         DRM_DEBUG("IH: DMA trap\n");
6384                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6385                         break;
6386                 case 230: /* thermal low to high */
6387                         DRM_DEBUG("IH: thermal low to high\n");
6388                         rdev->pm.dpm.thermal.high_to_low = false;
6389                         queue_thermal = true;
6390                         break;
6391                 case 231: /* thermal high to low */
6392                         DRM_DEBUG("IH: thermal high to low\n");
6393                         rdev->pm.dpm.thermal.high_to_low = true;
6394                         queue_thermal = true;
6395                         break;
6396                 case 233: /* GUI IDLE */
6397                         DRM_DEBUG("IH: GUI idle\n");
6398                         break;
6399                 case 244: /* DMA trap event */
6400                         DRM_DEBUG("IH: DMA1 trap\n");
6401                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6402                         break;
6403                 default:
6404                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6405                         break;
6406                 }
6407
6408                 /* wptr/rptr are in bytes! */
6409                 rptr += 16;
6410                 rptr &= rdev->ih.ptr_mask;
6411         }
6412         if (queue_hotplug)
6413                 schedule_work(&rdev->hotplug_work);
6414         if (queue_thermal && rdev->pm.dpm_enabled)
6415                 schedule_work(&rdev->pm.dpm.thermal.work);
6416         rdev->ih.rptr = rptr;
6417         WREG32(IH_RB_RPTR, rdev->ih.rptr);
6418         atomic_set(&rdev->ih.lock, 0);
6419
6420         /* make sure wptr hasn't changed while processing */
6421         wptr = si_get_ih_wptr(rdev);
6422         if (wptr != rptr)
6423                 goto restart_ih;
6424
6425         return IRQ_HANDLED;
6426 }
6427
6428 /*
6429  * startup/shutdown callbacks
6430  */
6431 static int si_startup(struct radeon_device *rdev)
6432 {
6433         struct radeon_ring *ring;
6434         int r;
6435
6436         /* enable pcie gen2/3 link */
6437         si_pcie_gen3_enable(rdev);
6438         /* enable aspm */
6439         si_program_aspm(rdev);
6440
6441         /* scratch needs to be initialized before MC */
6442         r = r600_vram_scratch_init(rdev);
6443         if (r)
6444                 return r;
6445
6446         si_mc_program(rdev);
6447
6448         if (!rdev->pm.dpm_enabled) {
6449                 r = si_mc_load_microcode(rdev);
6450                 if (r) {
6451                         DRM_ERROR("Failed to load MC firmware!\n");
6452                         return r;
6453                 }
6454         }
6455
6456         r = si_pcie_gart_enable(rdev);
6457         if (r)
6458                 return r;
6459         si_gpu_init(rdev);
6460
6461         /* allocate rlc buffers */
6462         if (rdev->family == CHIP_VERDE) {
6463                 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6464                 rdev->rlc.reg_list_size =
6465                         (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6466         }
6467         rdev->rlc.cs_data = si_cs_data;
6468         r = sumo_rlc_init(rdev);
6469         if (r) {
6470                 DRM_ERROR("Failed to init rlc BOs!\n");
6471                 return r;
6472         }
6473
6474         /* allocate wb buffer */
6475         r = radeon_wb_init(rdev);
6476         if (r)
6477                 return r;
6478
6479         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6480         if (r) {
6481                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6482                 return r;
6483         }
6484
6485         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6486         if (r) {
6487                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6488                 return r;
6489         }
6490
6491         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6492         if (r) {
6493                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6494                 return r;
6495         }
6496
6497         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6498         if (r) {
6499                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6500                 return r;
6501         }
6502
6503         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6504         if (r) {
6505                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6506                 return r;
6507         }
6508
6509         if (rdev->has_uvd) {
6510                 r = uvd_v2_2_resume(rdev);
6511                 if (!r) {
6512                         r = radeon_fence_driver_start_ring(rdev,
6513                                                            R600_RING_TYPE_UVD_INDEX);
6514                         if (r)
6515                                 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6516                 }
6517                 if (r)
6518                         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6519         }
6520
6521         /* Enable IRQ */
6522         if (!rdev->irq.installed) {
6523                 r = radeon_irq_kms_init(rdev);
6524                 if (r)
6525                         return r;
6526         }
6527
6528         r = si_irq_init(rdev);
6529         if (r) {
6530                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6531                 radeon_irq_kms_fini(rdev);
6532                 return r;
6533         }
6534         si_irq_set(rdev);
6535
6536         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6537         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6538                              RADEON_CP_PACKET2);
6539         if (r)
6540                 return r;
6541
6542         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6543         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6544                              RADEON_CP_PACKET2);
6545         if (r)
6546                 return r;
6547
6548         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6549         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6550                              RADEON_CP_PACKET2);
6551         if (r)
6552                 return r;
6553
6554         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6555         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6556                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6557         if (r)
6558                 return r;
6559
6560         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6561         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6562                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6563         if (r)
6564                 return r;
6565
6566         r = si_cp_load_microcode(rdev);
6567         if (r)
6568                 return r;
6569         r = si_cp_resume(rdev);
6570         if (r)
6571                 return r;
6572
6573         r = cayman_dma_resume(rdev);
6574         if (r)
6575                 return r;
6576
6577         if (rdev->has_uvd) {
6578                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6579                 if (ring->ring_size) {
6580                         r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
6581                                              RADEON_CP_PACKET2);
6582                         if (!r)
6583                                 r = uvd_v1_0_init(rdev);
6584                         if (r)
6585                                 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6586                 }
6587         }
6588
6589         r = radeon_ib_pool_init(rdev);
6590         if (r) {
6591                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
6592                 return r;
6593         }
6594
6595         r = radeon_vm_manager_init(rdev);
6596         if (r) {
6597                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
6598                 return r;
6599         }
6600
6601         r = dce6_audio_init(rdev);
6602         if (r)
6603                 return r;
6604
6605         return 0;
6606 }
6607
6608 int si_resume(struct radeon_device *rdev)
6609 {
6610         int r;
6611
6612         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6613          * posting will perform necessary task to bring back GPU into good
6614          * shape.
6615          */
6616         /* post card */
6617         atom_asic_init(rdev->mode_info.atom_context);
6618
6619         /* init golden registers */
6620         si_init_golden_registers(rdev);
6621
6622         if (rdev->pm.pm_method == PM_METHOD_DPM)
6623                 radeon_pm_resume(rdev);
6624
6625         rdev->accel_working = true;
6626         r = si_startup(rdev);
6627         if (r) {
6628                 DRM_ERROR("si startup failed on resume\n");
6629                 rdev->accel_working = false;
6630                 return r;
6631         }
6632
6633         return r;
6634
6635 }
6636
6637 int si_suspend(struct radeon_device *rdev)
6638 {
6639         radeon_pm_suspend(rdev);
6640         dce6_audio_fini(rdev);
6641         radeon_vm_manager_fini(rdev);
6642         si_cp_enable(rdev, false);
6643         cayman_dma_stop(rdev);
6644         if (rdev->has_uvd) {
6645                 uvd_v1_0_fini(rdev);
6646                 radeon_uvd_suspend(rdev);
6647         }
6648         si_fini_pg(rdev);
6649         si_fini_cg(rdev);
6650         si_irq_suspend(rdev);
6651         radeon_wb_disable(rdev);
6652         si_pcie_gart_disable(rdev);
6653         return 0;
6654 }
6655
6656 /* Plan is to move initialization in that function and use
6657  * helper function so that radeon_device_init pretty much
6658  * do nothing more than calling asic specific function. This
6659  * should also allow to remove a bunch of callback function
6660  * like vram_info.
6661  */
6662 int si_init(struct radeon_device *rdev)
6663 {
6664         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6665         int r;
6666
6667         /* Read BIOS */
6668         if (!radeon_get_bios(rdev)) {
6669                 if (ASIC_IS_AVIVO(rdev))
6670                         return -EINVAL;
6671         }
6672         /* Must be an ATOMBIOS */
6673         if (!rdev->is_atom_bios) {
6674                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6675                 return -EINVAL;
6676         }
6677         r = radeon_atombios_init(rdev);
6678         if (r)
6679                 return r;
6680
6681         /* Post card if necessary */
6682         if (!radeon_card_posted(rdev)) {
6683                 if (!rdev->bios) {
6684                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6685                         return -EINVAL;
6686                 }
6687                 DRM_INFO("GPU not posted. posting now...\n");
6688                 atom_asic_init(rdev->mode_info.atom_context);
6689         }
6690         /* init golden registers */
6691         si_init_golden_registers(rdev);
6692         /* Initialize scratch registers */
6693         si_scratch_init(rdev);
6694         /* Initialize surface registers */
6695         radeon_surface_init(rdev);
6696         /* Initialize clocks */
6697         radeon_get_clock_info(rdev->ddev);
6698
6699         /* Fence driver */
6700         r = radeon_fence_driver_init(rdev);
6701         if (r)
6702                 return r;
6703
6704         /* initialize memory controller */
6705         r = si_mc_init(rdev);
6706         if (r)
6707                 return r;
6708         /* Memory manager */
6709         r = radeon_bo_init(rdev);
6710         if (r)
6711                 return r;
6712
6713         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6714             !rdev->rlc_fw || !rdev->mc_fw) {
6715                 r = si_init_microcode(rdev);
6716                 if (r) {
6717                         DRM_ERROR("Failed to load firmware!\n");
6718                         return r;
6719                 }
6720         }
6721
6722         /* Initialize power management */
6723         radeon_pm_init(rdev);
6724
6725         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6726         ring->ring_obj = NULL;
6727         r600_ring_init(rdev, ring, 1024 * 1024);
6728
6729         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6730         ring->ring_obj = NULL;
6731         r600_ring_init(rdev, ring, 1024 * 1024);
6732
6733         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6734         ring->ring_obj = NULL;
6735         r600_ring_init(rdev, ring, 1024 * 1024);
6736
6737         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6738         ring->ring_obj = NULL;
6739         r600_ring_init(rdev, ring, 64 * 1024);
6740
6741         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6742         ring->ring_obj = NULL;
6743         r600_ring_init(rdev, ring, 64 * 1024);
6744
6745         if (rdev->has_uvd) {
6746                 r = radeon_uvd_init(rdev);
6747                 if (!r) {
6748                         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6749                         ring->ring_obj = NULL;
6750                         r600_ring_init(rdev, ring, 4096);
6751                 }
6752         }
6753
6754         rdev->ih.ring_obj = NULL;
6755         r600_ih_ring_init(rdev, 64 * 1024);
6756
6757         r = r600_pcie_gart_init(rdev);
6758         if (r)
6759                 return r;
6760
6761         rdev->accel_working = true;
6762         r = si_startup(rdev);
6763         if (r) {
6764                 dev_err(rdev->dev, "disabling GPU acceleration\n");
6765                 si_cp_fini(rdev);
6766                 cayman_dma_fini(rdev);
6767                 si_irq_fini(rdev);
6768                 sumo_rlc_fini(rdev);
6769                 radeon_wb_fini(rdev);
6770                 radeon_ib_pool_fini(rdev);
6771                 radeon_vm_manager_fini(rdev);
6772                 radeon_irq_kms_fini(rdev);
6773                 si_pcie_gart_fini(rdev);
6774                 rdev->accel_working = false;
6775         }
6776
6777         /* Don't start up if the MC ucode is missing.
6778          * The default clocks and voltages before the MC ucode
6779          * is loaded are not suffient for advanced operations.
6780          */
6781         if (!rdev->mc_fw) {
6782                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6783                 return -EINVAL;
6784         }
6785
6786         return 0;
6787 }
6788
6789 void si_fini(struct radeon_device *rdev)
6790 {
6791         radeon_pm_fini(rdev);
6792         si_cp_fini(rdev);
6793         cayman_dma_fini(rdev);
6794         si_fini_pg(rdev);
6795         si_fini_cg(rdev);
6796         si_irq_fini(rdev);
6797         sumo_rlc_fini(rdev);
6798         radeon_wb_fini(rdev);
6799         radeon_vm_manager_fini(rdev);
6800         radeon_ib_pool_fini(rdev);
6801         radeon_irq_kms_fini(rdev);
6802         if (rdev->has_uvd) {
6803                 uvd_v1_0_fini(rdev);
6804                 radeon_uvd_fini(rdev);
6805         }
6806         si_pcie_gart_fini(rdev);
6807         r600_vram_scratch_fini(rdev);
6808         radeon_gem_fini(rdev);
6809         radeon_fence_driver_fini(rdev);
6810         radeon_bo_fini(rdev);
6811         radeon_atombios_fini(rdev);
6812         kfree(rdev->bios);
6813         rdev->bios = NULL;
6814 }
6815
6816 /**
6817  * si_get_gpu_clock_counter - return GPU clock counter snapshot
6818  *
6819  * @rdev: radeon_device pointer
6820  *
6821  * Fetches a GPU clock counter snapshot (SI).
6822  * Returns the 64 bit clock counter snapshot.
6823  */
6824 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6825 {
6826         uint64_t clock;
6827
6828         mutex_lock(&rdev->gpu_clock_mutex);
6829         WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6830         clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6831                 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6832         mutex_unlock(&rdev->gpu_clock_mutex);
6833         return clock;
6834 }
6835
6836 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6837 {
6838         unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
6839         int r;
6840
6841         /* bypass vclk and dclk with bclk */
6842         WREG32_P(CG_UPLL_FUNC_CNTL_2,
6843                 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6844                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6845
6846         /* put PLL in bypass mode */
6847         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
6848
6849         if (!vclk || !dclk) {
6850                 /* keep the Bypass mode, put PLL to sleep */
6851                 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6852                 return 0;
6853         }
6854
6855         r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
6856                                           16384, 0x03FFFFFF, 0, 128, 5,
6857                                           &fb_div, &vclk_div, &dclk_div);
6858         if (r)
6859                 return r;
6860
6861         /* set RESET_ANTI_MUX to 0 */
6862         WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
6863
6864         /* set VCO_MODE to 1 */
6865         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
6866
6867         /* toggle UPLL_SLEEP to 1 then back to 0 */
6868         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6869         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
6870
6871         /* deassert UPLL_RESET */
6872         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6873
6874         mdelay(1);
6875
6876         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
6877         if (r)
6878                 return r;
6879
6880         /* assert UPLL_RESET again */
6881         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
6882
6883         /* disable spread spectrum. */
6884         WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
6885
6886         /* set feedback divider */
6887         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
6888
6889         /* set ref divider to 0 */
6890         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
6891
6892         if (fb_div < 307200)
6893                 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
6894         else
6895                 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
6896
6897         /* set PDIV_A and PDIV_B */
6898         WREG32_P(CG_UPLL_FUNC_CNTL_2,
6899                 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
6900                 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
6901
6902         /* give the PLL some time to settle */
6903         mdelay(15);
6904
6905         /* deassert PLL_RESET */
6906         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6907
6908         mdelay(15);
6909
6910         /* switch from bypass mode to normal mode */
6911         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
6912
6913         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
6914         if (r)
6915                 return r;
6916
6917         /* switch VCLK and DCLK selection */
6918         WREG32_P(CG_UPLL_FUNC_CNTL_2,
6919                 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
6920                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6921
6922         mdelay(100);
6923
6924         return 0;
6925 }
6926
6927 static void si_pcie_gen3_enable(struct radeon_device *rdev)
6928 {
6929         struct pci_dev *root = rdev->pdev->bus->self;
6930         int bridge_pos, gpu_pos;
6931         u32 speed_cntl, mask, current_data_rate;
6932         int ret, i;
6933         u16 tmp16;
6934
6935         if (radeon_pcie_gen2 == 0)
6936                 return;
6937
6938         if (rdev->flags & RADEON_IS_IGP)
6939                 return;
6940
6941         if (!(rdev->flags & RADEON_IS_PCIE))
6942                 return;
6943
6944         ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
6945         if (ret != 0)
6946                 return;
6947
6948         if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
6949                 return;
6950
6951         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6952         current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
6953                 LC_CURRENT_DATA_RATE_SHIFT;
6954         if (mask & DRM_PCIE_SPEED_80) {
6955                 if (current_data_rate == 2) {
6956                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
6957                         return;
6958                 }
6959                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
6960         } else if (mask & DRM_PCIE_SPEED_50) {
6961                 if (current_data_rate == 1) {
6962                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
6963                         return;
6964                 }
6965                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
6966         }
6967
6968         bridge_pos = pci_pcie_cap(root);
6969         if (!bridge_pos)
6970                 return;
6971
6972         gpu_pos = pci_pcie_cap(rdev->pdev);
6973         if (!gpu_pos)
6974                 return;
6975
6976         if (mask & DRM_PCIE_SPEED_80) {
6977                 /* re-try equalization if gen3 is not already enabled */
6978                 if (current_data_rate != 2) {
6979                         u16 bridge_cfg, gpu_cfg;
6980                         u16 bridge_cfg2, gpu_cfg2;
6981                         u32 max_lw, current_lw, tmp;
6982
6983                         pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6984                         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6985
6986                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
6987                         pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6988
6989                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
6990                         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6991
6992                         tmp = RREG32_PCIE(PCIE_LC_STATUS1);
6993                         max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
6994                         current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
6995
6996                         if (current_lw < max_lw) {
6997                                 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6998                                 if (tmp & LC_RENEGOTIATION_SUPPORT) {
6999                                         tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
7000                                         tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
7001                                         tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
7002                                         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
7003                                 }
7004                         }
7005
7006                         for (i = 0; i < 10; i++) {
7007                                 /* check status */
7008                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
7009                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
7010                                         break;
7011
7012                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7013                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7014
7015                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
7016                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
7017
7018                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7019                                 tmp |= LC_SET_QUIESCE;
7020                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7021
7022                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7023                                 tmp |= LC_REDO_EQ;
7024                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7025
7026                                 mdelay(100);
7027
7028                                 /* linkctl */
7029                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
7030                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7031                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
7032                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7033
7034                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
7035                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7036                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
7037                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7038
7039                                 /* linkctl2 */
7040                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
7041                                 tmp16 &= ~((1 << 4) | (7 << 9));
7042                                 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
7043                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
7044
7045                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7046                                 tmp16 &= ~((1 << 4) | (7 << 9));
7047                                 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
7048                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7049
7050                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7051                                 tmp &= ~LC_SET_QUIESCE;
7052                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7053                         }
7054                 }
7055         }
7056
7057         /* set the link speed */
7058         speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
7059         speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
7060         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7061
7062         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7063         tmp16 &= ~0xf;
7064         if (mask & DRM_PCIE_SPEED_80)
7065                 tmp16 |= 3; /* gen3 */
7066         else if (mask & DRM_PCIE_SPEED_50)
7067                 tmp16 |= 2; /* gen2 */
7068         else
7069                 tmp16 |= 1; /* gen1 */
7070         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7071
7072         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7073         speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7074         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7075
7076         for (i = 0; i < rdev->usec_timeout; i++) {
7077                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7078                 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7079                         break;
7080                 udelay(1);
7081         }
7082 }
7083
7084 static void si_program_aspm(struct radeon_device *rdev)
7085 {
7086         u32 data, orig;
7087         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7088         bool disable_clkreq = false;
7089
7090         if (radeon_aspm == 0)
7091                 return;
7092
7093         if (!(rdev->flags & RADEON_IS_PCIE))
7094                 return;
7095
7096         orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7097         data &= ~LC_XMIT_N_FTS_MASK;
7098         data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7099         if (orig != data)
7100                 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7101
7102         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7103         data |= LC_GO_TO_RECOVERY;
7104         if (orig != data)
7105                 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7106
7107         orig = data = RREG32_PCIE(PCIE_P_CNTL);
7108         data |= P_IGNORE_EDB_ERR;
7109         if (orig != data)
7110                 WREG32_PCIE(PCIE_P_CNTL, data);
7111
7112         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7113         data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7114         data |= LC_PMI_TO_L1_DIS;
7115         if (!disable_l0s)
7116                 data |= LC_L0S_INACTIVITY(7);
7117
7118         if (!disable_l1) {
7119                 data |= LC_L1_INACTIVITY(7);
7120                 data &= ~LC_PMI_TO_L1_DIS;
7121                 if (orig != data)
7122                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7123
7124                 if (!disable_plloff_in_l1) {
7125                         bool clk_req_support;
7126
7127                         orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7128                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7129                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7130                         if (orig != data)
7131                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7132
7133                         orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7134                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7135                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7136                         if (orig != data)
7137                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7138
7139                         orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7140                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7141                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7142                         if (orig != data)
7143                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7144
7145                         orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7146                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7147                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7148                         if (orig != data)
7149                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7150
7151                         if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7152                                 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7153                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7154                                 if (orig != data)
7155                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7156
7157                                 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7158                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7159                                 if (orig != data)
7160                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7161
7162                                 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7163                                 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7164                                 if (orig != data)
7165                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7166
7167                                 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7168                                 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7169                                 if (orig != data)
7170                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7171
7172                                 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7173                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7174                                 if (orig != data)
7175                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7176
7177                                 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7178                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7179                                 if (orig != data)
7180                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7181
7182                                 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7183                                 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7184                                 if (orig != data)
7185                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7186
7187                                 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7188                                 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7189                                 if (orig != data)
7190                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7191                         }
7192                         orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7193                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7194                         data |= LC_DYN_LANES_PWR_STATE(3);
7195                         if (orig != data)
7196                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7197
7198                         orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7199                         data &= ~LS2_EXIT_TIME_MASK;
7200                         if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7201                                 data |= LS2_EXIT_TIME(5);
7202                         if (orig != data)
7203                                 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7204
7205                         orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7206                         data &= ~LS2_EXIT_TIME_MASK;
7207                         if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7208                                 data |= LS2_EXIT_TIME(5);
7209                         if (orig != data)
7210                                 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7211
7212                         if (!disable_clkreq) {
7213                                 struct pci_dev *root = rdev->pdev->bus->self;
7214                                 u32 lnkcap;
7215
7216                                 clk_req_support = false;
7217                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7218                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7219                                         clk_req_support = true;
7220                         } else {
7221                                 clk_req_support = false;
7222                         }
7223
7224                         if (clk_req_support) {
7225                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7226                                 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7227                                 if (orig != data)
7228                                         WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7229
7230                                 orig = data = RREG32(THM_CLK_CNTL);
7231                                 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7232                                 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7233                                 if (orig != data)
7234                                         WREG32(THM_CLK_CNTL, data);
7235
7236                                 orig = data = RREG32(MISC_CLK_CNTL);
7237                                 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7238                                 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7239                                 if (orig != data)
7240                                         WREG32(MISC_CLK_CNTL, data);
7241
7242                                 orig = data = RREG32(CG_CLKPIN_CNTL);
7243                                 data &= ~BCLK_AS_XCLK;
7244                                 if (orig != data)
7245                                         WREG32(CG_CLKPIN_CNTL, data);
7246
7247                                 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7248                                 data &= ~FORCE_BIF_REFCLK_EN;
7249                                 if (orig != data)
7250                                         WREG32(CG_CLKPIN_CNTL_2, data);
7251
7252                                 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7253                                 data &= ~MPLL_CLKOUT_SEL_MASK;
7254                                 data |= MPLL_CLKOUT_SEL(4);
7255                                 if (orig != data)
7256                                         WREG32(MPLL_BYPASSCLK_SEL, data);
7257
7258                                 orig = data = RREG32(SPLL_CNTL_MODE);
7259                                 data &= ~SPLL_REFCLK_SEL_MASK;
7260                                 if (orig != data)
7261                                         WREG32(SPLL_CNTL_MODE, data);
7262                         }
7263                 }
7264         } else {
7265                 if (orig != data)
7266                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7267         }
7268
7269         orig = data = RREG32_PCIE(PCIE_CNTL2);
7270         data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7271         if (orig != data)
7272                 WREG32_PCIE(PCIE_CNTL2, data);
7273
7274         if (!disable_l0s) {
7275                 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7276                 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7277                         data = RREG32_PCIE(PCIE_LC_STATUS1);
7278                         if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7279                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7280                                 data &= ~LC_L0S_INACTIVITY_MASK;
7281                                 if (orig != data)
7282                                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7283                         }
7284                 }
7285         }
7286 }