drivers/gpu/drm/radeon/si.c

   1 /*
   2  * Copyright 2011 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <drm/drmP.h>
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include <drm/radeon_drm.h>
  31 #include "sid.h"
  32 #include "atom.h"
  33 #include "si_blit_shaders.h"
  34 #include "clearstate_si.h"
  35 #include "radeon_ucode.h"
  36
  37
  38 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
  39 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
  40 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
  41 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
  42 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
  43 MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
  44 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
  45 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
  46 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
  47 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
  48 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
  49 MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
  50 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
  51 MODULE_FIRMWARE("radeon/VERDE_me.bin");
  52 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
  53 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
  54 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
  55 MODULE_FIRMWARE("radeon/VERDE_smc.bin");
  56 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
  57 MODULE_FIRMWARE("radeon/OLAND_me.bin");
  58 MODULE_FIRMWARE("radeon/OLAND_ce.bin");
  59 MODULE_FIRMWARE("radeon/OLAND_mc.bin");
  60 MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
  61 MODULE_FIRMWARE("radeon/OLAND_smc.bin");
  62 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
  63 MODULE_FIRMWARE("radeon/HAINAN_me.bin");
  64 MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
  65 MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
  66 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
  67 MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
  68
  69 static void si_pcie_gen3_enable(struct radeon_device *rdev);
  70 static void si_program_aspm(struct radeon_device *rdev);
  71 extern void sumo_rlc_fini(struct radeon_device *rdev);
  72 extern int sumo_rlc_init(struct radeon_device *rdev);
  73 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
  74 extern void r600_ih_ring_fini(struct radeon_device *rdev);
  75 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
  76 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
  77 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
  78 extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
  79 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
  80 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
  81 extern void si_dma_vm_set_page(struct radeon_device *rdev,
  82                                struct radeon_ib *ib,
  83                                uint64_t pe,
  84                                uint64_t addr, unsigned count,
  85                                uint32_t incr, uint32_t flags);
  86 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
  87                                          bool enable);
  88 static void si_fini_pg(struct radeon_device *rdev);
  89 static void si_fini_cg(struct radeon_device *rdev);
  90 static void si_rlc_stop(struct radeon_device *rdev);
  91
  92 static const u32 verde_rlc_save_restore_register_list[] =
  93 {
  94         (0x8000 << 16) | (0x98f4 >> 2),
  95         0x00000000,
  96         (0x8040 << 16) | (0x98f4 >> 2),
  97         0x00000000,
  98         (0x8000 << 16) | (0xe80 >> 2),
  99         0x00000000,
 100         (0x8040 << 16) | (0xe80 >> 2),
 101         0x00000000,
 102         (0x8000 << 16) | (0x89bc >> 2),
 103         0x00000000,
 104         (0x8040 << 16) | (0x89bc >> 2),
 105         0x00000000,
 106         (0x8000 << 16) | (0x8c1c >> 2),
 107         0x00000000,
 108         (0x8040 << 16) | (0x8c1c >> 2),
 109         0x00000000,
 110         (0x9c00 << 16) | (0x98f0 >> 2),
 111         0x00000000,
 112         (0x9c00 << 16) | (0xe7c >> 2),
 113         0x00000000,
 114         (0x8000 << 16) | (0x9148 >> 2),
 115         0x00000000,
 116         (0x8040 << 16) | (0x9148 >> 2),
 117         0x00000000,
 118         (0x9c00 << 16) | (0x9150 >> 2),
 119         0x00000000,
 120         (0x9c00 << 16) | (0x897c >> 2),
 121         0x00000000,
 122         (0x9c00 << 16) | (0x8d8c >> 2),
 123         0x00000000,
 124         (0x9c00 << 16) | (0xac54 >> 2),
 125         0X00000000,
 126         0x3,
 127         (0x9c00 << 16) | (0x98f8 >> 2),
 128         0x00000000,
 129         (0x9c00 << 16) | (0x9910 >> 2),
 130         0x00000000,
 131         (0x9c00 << 16) | (0x9914 >> 2),
 132         0x00000000,
 133         (0x9c00 << 16) | (0x9918 >> 2),
 134         0x00000000,
 135         (0x9c00 << 16) | (0x991c >> 2),
 136         0x00000000,
 137         (0x9c00 << 16) | (0x9920 >> 2),
 138         0x00000000,
 139         (0x9c00 << 16) | (0x9924 >> 2),
 140         0x00000000,
 141         (0x9c00 << 16) | (0x9928 >> 2),
 142         0x00000000,
 143         (0x9c00 << 16) | (0x992c >> 2),
 144         0x00000000,
 145         (0x9c00 << 16) | (0x9930 >> 2),
 146         0x00000000,
 147         (0x9c00 << 16) | (0x9934 >> 2),
 148         0x00000000,
 149         (0x9c00 << 16) | (0x9938 >> 2),
 150         0x00000000,
 151         (0x9c00 << 16) | (0x993c >> 2),
 152         0x00000000,
 153         (0x9c00 << 16) | (0x9940 >> 2),
 154         0x00000000,
 155         (0x9c00 << 16) | (0x9944 >> 2),
 156         0x00000000,
 157         (0x9c00 << 16) | (0x9948 >> 2),
 158         0x00000000,
 159         (0x9c00 << 16) | (0x994c >> 2),
 160         0x00000000,
 161         (0x9c00 << 16) | (0x9950 >> 2),
 162         0x00000000,
 163         (0x9c00 << 16) | (0x9954 >> 2),
 164         0x00000000,
 165         (0x9c00 << 16) | (0x9958 >> 2),
 166         0x00000000,
 167         (0x9c00 << 16) | (0x995c >> 2),
 168         0x00000000,
 169         (0x9c00 << 16) | (0x9960 >> 2),
 170         0x00000000,
 171         (0x9c00 << 16) | (0x9964 >> 2),
 172         0x00000000,
 173         (0x9c00 << 16) | (0x9968 >> 2),
 174         0x00000000,
 175         (0x9c00 << 16) | (0x996c >> 2),
 176         0x00000000,
 177         (0x9c00 << 16) | (0x9970 >> 2),
 178         0x00000000,
 179         (0x9c00 << 16) | (0x9974 >> 2),
 180         0x00000000,
 181         (0x9c00 << 16) | (0x9978 >> 2),
 182         0x00000000,
 183         (0x9c00 << 16) | (0x997c >> 2),
 184         0x00000000,
 185         (0x9c00 << 16) | (0x9980 >> 2),
 186         0x00000000,
 187         (0x9c00 << 16) | (0x9984 >> 2),
 188         0x00000000,
 189         (0x9c00 << 16) | (0x9988 >> 2),
 190         0x00000000,
 191         (0x9c00 << 16) | (0x998c >> 2),
 192         0x00000000,
 193         (0x9c00 << 16) | (0x8c00 >> 2),
 194         0x00000000,
 195         (0x9c00 << 16) | (0x8c14 >> 2),
 196         0x00000000,
 197         (0x9c00 << 16) | (0x8c04 >> 2),
 198         0x00000000,
 199         (0x9c00 << 16) | (0x8c08 >> 2),
 200         0x00000000,
 201         (0x8000 << 16) | (0x9b7c >> 2),
 202         0x00000000,
 203         (0x8040 << 16) | (0x9b7c >> 2),
 204         0x00000000,
 205         (0x8000 << 16) | (0xe84 >> 2),
 206         0x00000000,
 207         (0x8040 << 16) | (0xe84 >> 2),
 208         0x00000000,
 209         (0x8000 << 16) | (0x89c0 >> 2),
 210         0x00000000,
 211         (0x8040 << 16) | (0x89c0 >> 2),
 212         0x00000000,
 213         (0x8000 << 16) | (0x914c >> 2),
 214         0x00000000,
 215         (0x8040 << 16) | (0x914c >> 2),
 216         0x00000000,
 217         (0x8000 << 16) | (0x8c20 >> 2),
 218         0x00000000,
 219         (0x8040 << 16) | (0x8c20 >> 2),
 220         0x00000000,
 221         (0x8000 << 16) | (0x9354 >> 2),
 222         0x00000000,
 223         (0x8040 << 16) | (0x9354 >> 2),
 224         0x00000000,
 225         (0x9c00 << 16) | (0x9060 >> 2),
 226         0x00000000,
 227         (0x9c00 << 16) | (0x9364 >> 2),
 228         0x00000000,
 229         (0x9c00 << 16) | (0x9100 >> 2),
 230         0x00000000,
 231         (0x9c00 << 16) | (0x913c >> 2),
 232         0x00000000,
 233         (0x8000 << 16) | (0x90e0 >> 2),
 234         0x00000000,
 235         (0x8000 << 16) | (0x90e4 >> 2),
 236         0x00000000,
 237         (0x8000 << 16) | (0x90e8 >> 2),
 238         0x00000000,
 239         (0x8040 << 16) | (0x90e0 >> 2),
 240         0x00000000,
 241         (0x8040 << 16) | (0x90e4 >> 2),
 242         0x00000000,
 243         (0x8040 << 16) | (0x90e8 >> 2),
 244         0x00000000,
 245         (0x9c00 << 16) | (0x8bcc >> 2),
 246         0x00000000,
 247         (0x9c00 << 16) | (0x8b24 >> 2),
 248         0x00000000,
 249         (0x9c00 << 16) | (0x88c4 >> 2),
 250         0x00000000,
 251         (0x9c00 << 16) | (0x8e50 >> 2),
 252         0x00000000,
 253         (0x9c00 << 16) | (0x8c0c >> 2),
 254         0x00000000,
 255         (0x9c00 << 16) | (0x8e58 >> 2),
 256         0x00000000,
 257         (0x9c00 << 16) | (0x8e5c >> 2),
 258         0x00000000,
 259         (0x9c00 << 16) | (0x9508 >> 2),
 260         0x00000000,
 261         (0x9c00 << 16) | (0x950c >> 2),
 262         0x00000000,
 263         (0x9c00 << 16) | (0x9494 >> 2),
 264         0x00000000,
 265         (0x9c00 << 16) | (0xac0c >> 2),
 266         0x00000000,
 267         (0x9c00 << 16) | (0xac10 >> 2),
 268         0x00000000,
 269         (0x9c00 << 16) | (0xac14 >> 2),
 270         0x00000000,
 271         (0x9c00 << 16) | (0xae00 >> 2),
 272         0x00000000,
 273         (0x9c00 << 16) | (0xac08 >> 2),
 274         0x00000000,
 275         (0x9c00 << 16) | (0x88d4 >> 2),
 276         0x00000000,
 277         (0x9c00 << 16) | (0x88c8 >> 2),
 278         0x00000000,
 279         (0x9c00 << 16) | (0x88cc >> 2),
 280         0x00000000,
 281         (0x9c00 << 16) | (0x89b0 >> 2),
 282         0x00000000,
 283         (0x9c00 << 16) | (0x8b10 >> 2),
 284         0x00000000,
 285         (0x9c00 << 16) | (0x8a14 >> 2),
 286         0x00000000,
 287         (0x9c00 << 16) | (0x9830 >> 2),
 288         0x00000000,
 289         (0x9c00 << 16) | (0x9834 >> 2),
 290         0x00000000,
 291         (0x9c00 << 16) | (0x9838 >> 2),
 292         0x00000000,
 293         (0x9c00 << 16) | (0x9a10 >> 2),
 294         0x00000000,
 295         (0x8000 << 16) | (0x9870 >> 2),
 296         0x00000000,
 297         (0x8000 << 16) | (0x9874 >> 2),
 298         0x00000000,
 299         (0x8001 << 16) | (0x9870 >> 2),
 300         0x00000000,
 301         (0x8001 << 16) | (0x9874 >> 2),
 302         0x00000000,
 303         (0x8040 << 16) | (0x9870 >> 2),
 304         0x00000000,
 305         (0x8040 << 16) | (0x9874 >> 2),
 306         0x00000000,
 307         (0x8041 << 16) | (0x9870 >> 2),
 308         0x00000000,
 309         (0x8041 << 16) | (0x9874 >> 2),
 310         0x00000000,
 311         0x00000000
 312 };
 313
 314 static const u32 tahiti_golden_rlc_registers[] =
 315 {
 316         0xc424, 0xffffffff, 0x00601005,
 317         0xc47c, 0xffffffff, 0x10104040,
 318         0xc488, 0xffffffff, 0x0100000a,
 319         0xc314, 0xffffffff, 0x00000800,
 320         0xc30c, 0xffffffff, 0x800000f4,
 321         0xf4a8, 0xffffffff, 0x00000000
 322 };
 323
 324 static const u32 tahiti_golden_registers[] =
 325 {
 326         0x9a10, 0x00010000, 0x00018208,
 327         0x9830, 0xffffffff, 0x00000000,
 328         0x9834, 0xf00fffff, 0x00000400,
 329         0x9838, 0x0002021c, 0x00020200,
 330         0xc78, 0x00000080, 0x00000000,
 331         0xd030, 0x000300c0, 0x00800040,
 332         0xd830, 0x000300c0, 0x00800040,
 333         0x5bb0, 0x000000f0, 0x00000070,
 334         0x5bc0, 0x00200000, 0x50100000,
 335         0x7030, 0x31000311, 0x00000011,
 336         0x277c, 0x00000003, 0x000007ff,
 337         0x240c, 0x000007ff, 0x00000000,
 338         0x8a14, 0xf000001f, 0x00000007,
 339         0x8b24, 0xffffffff, 0x00ffffff,
 340         0x8b10, 0x0000ff0f, 0x00000000,
 341         0x28a4c, 0x07ffffff, 0x4e000000,
 342         0x28350, 0x3f3f3fff, 0x2a00126a,
 343         0x30, 0x000000ff, 0x0040,
 344         0x34, 0x00000040, 0x00004040,
 345         0x9100, 0x07ffffff, 0x03000000,
 346         0x8e88, 0x01ff1f3f, 0x00000000,
 347         0x8e84, 0x01ff1f3f, 0x00000000,
 348         0x9060, 0x0000007f, 0x00000020,
 349         0x9508, 0x00010000, 0x00010000,
 350         0xac14, 0x00000200, 0x000002fb,
 351         0xac10, 0xffffffff, 0x0000543b,
 352         0xac0c, 0xffffffff, 0xa9210876,
 353         0x88d0, 0xffffffff, 0x000fff40,
 354         0x88d4, 0x0000001f, 0x00000010,
 355         0x1410, 0x20000000, 0x20fffed8,
 356         0x15c0, 0x000c0fc0, 0x000c0400
 357 };
 358
 359 static const u32 tahiti_golden_registers2[] =
 360 {
 361         0xc64, 0x00000001, 0x00000001
 362 };
 363
 364 static const u32 pitcairn_golden_rlc_registers[] =
 365 {
 366         0xc424, 0xffffffff, 0x00601004,
 367         0xc47c, 0xffffffff, 0x10102020,
 368         0xc488, 0xffffffff, 0x01000020,
 369         0xc314, 0xffffffff, 0x00000800,
 370         0xc30c, 0xffffffff, 0x800000a4
 371 };
 372
 373 static const u32 pitcairn_golden_registers[] =
 374 {
 375         0x9a10, 0x00010000, 0x00018208,
 376         0x9830, 0xffffffff, 0x00000000,
 377         0x9834, 0xf00fffff, 0x00000400,
 378         0x9838, 0x0002021c, 0x00020200,
 379         0xc78, 0x00000080, 0x00000000,
 380         0xd030, 0x000300c0, 0x00800040,
 381         0xd830, 0x000300c0, 0x00800040,
 382         0x5bb0, 0x000000f0, 0x00000070,
 383         0x5bc0, 0x00200000, 0x50100000,
 384         0x7030, 0x31000311, 0x00000011,
 385         0x2ae4, 0x00073ffe, 0x000022a2,
 386         0x240c, 0x000007ff, 0x00000000,
 387         0x8a14, 0xf000001f, 0x00000007,
 388         0x8b24, 0xffffffff, 0x00ffffff,
 389         0x8b10, 0x0000ff0f, 0x00000000,
 390         0x28a4c, 0x07ffffff, 0x4e000000,
 391         0x28350, 0x3f3f3fff, 0x2a00126a,
 392         0x30, 0x000000ff, 0x0040,
 393         0x34, 0x00000040, 0x00004040,
 394         0x9100, 0x07ffffff, 0x03000000,
 395         0x9060, 0x0000007f, 0x00000020,
 396         0x9508, 0x00010000, 0x00010000,
 397         0xac14, 0x000003ff, 0x000000f7,
 398         0xac10, 0xffffffff, 0x00000000,
 399         0xac0c, 0xffffffff, 0x32761054,
 400         0x88d4, 0x0000001f, 0x00000010,
 401         0x15c0, 0x000c0fc0, 0x000c0400
 402 };
 403
 404 static const u32 verde_golden_rlc_registers[] =
 405 {
 406         0xc424, 0xffffffff, 0x033f1005,
 407         0xc47c, 0xffffffff, 0x10808020,
 408         0xc488, 0xffffffff, 0x00800008,
 409         0xc314, 0xffffffff, 0x00001000,
 410         0xc30c, 0xffffffff, 0x80010014
 411 };
 412
 413 static const u32 verde_golden_registers[] =
 414 {
 415         0x9a10, 0x00010000, 0x00018208,
 416         0x9830, 0xffffffff, 0x00000000,
 417         0x9834, 0xf00fffff, 0x00000400,
 418         0x9838, 0x0002021c, 0x00020200,
 419         0xc78, 0x00000080, 0x00000000,
 420         0xd030, 0x000300c0, 0x00800040,
 421         0xd030, 0x000300c0, 0x00800040,
 422         0xd830, 0x000300c0, 0x00800040,
 423         0xd830, 0x000300c0, 0x00800040,
 424         0x5bb0, 0x000000f0, 0x00000070,
 425         0x5bc0, 0x00200000, 0x50100000,
 426         0x7030, 0x31000311, 0x00000011,
 427         0x2ae4, 0x00073ffe, 0x000022a2,
 428         0x2ae4, 0x00073ffe, 0x000022a2,
 429         0x2ae4, 0x00073ffe, 0x000022a2,
 430         0x240c, 0x000007ff, 0x00000000,
 431         0x240c, 0x000007ff, 0x00000000,
 432         0x240c, 0x000007ff, 0x00000000,
 433         0x8a14, 0xf000001f, 0x00000007,
 434         0x8a14, 0xf000001f, 0x00000007,
 435         0x8a14, 0xf000001f, 0x00000007,
 436         0x8b24, 0xffffffff, 0x00ffffff,
 437         0x8b10, 0x0000ff0f, 0x00000000,
 438         0x28a4c, 0x07ffffff, 0x4e000000,
 439         0x28350, 0x3f3f3fff, 0x0000124a,
 440         0x28350, 0x3f3f3fff, 0x0000124a,
 441         0x28350, 0x3f3f3fff, 0x0000124a,
 442         0x30, 0x000000ff, 0x0040,
 443         0x34, 0x00000040, 0x00004040,
 444         0x9100, 0x07ffffff, 0x03000000,
 445         0x9100, 0x07ffffff, 0x03000000,
 446         0x8e88, 0x01ff1f3f, 0x00000000,
 447         0x8e88, 0x01ff1f3f, 0x00000000,
 448         0x8e88, 0x01ff1f3f, 0x00000000,
 449         0x8e84, 0x01ff1f3f, 0x00000000,
 450         0x8e84, 0x01ff1f3f, 0x00000000,
 451         0x8e84, 0x01ff1f3f, 0x00000000,
 452         0x9060, 0x0000007f, 0x00000020,
 453         0x9508, 0x00010000, 0x00010000,
 454         0xac14, 0x000003ff, 0x00000003,
 455         0xac14, 0x000003ff, 0x00000003,
 456         0xac14, 0x000003ff, 0x00000003,
 457         0xac10, 0xffffffff, 0x00000000,
 458         0xac10, 0xffffffff, 0x00000000,
 459         0xac10, 0xffffffff, 0x00000000,
 460         0xac0c, 0xffffffff, 0x00001032,
 461         0xac0c, 0xffffffff, 0x00001032,
 462         0xac0c, 0xffffffff, 0x00001032,
 463         0x88d4, 0x0000001f, 0x00000010,
 464         0x88d4, 0x0000001f, 0x00000010,
 465         0x88d4, 0x0000001f, 0x00000010,
 466         0x15c0, 0x000c0fc0, 0x000c0400
 467 };
 468
 469 static const u32 oland_golden_rlc_registers[] =
 470 {
 471         0xc424, 0xffffffff, 0x00601005,
 472         0xc47c, 0xffffffff, 0x10104040,
 473         0xc488, 0xffffffff, 0x0100000a,
 474         0xc314, 0xffffffff, 0x00000800,
 475         0xc30c, 0xffffffff, 0x800000f4
 476 };
 477
 478 static const u32 oland_golden_registers[] =
 479 {
 480         0x9a10, 0x00010000, 0x00018208,
 481         0x9830, 0xffffffff, 0x00000000,
 482         0x9834, 0xf00fffff, 0x00000400,
 483         0x9838, 0x0002021c, 0x00020200,
 484         0xc78, 0x00000080, 0x00000000,
 485         0xd030, 0x000300c0, 0x00800040,
 486         0xd830, 0x000300c0, 0x00800040,
 487         0x5bb0, 0x000000f0, 0x00000070,
 488         0x5bc0, 0x00200000, 0x50100000,
 489         0x7030, 0x31000311, 0x00000011,
 490         0x2ae4, 0x00073ffe, 0x000022a2,
 491         0x240c, 0x000007ff, 0x00000000,
 492         0x8a14, 0xf000001f, 0x00000007,
 493         0x8b24, 0xffffffff, 0x00ffffff,
 494         0x8b10, 0x0000ff0f, 0x00000000,
 495         0x28a4c, 0x07ffffff, 0x4e000000,
 496         0x28350, 0x3f3f3fff, 0x00000082,
 497         0x30, 0x000000ff, 0x0040,
 498         0x34, 0x00000040, 0x00004040,
 499         0x9100, 0x07ffffff, 0x03000000,
 500         0x9060, 0x0000007f, 0x00000020,
 501         0x9508, 0x00010000, 0x00010000,
 502         0xac14, 0x000003ff, 0x000000f3,
 503         0xac10, 0xffffffff, 0x00000000,
 504         0xac0c, 0xffffffff, 0x00003210,
 505         0x88d4, 0x0000001f, 0x00000010,
 506         0x15c0, 0x000c0fc0, 0x000c0400
 507 };
 508
 509 static const u32 hainan_golden_registers[] =
 510 {
 511         0x9a10, 0x00010000, 0x00018208,
 512         0x9830, 0xffffffff, 0x00000000,
 513         0x9834, 0xf00fffff, 0x00000400,
 514         0x9838, 0x0002021c, 0x00020200,
 515         0xd0c0, 0xff000fff, 0x00000100,
 516         0xd030, 0x000300c0, 0x00800040,
 517         0xd8c0, 0xff000fff, 0x00000100,
 518         0xd830, 0x000300c0, 0x00800040,
 519         0x2ae4, 0x00073ffe, 0x000022a2,
 520         0x240c, 0x000007ff, 0x00000000,
 521         0x8a14, 0xf000001f, 0x00000007,
 522         0x8b24, 0xffffffff, 0x00ffffff,
 523         0x8b10, 0x0000ff0f, 0x00000000,
 524         0x28a4c, 0x07ffffff, 0x4e000000,
 525         0x28350, 0x3f3f3fff, 0x00000000,
 526         0x30, 0x000000ff, 0x0040,
 527         0x34, 0x00000040, 0x00004040,
 528         0x9100, 0x03e00000, 0x03600000,
 529         0x9060, 0x0000007f, 0x00000020,
 530         0x9508, 0x00010000, 0x00010000,
 531         0xac14, 0x000003ff, 0x000000f1,
 532         0xac10, 0xffffffff, 0x00000000,
 533         0xac0c, 0xffffffff, 0x00003210,
 534         0x88d4, 0x0000001f, 0x00000010,
 535         0x15c0, 0x000c0fc0, 0x000c0400
 536 };
 537
 538 static const u32 hainan_golden_registers2[] =
 539 {
 540         0x98f8, 0xffffffff, 0x02010001
 541 };
 542
 543 static const u32 tahiti_mgcg_cgcg_init[] =
 544 {
 545         0xc400, 0xffffffff, 0xfffffffc,
 546         0x802c, 0xffffffff, 0xe0000000,
 547         0x9a60, 0xffffffff, 0x00000100,
 548         0x92a4, 0xffffffff, 0x00000100,
 549         0xc164, 0xffffffff, 0x00000100,
 550         0x9774, 0xffffffff, 0x00000100,
 551         0x8984, 0xffffffff, 0x06000100,
 552         0x8a18, 0xffffffff, 0x00000100,
 553         0x92a0, 0xffffffff, 0x00000100,
 554         0xc380, 0xffffffff, 0x00000100,
 555         0x8b28, 0xffffffff, 0x00000100,
 556         0x9144, 0xffffffff, 0x00000100,
 557         0x8d88, 0xffffffff, 0x00000100,
 558         0x8d8c, 0xffffffff, 0x00000100,
 559         0x9030, 0xffffffff, 0x00000100,
 560         0x9034, 0xffffffff, 0x00000100,
 561         0x9038, 0xffffffff, 0x00000100,
 562         0x903c, 0xffffffff, 0x00000100,
 563         0xad80, 0xffffffff, 0x00000100,
 564         0xac54, 0xffffffff, 0x00000100,
 565         0x897c, 0xffffffff, 0x06000100,
 566         0x9868, 0xffffffff, 0x00000100,
 567         0x9510, 0xffffffff, 0x00000100,
 568         0xaf04, 0xffffffff, 0x00000100,
 569         0xae04, 0xffffffff, 0x00000100,
 570         0x949c, 0xffffffff, 0x00000100,
 571         0x802c, 0xffffffff, 0xe0000000,
 572         0x9160, 0xffffffff, 0x00010000,
 573         0x9164, 0xffffffff, 0x00030002,
 574         0x9168, 0xffffffff, 0x00040007,
 575         0x916c, 0xffffffff, 0x00060005,
 576         0x9170, 0xffffffff, 0x00090008,
 577         0x9174, 0xffffffff, 0x00020001,
 578         0x9178, 0xffffffff, 0x00040003,
 579         0x917c, 0xffffffff, 0x00000007,
 580         0x9180, 0xffffffff, 0x00060005,
 581         0x9184, 0xffffffff, 0x00090008,
 582         0x9188, 0xffffffff, 0x00030002,
 583         0x918c, 0xffffffff, 0x00050004,
 584         0x9190, 0xffffffff, 0x00000008,
 585         0x9194, 0xffffffff, 0x00070006,
 586         0x9198, 0xffffffff, 0x000a0009,
 587         0x919c, 0xffffffff, 0x00040003,
 588         0x91a0, 0xffffffff, 0x00060005,
 589         0x91a4, 0xffffffff, 0x00000009,
 590         0x91a8, 0xffffffff, 0x00080007,
 591         0x91ac, 0xffffffff, 0x000b000a,
 592         0x91b0, 0xffffffff, 0x00050004,
 593         0x91b4, 0xffffffff, 0x00070006,
 594         0x91b8, 0xffffffff, 0x0008000b,
 595         0x91bc, 0xffffffff, 0x000a0009,
 596         0x91c0, 0xffffffff, 0x000d000c,
 597         0x91c4, 0xffffffff, 0x00060005,
 598         0x91c8, 0xffffffff, 0x00080007,
 599         0x91cc, 0xffffffff, 0x0000000b,
 600         0x91d0, 0xffffffff, 0x000a0009,
 601         0x91d4, 0xffffffff, 0x000d000c,
 602         0x91d8, 0xffffffff, 0x00070006,
 603         0x91dc, 0xffffffff, 0x00090008,
 604         0x91e0, 0xffffffff, 0x0000000c,
 605         0x91e4, 0xffffffff, 0x000b000a,
 606         0x91e8, 0xffffffff, 0x000e000d,
 607         0x91ec, 0xffffffff, 0x00080007,
 608         0x91f0, 0xffffffff, 0x000a0009,
 609         0x91f4, 0xffffffff, 0x0000000d,
 610         0x91f8, 0xffffffff, 0x000c000b,
 611         0x91fc, 0xffffffff, 0x000f000e,
 612         0x9200, 0xffffffff, 0x00090008,
 613         0x9204, 0xffffffff, 0x000b000a,
 614         0x9208, 0xffffffff, 0x000c000f,
 615         0x920c, 0xffffffff, 0x000e000d,
 616         0x9210, 0xffffffff, 0x00110010,
 617         0x9214, 0xffffffff, 0x000a0009,
 618         0x9218, 0xffffffff, 0x000c000b,
 619         0x921c, 0xffffffff, 0x0000000f,
 620         0x9220, 0xffffffff, 0x000e000d,
 621         0x9224, 0xffffffff, 0x00110010,
 622         0x9228, 0xffffffff, 0x000b000a,
 623         0x922c, 0xffffffff, 0x000d000c,
 624         0x9230, 0xffffffff, 0x00000010,
 625         0x9234, 0xffffffff, 0x000f000e,
 626         0x9238, 0xffffffff, 0x00120011,
 627         0x923c, 0xffffffff, 0x000c000b,
 628         0x9240, 0xffffffff, 0x000e000d,
 629         0x9244, 0xffffffff, 0x00000011,
 630         0x9248, 0xffffffff, 0x0010000f,
 631         0x924c, 0xffffffff, 0x00130012,
 632         0x9250, 0xffffffff, 0x000d000c,
 633         0x9254, 0xffffffff, 0x000f000e,
 634         0x9258, 0xffffffff, 0x00100013,
 635         0x925c, 0xffffffff, 0x00120011,
 636         0x9260, 0xffffffff, 0x00150014,
 637         0x9264, 0xffffffff, 0x000e000d,
 638         0x9268, 0xffffffff, 0x0010000f,
 639         0x926c, 0xffffffff, 0x00000013,
 640         0x9270, 0xffffffff, 0x00120011,
 641         0x9274, 0xffffffff, 0x00150014,
 642         0x9278, 0xffffffff, 0x000f000e,
 643         0x927c, 0xffffffff, 0x00110010,
 644         0x9280, 0xffffffff, 0x00000014,
 645         0x9284, 0xffffffff, 0x00130012,
 646         0x9288, 0xffffffff, 0x00160015,
 647         0x928c, 0xffffffff, 0x0010000f,
 648         0x9290, 0xffffffff, 0x00120011,
 649         0x9294, 0xffffffff, 0x00000015,
 650         0x9298, 0xffffffff, 0x00140013,
 651         0x929c, 0xffffffff, 0x00170016,
 652         0x9150, 0xffffffff, 0x96940200,
 653         0x8708, 0xffffffff, 0x00900100,
 654         0xc478, 0xffffffff, 0x00000080,
 655         0xc404, 0xffffffff, 0x0020003f,
 656         0x30, 0xffffffff, 0x0000001c,
 657         0x34, 0x000f0000, 0x000f0000,
 658         0x160c, 0xffffffff, 0x00000100,
 659         0x1024, 0xffffffff, 0x00000100,
 660         0x102c, 0x00000101, 0x00000000,
 661         0x20a8, 0xffffffff, 0x00000104,
 662         0x264c, 0x000c0000, 0x000c0000,
 663         0x2648, 0x000c0000, 0x000c0000,
 664         0x55e4, 0xff000fff, 0x00000100,
 665         0x55e8, 0x00000001, 0x00000001,
 666         0x2f50, 0x00000001, 0x00000001,
 667         0x30cc, 0xc0000fff, 0x00000104,
 668         0xc1e4, 0x00000001, 0x00000001,
 669         0xd0c0, 0xfffffff0, 0x00000100,
 670         0xd8c0, 0xfffffff0, 0x00000100
 671 };
 672
 673 static const u32 pitcairn_mgcg_cgcg_init[] =
 674 {
 675         0xc400, 0xffffffff, 0xfffffffc,
 676         0x802c, 0xffffffff, 0xe0000000,
 677         0x9a60, 0xffffffff, 0x00000100,
 678         0x92a4, 0xffffffff, 0x00000100,
 679         0xc164, 0xffffffff, 0x00000100,
 680         0x9774, 0xffffffff, 0x00000100,
 681         0x8984, 0xffffffff, 0x06000100,
 682         0x8a18, 0xffffffff, 0x00000100,
 683         0x92a0, 0xffffffff, 0x00000100,
 684         0xc380, 0xffffffff, 0x00000100,
 685         0x8b28, 0xffffffff, 0x00000100,
 686         0x9144, 0xffffffff, 0x00000100,
 687         0x8d88, 0xffffffff, 0x00000100,
 688         0x8d8c, 0xffffffff, 0x00000100,
 689         0x9030, 0xffffffff, 0x00000100,
 690         0x9034, 0xffffffff, 0x00000100,
 691         0x9038, 0xffffffff, 0x00000100,
 692         0x903c, 0xffffffff, 0x00000100,
 693         0xad80, 0xffffffff, 0x00000100,
 694         0xac54, 0xffffffff, 0x00000100,
 695         0x897c, 0xffffffff, 0x06000100,
 696         0x9868, 0xffffffff, 0x00000100,
 697         0x9510, 0xffffffff, 0x00000100,
 698         0xaf04, 0xffffffff, 0x00000100,
 699         0xae04, 0xffffffff, 0x00000100,
 700         0x949c, 0xffffffff, 0x00000100,
 701         0x802c, 0xffffffff, 0xe0000000,
 702         0x9160, 0xffffffff, 0x00010000,
 703         0x9164, 0xffffffff, 0x00030002,
 704         0x9168, 0xffffffff, 0x00040007,
 705         0x916c, 0xffffffff, 0x00060005,
 706         0x9170, 0xffffffff, 0x00090008,
 707         0x9174, 0xffffffff, 0x00020001,
 708         0x9178, 0xffffffff, 0x00040003,
 709         0x917c, 0xffffffff, 0x00000007,
 710         0x9180, 0xffffffff, 0x00060005,
 711         0x9184, 0xffffffff, 0x00090008,
 712         0x9188, 0xffffffff, 0x00030002,
 713         0x918c, 0xffffffff, 0x00050004,
 714         0x9190, 0xffffffff, 0x00000008,
 715         0x9194, 0xffffffff, 0x00070006,
 716         0x9198, 0xffffffff, 0x000a0009,
 717         0x919c, 0xffffffff, 0x00040003,
 718         0x91a0, 0xffffffff, 0x00060005,
 719         0x91a4, 0xffffffff, 0x00000009,
 720         0x91a8, 0xffffffff, 0x00080007,
 721         0x91ac, 0xffffffff, 0x000b000a,
 722         0x91b0, 0xffffffff, 0x00050004,
 723         0x91b4, 0xffffffff, 0x00070006,
 724         0x91b8, 0xffffffff, 0x0008000b,
 725         0x91bc, 0xffffffff, 0x000a0009,
 726         0x91c0, 0xffffffff, 0x000d000c,
 727         0x9200, 0xffffffff, 0x00090008,
 728         0x9204, 0xffffffff, 0x000b000a,
 729         0x9208, 0xffffffff, 0x000c000f,
 730         0x920c, 0xffffffff, 0x000e000d,
 731         0x9210, 0xffffffff, 0x00110010,
 732         0x9214, 0xffffffff, 0x000a0009,
 733         0x9218, 0xffffffff, 0x000c000b,
 734         0x921c, 0xffffffff, 0x0000000f,
 735         0x9220, 0xffffffff, 0x000e000d,
 736         0x9224, 0xffffffff, 0x00110010,
 737         0x9228, 0xffffffff, 0x000b000a,
 738         0x922c, 0xffffffff, 0x000d000c,
 739         0x9230, 0xffffffff, 0x00000010,
 740         0x9234, 0xffffffff, 0x000f000e,
 741         0x9238, 0xffffffff, 0x00120011,
 742         0x923c, 0xffffffff, 0x000c000b,
 743         0x9240, 0xffffffff, 0x000e000d,
 744         0x9244, 0xffffffff, 0x00000011,
 745         0x9248, 0xffffffff, 0x0010000f,
 746         0x924c, 0xffffffff, 0x00130012,
 747         0x9250, 0xffffffff, 0x000d000c,
 748         0x9254, 0xffffffff, 0x000f000e,
 749         0x9258, 0xffffffff, 0x00100013,
 750         0x925c, 0xffffffff, 0x00120011,
 751         0x9260, 0xffffffff, 0x00150014,
 752         0x9150, 0xffffffff, 0x96940200,
 753         0x8708, 0xffffffff, 0x00900100,
 754         0xc478, 0xffffffff, 0x00000080,
 755         0xc404, 0xffffffff, 0x0020003f,
 756         0x30, 0xffffffff, 0x0000001c,
 757         0x34, 0x000f0000, 0x000f0000,
 758         0x160c, 0xffffffff, 0x00000100,
 759         0x1024, 0xffffffff, 0x00000100,
 760         0x102c, 0x00000101, 0x00000000,
 761         0x20a8, 0xffffffff, 0x00000104,
 762         0x55e4, 0xff000fff, 0x00000100,
 763         0x55e8, 0x00000001, 0x00000001,
 764         0x2f50, 0x00000001, 0x00000001,
 765         0x30cc, 0xc0000fff, 0x00000104,
 766         0xc1e4, 0x00000001, 0x00000001,
 767         0xd0c0, 0xfffffff0, 0x00000100,
 768         0xd8c0, 0xfffffff0, 0x00000100
 769 };
 770
 771 static const u32 verde_mgcg_cgcg_init[] =
 772 {
 773         0xc400, 0xffffffff, 0xfffffffc,
 774         0x802c, 0xffffffff, 0xe0000000,
 775         0x9a60, 0xffffffff, 0x00000100,
 776         0x92a4, 0xffffffff, 0x00000100,
 777         0xc164, 0xffffffff, 0x00000100,
 778         0x9774, 0xffffffff, 0x00000100,
 779         0x8984, 0xffffffff, 0x06000100,
 780         0x8a18, 0xffffffff, 0x00000100,
 781         0x92a0, 0xffffffff, 0x00000100,
 782         0xc380, 0xffffffff, 0x00000100,
 783         0x8b28, 0xffffffff, 0x00000100,
 784         0x9144, 0xffffffff, 0x00000100,
 785         0x8d88, 0xffffffff, 0x00000100,
 786         0x8d8c, 0xffffffff, 0x00000100,
 787         0x9030, 0xffffffff, 0x00000100,
 788         0x9034, 0xffffffff, 0x00000100,
 789         0x9038, 0xffffffff, 0x00000100,
 790         0x903c, 0xffffffff, 0x00000100,
 791         0xad80, 0xffffffff, 0x00000100,
 792         0xac54, 0xffffffff, 0x00000100,
 793         0x897c, 0xffffffff, 0x06000100,
 794         0x9868, 0xffffffff, 0x00000100,
 795         0x9510, 0xffffffff, 0x00000100,
 796         0xaf04, 0xffffffff, 0x00000100,
 797         0xae04, 0xffffffff, 0x00000100,
 798         0x949c, 0xffffffff, 0x00000100,
 799         0x802c, 0xffffffff, 0xe0000000,
 800         0x9160, 0xffffffff, 0x00010000,
 801         0x9164, 0xffffffff, 0x00030002,
 802         0x9168, 0xffffffff, 0x00040007,
 803         0x916c, 0xffffffff, 0x00060005,
 804         0x9170, 0xffffffff, 0x00090008,
 805         0x9174, 0xffffffff, 0x00020001,
 806         0x9178, 0xffffffff, 0x00040003,
 807         0x917c, 0xffffffff, 0x00000007,
 808         0x9180, 0xffffffff, 0x00060005,
 809         0x9184, 0xffffffff, 0x00090008,
 810         0x9188, 0xffffffff, 0x00030002,
 811         0x918c, 0xffffffff, 0x00050004,
 812         0x9190, 0xffffffff, 0x00000008,
 813         0x9194, 0xffffffff, 0x00070006,
 814         0x9198, 0xffffffff, 0x000a0009,
 815         0x919c, 0xffffffff, 0x00040003,
 816         0x91a0, 0xffffffff, 0x00060005,
 817         0x91a4, 0xffffffff, 0x00000009,
 818         0x91a8, 0xffffffff, 0x00080007,
 819         0x91ac, 0xffffffff, 0x000b000a,
 820         0x91b0, 0xffffffff, 0x00050004,
 821         0x91b4, 0xffffffff, 0x00070006,
 822         0x91b8, 0xffffffff, 0x0008000b,
 823         0x91bc, 0xffffffff, 0x000a0009,
 824         0x91c0, 0xffffffff, 0x000d000c,
 825         0x9200, 0xffffffff, 0x00090008,
 826         0x9204, 0xffffffff, 0x000b000a,
 827         0x9208, 0xffffffff, 0x000c000f,
 828         0x920c, 0xffffffff, 0x000e000d,
 829         0x9210, 0xffffffff, 0x00110010,
 830         0x9214, 0xffffffff, 0x000a0009,
 831         0x9218, 0xffffffff, 0x000c000b,
 832         0x921c, 0xffffffff, 0x0000000f,
 833         0x9220, 0xffffffff, 0x000e000d,
 834         0x9224, 0xffffffff, 0x00110010,
 835         0x9228, 0xffffffff, 0x000b000a,
 836         0x922c, 0xffffffff, 0x000d000c,
 837         0x9230, 0xffffffff, 0x00000010,
 838         0x9234, 0xffffffff, 0x000f000e,
 839         0x9238, 0xffffffff, 0x00120011,
 840         0x923c, 0xffffffff, 0x000c000b,
 841         0x9240, 0xffffffff, 0x000e000d,
 842         0x9244, 0xffffffff, 0x00000011,
 843         0x9248, 0xffffffff, 0x0010000f,
 844         0x924c, 0xffffffff, 0x00130012,
 845         0x9250, 0xffffffff, 0x000d000c,
 846         0x9254, 0xffffffff, 0x000f000e,
 847         0x9258, 0xffffffff, 0x00100013,
 848         0x925c, 0xffffffff, 0x00120011,
 849         0x9260, 0xffffffff, 0x00150014,
 850         0x9150, 0xffffffff, 0x96940200,
 851         0x8708, 0xffffffff, 0x00900100,
 852         0xc478, 0xffffffff, 0x00000080,
 853         0xc404, 0xffffffff, 0x0020003f,
 854         0x30, 0xffffffff, 0x0000001c,
 855         0x34, 0x000f0000, 0x000f0000,
 856         0x160c, 0xffffffff, 0x00000100,
 857         0x1024, 0xffffffff, 0x00000100,
 858         0x102c, 0x00000101, 0x00000000,
 859         0x20a8, 0xffffffff, 0x00000104,
 860         0x264c, 0x000c0000, 0x000c0000,
 861         0x2648, 0x000c0000, 0x000c0000,
 862         0x55e4, 0xff000fff, 0x00000100,
 863         0x55e8, 0x00000001, 0x00000001,
 864         0x2f50, 0x00000001, 0x00000001,
 865         0x30cc, 0xc0000fff, 0x00000104,
 866         0xc1e4, 0x00000001, 0x00000001,
 867         0xd0c0, 0xfffffff0, 0x00000100,
 868         0xd8c0, 0xfffffff0, 0x00000100
 869 };
 870
 871 static const u32 oland_mgcg_cgcg_init[] =
 872 {
 873         0xc400, 0xffffffff, 0xfffffffc,
 874         0x802c, 0xffffffff, 0xe0000000,
 875         0x9a60, 0xffffffff, 0x00000100,
 876         0x92a4, 0xffffffff, 0x00000100,
 877         0xc164, 0xffffffff, 0x00000100,
 878         0x9774, 0xffffffff, 0x00000100,
 879         0x8984, 0xffffffff, 0x06000100,
 880         0x8a18, 0xffffffff, 0x00000100,
 881         0x92a0, 0xffffffff, 0x00000100,
 882         0xc380, 0xffffffff, 0x00000100,
 883         0x8b28, 0xffffffff, 0x00000100,
 884         0x9144, 0xffffffff, 0x00000100,
 885         0x8d88, 0xffffffff, 0x00000100,
 886         0x8d8c, 0xffffffff, 0x00000100,
 887         0x9030, 0xffffffff, 0x00000100,
 888         0x9034, 0xffffffff, 0x00000100,
 889         0x9038, 0xffffffff, 0x00000100,
 890         0x903c, 0xffffffff, 0x00000100,
 891         0xad80, 0xffffffff, 0x00000100,
 892         0xac54, 0xffffffff, 0x00000100,
 893         0x897c, 0xffffffff, 0x06000100,
 894         0x9868, 0xffffffff, 0x00000100,
 895         0x9510, 0xffffffff, 0x00000100,
 896         0xaf04, 0xffffffff, 0x00000100,
 897         0xae04, 0xffffffff, 0x00000100,
 898         0x949c, 0xffffffff, 0x00000100,
 899         0x802c, 0xffffffff, 0xe0000000,
 900         0x9160, 0xffffffff, 0x00010000,
 901         0x9164, 0xffffffff, 0x00030002,
 902         0x9168, 0xffffffff, 0x00040007,
 903         0x916c, 0xffffffff, 0x00060005,
 904         0x9170, 0xffffffff, 0x00090008,
 905         0x9174, 0xffffffff, 0x00020001,
 906         0x9178, 0xffffffff, 0x00040003,
 907         0x917c, 0xffffffff, 0x00000007,
 908         0x9180, 0xffffffff, 0x00060005,
 909         0x9184, 0xffffffff, 0x00090008,
 910         0x9188, 0xffffffff, 0x00030002,
 911         0x918c, 0xffffffff, 0x00050004,
 912         0x9190, 0xffffffff, 0x00000008,
 913         0x9194, 0xffffffff, 0x00070006,
 914         0x9198, 0xffffffff, 0x000a0009,
 915         0x919c, 0xffffffff, 0x00040003,
 916         0x91a0, 0xffffffff, 0x00060005,
 917         0x91a4, 0xffffffff, 0x00000009,
 918         0x91a8, 0xffffffff, 0x00080007,
 919         0x91ac, 0xffffffff, 0x000b000a,
 920         0x91b0, 0xffffffff, 0x00050004,
 921         0x91b4, 0xffffffff, 0x00070006,
 922         0x91b8, 0xffffffff, 0x0008000b,
 923         0x91bc, 0xffffffff, 0x000a0009,
 924         0x91c0, 0xffffffff, 0x000d000c,
 925         0x91c4, 0xffffffff, 0x00060005,
 926         0x91c8, 0xffffffff, 0x00080007,
 927         0x91cc, 0xffffffff, 0x0000000b,
 928         0x91d0, 0xffffffff, 0x000a0009,
 929         0x91d4, 0xffffffff, 0x000d000c,
 930         0x9150, 0xffffffff, 0x96940200,
 931         0x8708, 0xffffffff, 0x00900100,
 932         0xc478, 0xffffffff, 0x00000080,
 933         0xc404, 0xffffffff, 0x0020003f,
 934         0x30, 0xffffffff, 0x0000001c,
 935         0x34, 0x000f0000, 0x000f0000,
 936         0x160c, 0xffffffff, 0x00000100,
 937         0x1024, 0xffffffff, 0x00000100,
 938         0x102c, 0x00000101, 0x00000000,
 939         0x20a8, 0xffffffff, 0x00000104,
 940         0x264c, 0x000c0000, 0x000c0000,
 941         0x2648, 0x000c0000, 0x000c0000,
 942         0x55e4, 0xff000fff, 0x00000100,
 943         0x55e8, 0x00000001, 0x00000001,
 944         0x2f50, 0x00000001, 0x00000001,
 945         0x30cc, 0xc0000fff, 0x00000104,
 946         0xc1e4, 0x00000001, 0x00000001,
 947         0xd0c0, 0xfffffff0, 0x00000100,
 948         0xd8c0, 0xfffffff0, 0x00000100
 949 };
 950
 951 static const u32 hainan_mgcg_cgcg_init[] =
 952 {
 953         0xc400, 0xffffffff, 0xfffffffc,
 954         0x802c, 0xffffffff, 0xe0000000,
 955         0x9a60, 0xffffffff, 0x00000100,
 956         0x92a4, 0xffffffff, 0x00000100,
 957         0xc164, 0xffffffff, 0x00000100,
 958         0x9774, 0xffffffff, 0x00000100,
 959         0x8984, 0xffffffff, 0x06000100,
 960         0x8a18, 0xffffffff, 0x00000100,
 961         0x92a0, 0xffffffff, 0x00000100,
 962         0xc380, 0xffffffff, 0x00000100,
 963         0x8b28, 0xffffffff, 0x00000100,
 964         0x9144, 0xffffffff, 0x00000100,
 965         0x8d88, 0xffffffff, 0x00000100,
 966         0x8d8c, 0xffffffff, 0x00000100,
 967         0x9030, 0xffffffff, 0x00000100,
 968         0x9034, 0xffffffff, 0x00000100,
 969         0x9038, 0xffffffff, 0x00000100,
 970         0x903c, 0xffffffff, 0x00000100,
 971         0xad80, 0xffffffff, 0x00000100,
 972         0xac54, 0xffffffff, 0x00000100,
 973         0x897c, 0xffffffff, 0x06000100,
 974         0x9868, 0xffffffff, 0x00000100,
 975         0x9510, 0xffffffff, 0x00000100,
 976         0xaf04, 0xffffffff, 0x00000100,
 977         0xae04, 0xffffffff, 0x00000100,
 978         0x949c, 0xffffffff, 0x00000100,
 979         0x802c, 0xffffffff, 0xe0000000,
 980         0x9160, 0xffffffff, 0x00010000,
 981         0x9164, 0xffffffff, 0x00030002,
 982         0x9168, 0xffffffff, 0x00040007,
 983         0x916c, 0xffffffff, 0x00060005,
 984         0x9170, 0xffffffff, 0x00090008,
 985         0x9174, 0xffffffff, 0x00020001,
 986         0x9178, 0xffffffff, 0x00040003,
 987         0x917c, 0xffffffff, 0x00000007,
 988         0x9180, 0xffffffff, 0x00060005,
 989         0x9184, 0xffffffff, 0x00090008,
 990         0x9188, 0xffffffff, 0x00030002,
 991         0x918c, 0xffffffff, 0x00050004,
 992         0x9190, 0xffffffff, 0x00000008,
 993         0x9194, 0xffffffff, 0x00070006,
 994         0x9198, 0xffffffff, 0x000a0009,
 995         0x919c, 0xffffffff, 0x00040003,
 996         0x91a0, 0xffffffff, 0x00060005,
 997         0x91a4, 0xffffffff, 0x00000009,
 998         0x91a8, 0xffffffff, 0x00080007,
 999         0x91ac, 0xffffffff, 0x000b000a,
1000         0x91b0, 0xffffffff, 0x00050004,
1001         0x91b4, 0xffffffff, 0x00070006,
1002         0x91b8, 0xffffffff, 0x0008000b,
1003         0x91bc, 0xffffffff, 0x000a0009,
1004         0x91c0, 0xffffffff, 0x000d000c,
1005         0x91c4, 0xffffffff, 0x00060005,
1006         0x91c8, 0xffffffff, 0x00080007,
1007         0x91cc, 0xffffffff, 0x0000000b,
1008         0x91d0, 0xffffffff, 0x000a0009,
1009         0x91d4, 0xffffffff, 0x000d000c,
1010         0x9150, 0xffffffff, 0x96940200,
1011         0x8708, 0xffffffff, 0x00900100,
1012         0xc478, 0xffffffff, 0x00000080,
1013         0xc404, 0xffffffff, 0x0020003f,
1014         0x30, 0xffffffff, 0x0000001c,
1015         0x34, 0x000f0000, 0x000f0000,
1016         0x160c, 0xffffffff, 0x00000100,
1017         0x1024, 0xffffffff, 0x00000100,
1018         0x20a8, 0xffffffff, 0x00000104,
1019         0x264c, 0x000c0000, 0x000c0000,
1020         0x2648, 0x000c0000, 0x000c0000,
1021         0x2f50, 0x00000001, 0x00000001,
1022         0x30cc, 0xc0000fff, 0x00000104,
1023         0xc1e4, 0x00000001, 0x00000001,
1024         0xd0c0, 0xfffffff0, 0x00000100,
1025         0xd8c0, 0xfffffff0, 0x00000100
1026 };
1027
1028 static u32 verde_pg_init[] =
1029 {
1030         0x353c, 0xffffffff, 0x40000,
1031         0x3538, 0xffffffff, 0x200010ff,
1032         0x353c, 0xffffffff, 0x0,
1033         0x353c, 0xffffffff, 0x0,
1034         0x353c, 0xffffffff, 0x0,
1035         0x353c, 0xffffffff, 0x0,
1036         0x353c, 0xffffffff, 0x0,
1037         0x353c, 0xffffffff, 0x7007,
1038         0x3538, 0xffffffff, 0x300010ff,
1039         0x353c, 0xffffffff, 0x0,
1040         0x353c, 0xffffffff, 0x0,
1041         0x353c, 0xffffffff, 0x0,
1042         0x353c, 0xffffffff, 0x0,
1043         0x353c, 0xffffffff, 0x0,
1044         0x353c, 0xffffffff, 0x400000,
1045         0x3538, 0xffffffff, 0x100010ff,
1046         0x353c, 0xffffffff, 0x0,
1047         0x353c, 0xffffffff, 0x0,
1048         0x353c, 0xffffffff, 0x0,
1049         0x353c, 0xffffffff, 0x0,
1050         0x353c, 0xffffffff, 0x0,
1051         0x353c, 0xffffffff, 0x120200,
1052         0x3538, 0xffffffff, 0x500010ff,
1053         0x353c, 0xffffffff, 0x0,
1054         0x353c, 0xffffffff, 0x0,
1055         0x353c, 0xffffffff, 0x0,
1056         0x353c, 0xffffffff, 0x0,
1057         0x353c, 0xffffffff, 0x0,
1058         0x353c, 0xffffffff, 0x1e1e16,
1059         0x3538, 0xffffffff, 0x600010ff,
1060         0x353c, 0xffffffff, 0x0,
1061         0x353c, 0xffffffff, 0x0,
1062         0x353c, 0xffffffff, 0x0,
1063         0x353c, 0xffffffff, 0x0,
1064         0x353c, 0xffffffff, 0x0,
1065         0x353c, 0xffffffff, 0x171f1e,
1066         0x3538, 0xffffffff, 0x700010ff,
1067         0x353c, 0xffffffff, 0x0,
1068         0x353c, 0xffffffff, 0x0,
1069         0x353c, 0xffffffff, 0x0,
1070         0x353c, 0xffffffff, 0x0,
1071         0x353c, 0xffffffff, 0x0,
1072         0x353c, 0xffffffff, 0x0,
1073         0x3538, 0xffffffff, 0x9ff,
1074         0x3500, 0xffffffff, 0x0,
1075         0x3504, 0xffffffff, 0x10000800,
1076         0x3504, 0xffffffff, 0xf,
1077         0x3504, 0xffffffff, 0xf,
1078         0x3500, 0xffffffff, 0x4,
1079         0x3504, 0xffffffff, 0x1000051e,
1080         0x3504, 0xffffffff, 0xffff,
1081         0x3504, 0xffffffff, 0xffff,
1082         0x3500, 0xffffffff, 0x8,
1083         0x3504, 0xffffffff, 0x80500,
1084         0x3500, 0xffffffff, 0x12,
1085         0x3504, 0xffffffff, 0x9050c,
1086         0x3500, 0xffffffff, 0x1d,
1087         0x3504, 0xffffffff, 0xb052c,
1088         0x3500, 0xffffffff, 0x2a,
1089         0x3504, 0xffffffff, 0x1053e,
1090         0x3500, 0xffffffff, 0x2d,
1091         0x3504, 0xffffffff, 0x10546,
1092         0x3500, 0xffffffff, 0x30,
1093         0x3504, 0xffffffff, 0xa054e,
1094         0x3500, 0xffffffff, 0x3c,
1095         0x3504, 0xffffffff, 0x1055f,
1096         0x3500, 0xffffffff, 0x3f,
1097         0x3504, 0xffffffff, 0x10567,
1098         0x3500, 0xffffffff, 0x42,
1099         0x3504, 0xffffffff, 0x1056f,
1100         0x3500, 0xffffffff, 0x45,
1101         0x3504, 0xffffffff, 0x10572,
1102         0x3500, 0xffffffff, 0x48,
1103         0x3504, 0xffffffff, 0x20575,
1104         0x3500, 0xffffffff, 0x4c,
1105         0x3504, 0xffffffff, 0x190801,
1106         0x3500, 0xffffffff, 0x67,
1107         0x3504, 0xffffffff, 0x1082a,
1108         0x3500, 0xffffffff, 0x6a,
1109         0x3504, 0xffffffff, 0x1b082d,
1110         0x3500, 0xffffffff, 0x87,
1111         0x3504, 0xffffffff, 0x310851,
1112         0x3500, 0xffffffff, 0xba,
1113         0x3504, 0xffffffff, 0x891,
1114         0x3500, 0xffffffff, 0xbc,
1115         0x3504, 0xffffffff, 0x893,
1116         0x3500, 0xffffffff, 0xbe,
1117         0x3504, 0xffffffff, 0x20895,
1118         0x3500, 0xffffffff, 0xc2,
1119         0x3504, 0xffffffff, 0x20899,
1120         0x3500, 0xffffffff, 0xc6,
1121         0x3504, 0xffffffff, 0x2089d,
1122         0x3500, 0xffffffff, 0xca,
1123         0x3504, 0xffffffff, 0x8a1,
1124         0x3500, 0xffffffff, 0xcc,
1125         0x3504, 0xffffffff, 0x8a3,
1126         0x3500, 0xffffffff, 0xce,
1127         0x3504, 0xffffffff, 0x308a5,
1128         0x3500, 0xffffffff, 0xd3,
1129         0x3504, 0xffffffff, 0x6d08cd,
1130         0x3500, 0xffffffff, 0x142,
1131         0x3504, 0xffffffff, 0x2000095a,
1132         0x3504, 0xffffffff, 0x1,
1133         0x3500, 0xffffffff, 0x144,
1134         0x3504, 0xffffffff, 0x301f095b,
1135         0x3500, 0xffffffff, 0x165,
1136         0x3504, 0xffffffff, 0xc094d,
1137         0x3500, 0xffffffff, 0x173,
1138         0x3504, 0xffffffff, 0xf096d,
1139         0x3500, 0xffffffff, 0x184,
1140         0x3504, 0xffffffff, 0x15097f,
1141         0x3500, 0xffffffff, 0x19b,
1142         0x3504, 0xffffffff, 0xc0998,
1143         0x3500, 0xffffffff, 0x1a9,
1144         0x3504, 0xffffffff, 0x409a7,
1145         0x3500, 0xffffffff, 0x1af,
1146         0x3504, 0xffffffff, 0xcdc,
1147         0x3500, 0xffffffff, 0x1b1,
1148         0x3504, 0xffffffff, 0x800,
1149         0x3508, 0xffffffff, 0x6c9b2000,
1150         0x3510, 0xfc00, 0x2000,
1151         0x3544, 0xffffffff, 0xfc0,
1152         0x28d4, 0x00000100, 0x100
1153 };
1154
1155 static void si_init_golden_registers(struct radeon_device *rdev)
1156 {
1157         switch (rdev->family) {
1158         case CHIP_TAHITI:
1159                 radeon_program_register_sequence(rdev,
1160                                                  tahiti_golden_registers,
1161                                                  (const u32)ARRAY_SIZE(tahiti_golden_registers));
1162                 radeon_program_register_sequence(rdev,
1163                                                  tahiti_golden_rlc_registers,
1164                                                  (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1165                 radeon_program_register_sequence(rdev,
1166                                                  tahiti_mgcg_cgcg_init,
1167                                                  (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1168                 radeon_program_register_sequence(rdev,
1169                                                  tahiti_golden_registers2,
1170                                                  (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1171                 break;
1172         case CHIP_PITCAIRN:
1173                 radeon_program_register_sequence(rdev,
1174                                                  pitcairn_golden_registers,
1175                                                  (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1176                 radeon_program_register_sequence(rdev,
1177                                                  pitcairn_golden_rlc_registers,
1178                                                  (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1179                 radeon_program_register_sequence(rdev,
1180                                                  pitcairn_mgcg_cgcg_init,
1181                                                  (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1182                 break;
1183         case CHIP_VERDE:
1184                 radeon_program_register_sequence(rdev,
1185                                                  verde_golden_registers,
1186                                                  (const u32)ARRAY_SIZE(verde_golden_registers));
1187                 radeon_program_register_sequence(rdev,
1188                                                  verde_golden_rlc_registers,
1189                                                  (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1190                 radeon_program_register_sequence(rdev,
1191                                                  verde_mgcg_cgcg_init,
1192                                                  (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1193                 radeon_program_register_sequence(rdev,
1194                                                  verde_pg_init,
1195                                                  (const u32)ARRAY_SIZE(verde_pg_init));
1196                 break;
1197         case CHIP_OLAND:
1198                 radeon_program_register_sequence(rdev,
1199                                                  oland_golden_registers,
1200                                                  (const u32)ARRAY_SIZE(oland_golden_registers));
1201                 radeon_program_register_sequence(rdev,
1202                                                  oland_golden_rlc_registers,
1203                                                  (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1204                 radeon_program_register_sequence(rdev,
1205                                                  oland_mgcg_cgcg_init,
1206                                                  (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1207                 break;
1208         case CHIP_HAINAN:
1209                 radeon_program_register_sequence(rdev,
1210                                                  hainan_golden_registers,
1211                                                  (const u32)ARRAY_SIZE(hainan_golden_registers));
1212                 radeon_program_register_sequence(rdev,
1213                                                  hainan_golden_registers2,
1214                                                  (const u32)ARRAY_SIZE(hainan_golden_registers2));
1215                 radeon_program_register_sequence(rdev,
1216                                                  hainan_mgcg_cgcg_init,
1217                                                  (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1218                 break;
1219         default:
1220                 break;
1221         }
1222 }
1223
1224 #define PCIE_BUS_CLK                10000
1225 #define TCLK                        (PCIE_BUS_CLK / 10)
1226
1227 /**
1228  * si_get_xclk - get the xclk
1229  *
1230  * @rdev: radeon_device pointer
1231  *
1232  * Returns the reference clock used by the gfx engine
1233  * (SI).
1234  */
1235 u32 si_get_xclk(struct radeon_device *rdev)
1236 {
1237         u32 reference_clock = rdev->clock.spll.reference_freq;
1238         u32 tmp;
1239
1240         tmp = RREG32(CG_CLKPIN_CNTL_2);
1241         if (tmp & MUX_TCLK_TO_XCLK)
1242                 return TCLK;
1243
1244         tmp = RREG32(CG_CLKPIN_CNTL);
1245         if (tmp & XTALIN_DIVIDE)
1246                 return reference_clock / 4;
1247
1248         return reference_clock;
1249 }
1250
1251 /* get temperature in millidegrees */
1252 int si_get_temp(struct radeon_device *rdev)
1253 {
1254         u32 temp;
1255         int actual_temp = 0;
1256
1257         temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1258                 CTF_TEMP_SHIFT;
1259
1260         if (temp & 0x200)
1261                 actual_temp = 255;
1262         else
1263                 actual_temp = temp & 0x1ff;
1264
1265         actual_temp = (actual_temp * 1000);
1266
1267         return actual_temp;
1268 }
1269
1270 #define TAHITI_IO_MC_REGS_SIZE 36
1271
1272 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1273         {0x0000006f, 0x03044000},
1274         {0x00000070, 0x0480c018},
1275         {0x00000071, 0x00000040},
1276         {0x00000072, 0x01000000},
1277         {0x00000074, 0x000000ff},
1278         {0x00000075, 0x00143400},
1279         {0x00000076, 0x08ec0800},
1280         {0x00000077, 0x040000cc},
1281         {0x00000079, 0x00000000},
1282         {0x0000007a, 0x21000409},
1283         {0x0000007c, 0x00000000},
1284         {0x0000007d, 0xe8000000},
1285         {0x0000007e, 0x044408a8},
1286         {0x0000007f, 0x00000003},
1287         {0x00000080, 0x00000000},
1288         {0x00000081, 0x01000000},
1289         {0x00000082, 0x02000000},
1290         {0x00000083, 0x00000000},
1291         {0x00000084, 0xe3f3e4f4},
1292         {0x00000085, 0x00052024},
1293         {0x00000087, 0x00000000},
1294         {0x00000088, 0x66036603},
1295         {0x00000089, 0x01000000},
1296         {0x0000008b, 0x1c0a0000},
1297         {0x0000008c, 0xff010000},
1298         {0x0000008e, 0xffffefff},
1299         {0x0000008f, 0xfff3efff},
1300         {0x00000090, 0xfff3efbf},
1301         {0x00000094, 0x00101101},
1302         {0x00000095, 0x00000fff},
1303         {0x00000096, 0x00116fff},
1304         {0x00000097, 0x60010000},
1305         {0x00000098, 0x10010000},
1306         {0x00000099, 0x00006000},
1307         {0x0000009a, 0x00001000},
1308         {0x0000009f, 0x00a77400}
1309 };
1310
1311 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1312         {0x0000006f, 0x03044000},
1313         {0x00000070, 0x0480c018},
1314         {0x00000071, 0x00000040},
1315         {0x00000072, 0x01000000},
1316         {0x00000074, 0x000000ff},
1317         {0x00000075, 0x00143400},
1318         {0x00000076, 0x08ec0800},
1319         {0x00000077, 0x040000cc},
1320         {0x00000079, 0x00000000},
1321         {0x0000007a, 0x21000409},
1322         {0x0000007c, 0x00000000},
1323         {0x0000007d, 0xe8000000},
1324         {0x0000007e, 0x044408a8},
1325         {0x0000007f, 0x00000003},
1326         {0x00000080, 0x00000000},
1327         {0x00000081, 0x01000000},
1328         {0x00000082, 0x02000000},
1329         {0x00000083, 0x00000000},
1330         {0x00000084, 0xe3f3e4f4},
1331         {0x00000085, 0x00052024},
1332         {0x00000087, 0x00000000},
1333         {0x00000088, 0x66036603},
1334         {0x00000089, 0x01000000},
1335         {0x0000008b, 0x1c0a0000},
1336         {0x0000008c, 0xff010000},
1337         {0x0000008e, 0xffffefff},
1338         {0x0000008f, 0xfff3efff},
1339         {0x00000090, 0xfff3efbf},
1340         {0x00000094, 0x00101101},
1341         {0x00000095, 0x00000fff},
1342         {0x00000096, 0x00116fff},
1343         {0x00000097, 0x60010000},
1344         {0x00000098, 0x10010000},
1345         {0x00000099, 0x00006000},
1346         {0x0000009a, 0x00001000},
1347         {0x0000009f, 0x00a47400}
1348 };
1349
1350 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1351         {0x0000006f, 0x03044000},
1352         {0x00000070, 0x0480c018},
1353         {0x00000071, 0x00000040},
1354         {0x00000072, 0x01000000},
1355         {0x00000074, 0x000000ff},
1356         {0x00000075, 0x00143400},
1357         {0x00000076, 0x08ec0800},
1358         {0x00000077, 0x040000cc},
1359         {0x00000079, 0x00000000},
1360         {0x0000007a, 0x21000409},
1361         {0x0000007c, 0x00000000},
1362         {0x0000007d, 0xe8000000},
1363         {0x0000007e, 0x044408a8},
1364         {0x0000007f, 0x00000003},
1365         {0x00000080, 0x00000000},
1366         {0x00000081, 0x01000000},
1367         {0x00000082, 0x02000000},
1368         {0x00000083, 0x00000000},
1369         {0x00000084, 0xe3f3e4f4},
1370         {0x00000085, 0x00052024},
1371         {0x00000087, 0x00000000},
1372         {0x00000088, 0x66036603},
1373         {0x00000089, 0x01000000},
1374         {0x0000008b, 0x1c0a0000},
1375         {0x0000008c, 0xff010000},
1376         {0x0000008e, 0xffffefff},
1377         {0x0000008f, 0xfff3efff},
1378         {0x00000090, 0xfff3efbf},
1379         {0x00000094, 0x00101101},
1380         {0x00000095, 0x00000fff},
1381         {0x00000096, 0x00116fff},
1382         {0x00000097, 0x60010000},
1383         {0x00000098, 0x10010000},
1384         {0x00000099, 0x00006000},
1385         {0x0000009a, 0x00001000},
1386         {0x0000009f, 0x00a37400}
1387 };
1388
1389 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1390         {0x0000006f, 0x03044000},
1391         {0x00000070, 0x0480c018},
1392         {0x00000071, 0x00000040},
1393         {0x00000072, 0x01000000},
1394         {0x00000074, 0x000000ff},
1395         {0x00000075, 0x00143400},
1396         {0x00000076, 0x08ec0800},
1397         {0x00000077, 0x040000cc},
1398         {0x00000079, 0x00000000},
1399         {0x0000007a, 0x21000409},
1400         {0x0000007c, 0x00000000},
1401         {0x0000007d, 0xe8000000},
1402         {0x0000007e, 0x044408a8},
1403         {0x0000007f, 0x00000003},
1404         {0x00000080, 0x00000000},
1405         {0x00000081, 0x01000000},
1406         {0x00000082, 0x02000000},
1407         {0x00000083, 0x00000000},
1408         {0x00000084, 0xe3f3e4f4},
1409         {0x00000085, 0x00052024},
1410         {0x00000087, 0x00000000},
1411         {0x00000088, 0x66036603},
1412         {0x00000089, 0x01000000},
1413         {0x0000008b, 0x1c0a0000},
1414         {0x0000008c, 0xff010000},
1415         {0x0000008e, 0xffffefff},
1416         {0x0000008f, 0xfff3efff},
1417         {0x00000090, 0xfff3efbf},
1418         {0x00000094, 0x00101101},
1419         {0x00000095, 0x00000fff},
1420         {0x00000096, 0x00116fff},
1421         {0x00000097, 0x60010000},
1422         {0x00000098, 0x10010000},
1423         {0x00000099, 0x00006000},
1424         {0x0000009a, 0x00001000},
1425         {0x0000009f, 0x00a17730}
1426 };
1427
1428 static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1429         {0x0000006f, 0x03044000},
1430         {0x00000070, 0x0480c018},
1431         {0x00000071, 0x00000040},
1432         {0x00000072, 0x01000000},
1433         {0x00000074, 0x000000ff},
1434         {0x00000075, 0x00143400},
1435         {0x00000076, 0x08ec0800},
1436         {0x00000077, 0x040000cc},
1437         {0x00000079, 0x00000000},
1438         {0x0000007a, 0x21000409},
1439         {0x0000007c, 0x00000000},
1440         {0x0000007d, 0xe8000000},
1441         {0x0000007e, 0x044408a8},
1442         {0x0000007f, 0x00000003},
1443         {0x00000080, 0x00000000},
1444         {0x00000081, 0x01000000},
1445         {0x00000082, 0x02000000},
1446         {0x00000083, 0x00000000},
1447         {0x00000084, 0xe3f3e4f4},
1448         {0x00000085, 0x00052024},
1449         {0x00000087, 0x00000000},
1450         {0x00000088, 0x66036603},
1451         {0x00000089, 0x01000000},
1452         {0x0000008b, 0x1c0a0000},
1453         {0x0000008c, 0xff010000},
1454         {0x0000008e, 0xffffefff},
1455         {0x0000008f, 0xfff3efff},
1456         {0x00000090, 0xfff3efbf},
1457         {0x00000094, 0x00101101},
1458         {0x00000095, 0x00000fff},
1459         {0x00000096, 0x00116fff},
1460         {0x00000097, 0x60010000},
1461         {0x00000098, 0x10010000},
1462         {0x00000099, 0x00006000},
1463         {0x0000009a, 0x00001000},
1464         {0x0000009f, 0x00a07730}
1465 };
1466
1467 /* ucode loading */
1468 static int si_mc_load_microcode(struct radeon_device *rdev)
1469 {
1470         const __be32 *fw_data;
1471         u32 running, blackout = 0;
1472         u32 *io_mc_regs;
1473         int i, ucode_size, regs_size;
1474
1475         if (!rdev->mc_fw)
1476                 return -EINVAL;
1477
1478         switch (rdev->family) {
1479         case CHIP_TAHITI:
1480                 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1481                 ucode_size = SI_MC_UCODE_SIZE;
1482                 regs_size = TAHITI_IO_MC_REGS_SIZE;
1483                 break;
1484         case CHIP_PITCAIRN:
1485                 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1486                 ucode_size = SI_MC_UCODE_SIZE;
1487                 regs_size = TAHITI_IO_MC_REGS_SIZE;
1488                 break;
1489         case CHIP_VERDE:
1490         default:
1491                 io_mc_regs = (u32 *)&verde_io_mc_regs;
1492                 ucode_size = SI_MC_UCODE_SIZE;
1493                 regs_size = TAHITI_IO_MC_REGS_SIZE;
1494                 break;
1495         case CHIP_OLAND:
1496                 io_mc_regs = (u32 *)&oland_io_mc_regs;
1497                 ucode_size = OLAND_MC_UCODE_SIZE;
1498                 regs_size = TAHITI_IO_MC_REGS_SIZE;
1499                 break;
1500         case CHIP_HAINAN:
1501                 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1502                 ucode_size = OLAND_MC_UCODE_SIZE;
1503                 regs_size = TAHITI_IO_MC_REGS_SIZE;
1504                 break;
1505         }
1506
1507         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1508
1509         if (running == 0) {
1510                 if (running) {
1511                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1512                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1513                 }
1514
1515                 /* reset the engine and set to writable */
1516                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1517                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1518
1519                 /* load mc io regs */
1520                 for (i = 0; i < regs_size; i++) {
1521                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1522                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1523                 }
1524                 /* load the MC ucode */
1525                 fw_data = (const __be32 *)rdev->mc_fw->data;
1526                 for (i = 0; i < ucode_size; i++)
1527                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1528
1529                 /* put the engine back into the active state */
1530                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1531                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1532                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1533
1534                 /* wait for training to complete */
1535                 for (i = 0; i < rdev->usec_timeout; i++) {
1536                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1537                                 break;
1538                         udelay(1);
1539                 }
1540                 for (i = 0; i < rdev->usec_timeout; i++) {
1541                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1542                                 break;
1543                         udelay(1);
1544                 }
1545
1546                 if (running)
1547                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1548         }
1549
1550         return 0;
1551 }
1552
1553 static int si_init_microcode(struct radeon_device *rdev)
1554 {
1555         const char *chip_name;
1556         const char *rlc_chip_name;
1557         size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1558         size_t smc_req_size;
1559         char fw_name[30];
1560         int err;
1561
1562         DRM_DEBUG("\n");
1563
1564         switch (rdev->family) {
1565         case CHIP_TAHITI:
1566                 chip_name = "TAHITI";
1567                 rlc_chip_name = "TAHITI";
1568                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1569                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1570                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1571                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1572                 mc_req_size = SI_MC_UCODE_SIZE * 4;
1573                 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
1574                 break;
1575         case CHIP_PITCAIRN:
1576                 chip_name = "PITCAIRN";
1577                 rlc_chip_name = "PITCAIRN";
1578                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1579                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1580                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1581                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1582                 mc_req_size = SI_MC_UCODE_SIZE * 4;
1583                 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
1584                 break;
1585         case CHIP_VERDE:
1586                 chip_name = "VERDE";
1587                 rlc_chip_name = "VERDE";
1588                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1589                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1590                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1591                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1592                 mc_req_size = SI_MC_UCODE_SIZE * 4;
1593                 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
1594                 break;
1595         case CHIP_OLAND:
1596                 chip_name = "OLAND";
1597                 rlc_chip_name = "OLAND";
1598                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1599                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1600                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1601                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1602                 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1603                 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
1604                 break;
1605         case CHIP_HAINAN:
1606                 chip_name = "HAINAN";
1607                 rlc_chip_name = "HAINAN";
1608                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1609                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1610                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1611                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1612                 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1613                 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
1614                 break;
1615         default: BUG();
1616         }
1617
1618         DRM_INFO("Loading %s Microcode\n", chip_name);
1619
1620         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1621         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1622         if (err)
1623                 goto out;
1624         if (rdev->pfp_fw->size != pfp_req_size) {
1625                 printk(KERN_ERR
1626                        "si_cp: Bogus length %zu in firmware \"%s\"\n",
1627                        rdev->pfp_fw->size, fw_name);
1628                 err = -EINVAL;
1629                 goto out;
1630         }
1631
1632         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1633         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1634         if (err)
1635                 goto out;
1636         if (rdev->me_fw->size != me_req_size) {
1637                 printk(KERN_ERR
1638                        "si_cp: Bogus length %zu in firmware \"%s\"\n",
1639                        rdev->me_fw->size, fw_name);
1640                 err = -EINVAL;
1641         }
1642
1643         snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1644         err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1645         if (err)
1646                 goto out;
1647         if (rdev->ce_fw->size != ce_req_size) {
1648                 printk(KERN_ERR
1649                        "si_cp: Bogus length %zu in firmware \"%s\"\n",
1650                        rdev->ce_fw->size, fw_name);
1651                 err = -EINVAL;
1652         }
1653
1654         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
1655         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1656         if (err)
1657                 goto out;
1658         if (rdev->rlc_fw->size != rlc_req_size) {
1659                 printk(KERN_ERR
1660                        "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1661                        rdev->rlc_fw->size, fw_name);
1662                 err = -EINVAL;
1663         }
1664
1665         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1666         err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1667         if (err)
1668                 goto out;
1669         if (rdev->mc_fw->size != mc_req_size) {
1670                 printk(KERN_ERR
1671                        "si_mc: Bogus length %zu in firmware \"%s\"\n",
1672                        rdev->mc_fw->size, fw_name);
1673                 err = -EINVAL;
1674         }
1675
1676         snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1677         err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1678         if (err) {
1679                 printk(KERN_ERR
1680                        "smc: error loading firmware \"%s\"\n",
1681                        fw_name);
1682                 release_firmware(rdev->smc_fw);
1683                 rdev->smc_fw = NULL;
1684                 err = 0;
1685         } else if (rdev->smc_fw->size != smc_req_size) {
1686                 printk(KERN_ERR
1687                        "si_smc: Bogus length %zu in firmware \"%s\"\n",
1688                        rdev->smc_fw->size, fw_name);
1689                 err = -EINVAL;
1690         }
1691
1692 out:
1693         if (err) {
1694                 if (err != -EINVAL)
1695                         printk(KERN_ERR
1696                                "si_cp: Failed to load firmware \"%s\"\n",
1697                                fw_name);
1698                 release_firmware(rdev->pfp_fw);
1699                 rdev->pfp_fw = NULL;
1700                 release_firmware(rdev->me_fw);
1701                 rdev->me_fw = NULL;
1702                 release_firmware(rdev->ce_fw);
1703                 rdev->ce_fw = NULL;
1704                 release_firmware(rdev->rlc_fw);
1705                 rdev->rlc_fw = NULL;
1706                 release_firmware(rdev->mc_fw);
1707                 rdev->mc_fw = NULL;
1708                 release_firmware(rdev->smc_fw);
1709                 rdev->smc_fw = NULL;
1710         }
1711         return err;
1712 }
1713
1714 /* watermark setup */
1715 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1716                                    struct radeon_crtc *radeon_crtc,
1717                                    struct drm_display_mode *mode,
1718                                    struct drm_display_mode *other_mode)
1719 {
1720         u32 tmp, buffer_alloc, i;
1721         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1722         /*
1723          * Line Buffer Setup
1724          * There are 3 line buffers, each one shared by 2 display controllers.
1725          * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1726          * the display controllers.  The paritioning is done via one of four
1727          * preset allocations specified in bits 21:20:
1728          *  0 - half lb
1729          *  2 - whole lb, other crtc must be disabled
1730          */
1731         /* this can get tricky if we have two large displays on a paired group
1732          * of crtcs.  Ideally for multiple large displays we'd assign them to
1733          * non-linked crtcs for maximum line buffer allocation.
1734          */
1735         if (radeon_crtc->base.enabled && mode) {
1736                 if (other_mode) {
1737                         tmp = 0; /* 1/2 */
1738                         buffer_alloc = 1;
1739                 } else {
1740                         tmp = 2; /* whole */
1741                         buffer_alloc = 2;
1742                 }
1743         } else {
1744                 tmp = 0;
1745                 buffer_alloc = 0;
1746         }
1747
1748         WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1749                DC_LB_MEMORY_CONFIG(tmp));
1750
1751         WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1752                DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1753         for (i = 0; i < rdev->usec_timeout; i++) {
1754                 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1755                     DMIF_BUFFERS_ALLOCATED_COMPLETED)
1756                         break;
1757                 udelay(1);
1758         }
1759
1760         if (radeon_crtc->base.enabled && mode) {
1761                 switch (tmp) {
1762                 case 0:
1763                 default:
1764                         return 4096 * 2;
1765                 case 2:
1766                         return 8192 * 2;
1767                 }
1768         }
1769
1770         /* controller not enabled, so no lb used */
1771         return 0;
1772 }
1773
1774 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
1775 {
1776         u32 tmp = RREG32(MC_SHARED_CHMAP);
1777
1778         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1779         case 0:
1780         default:
1781                 return 1;
1782         case 1:
1783                 return 2;
1784         case 2:
1785                 return 4;
1786         case 3:
1787                 return 8;
1788         case 4:
1789                 return 3;
1790         case 5:
1791                 return 6;
1792         case 6:
1793                 return 10;
1794         case 7:
1795                 return 12;
1796         case 8:
1797                 return 16;
1798         }
1799 }
1800
1801 struct dce6_wm_params {
1802         u32 dram_channels; /* number of dram channels */
1803         u32 yclk;          /* bandwidth per dram data pin in kHz */
1804         u32 sclk;          /* engine clock in kHz */
1805         u32 disp_clk;      /* display clock in kHz */
1806         u32 src_width;     /* viewport width */
1807         u32 active_time;   /* active display time in ns */
1808         u32 blank_time;    /* blank time in ns */
1809         bool interlaced;    /* mode is interlaced */
1810         fixed20_12 vsc;    /* vertical scale ratio */
1811         u32 num_heads;     /* number of active crtcs */
1812         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1813         u32 lb_size;       /* line buffer allocated to pipe */
1814         u32 vtaps;         /* vertical scaler taps */
1815 };
1816
1817 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1818 {
1819         /* Calculate raw DRAM Bandwidth */
1820         fixed20_12 dram_efficiency; /* 0.7 */
1821         fixed20_12 yclk, dram_channels, bandwidth;
1822         fixed20_12 a;
1823
1824         a.full = dfixed_const(1000);
1825         yclk.full = dfixed_const(wm->yclk);
1826         yclk.full = dfixed_div(yclk, a);
1827         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1828         a.full = dfixed_const(10);
1829         dram_efficiency.full = dfixed_const(7);
1830         dram_efficiency.full = dfixed_div(dram_efficiency, a);
1831         bandwidth.full = dfixed_mul(dram_channels, yclk);
1832         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1833
1834         return dfixed_trunc(bandwidth);
1835 }
1836
1837 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1838 {
1839         /* Calculate DRAM Bandwidth and the part allocated to display. */
1840         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1841         fixed20_12 yclk, dram_channels, bandwidth;
1842         fixed20_12 a;
1843
1844         a.full = dfixed_const(1000);
1845         yclk.full = dfixed_const(wm->yclk);
1846         yclk.full = dfixed_div(yclk, a);
1847         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1848         a.full = dfixed_const(10);
1849         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1850         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1851         bandwidth.full = dfixed_mul(dram_channels, yclk);
1852         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1853
1854         return dfixed_trunc(bandwidth);
1855 }
1856
1857 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1858 {
1859         /* Calculate the display Data return Bandwidth */
1860         fixed20_12 return_efficiency; /* 0.8 */
1861         fixed20_12 sclk, bandwidth;
1862         fixed20_12 a;
1863
1864         a.full = dfixed_const(1000);
1865         sclk.full = dfixed_const(wm->sclk);
1866         sclk.full = dfixed_div(sclk, a);
1867         a.full = dfixed_const(10);
1868         return_efficiency.full = dfixed_const(8);
1869         return_efficiency.full = dfixed_div(return_efficiency, a);
1870         a.full = dfixed_const(32);
1871         bandwidth.full = dfixed_mul(a, sclk);
1872         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1873
1874         return dfixed_trunc(bandwidth);
1875 }
1876
1877 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1878 {
1879         return 32;
1880 }
1881
1882 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1883 {
1884         /* Calculate the DMIF Request Bandwidth */
1885         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1886         fixed20_12 disp_clk, sclk, bandwidth;
1887         fixed20_12 a, b1, b2;
1888         u32 min_bandwidth;
1889
1890         a.full = dfixed_const(1000);
1891         disp_clk.full = dfixed_const(wm->disp_clk);
1892         disp_clk.full = dfixed_div(disp_clk, a);
1893         a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1894         b1.full = dfixed_mul(a, disp_clk);
1895
1896         a.full = dfixed_const(1000);
1897         sclk.full = dfixed_const(wm->sclk);
1898         sclk.full = dfixed_div(sclk, a);
1899         a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1900         b2.full = dfixed_mul(a, sclk);
1901
1902         a.full = dfixed_const(10);
1903         disp_clk_request_efficiency.full = dfixed_const(8);
1904         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1905
1906         min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1907
1908         a.full = dfixed_const(min_bandwidth);
1909         bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1910
1911         return dfixed_trunc(bandwidth);
1912 }
1913
1914 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1915 {
1916         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1917         u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1918         u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1919         u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1920
1921         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1922 }
1923
1924 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1925 {
1926         /* Calculate the display mode Average Bandwidth
1927          * DisplayMode should contain the source and destination dimensions,
1928          * timing, etc.
1929          */
1930         fixed20_12 bpp;
1931         fixed20_12 line_time;
1932         fixed20_12 src_width;
1933         fixed20_12 bandwidth;
1934         fixed20_12 a;
1935
1936         a.full = dfixed_const(1000);
1937         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1938         line_time.full = dfixed_div(line_time, a);
1939         bpp.full = dfixed_const(wm->bytes_per_pixel);
1940         src_width.full = dfixed_const(wm->src_width);
1941         bandwidth.full = dfixed_mul(src_width, bpp);
1942         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1943         bandwidth.full = dfixed_div(bandwidth, line_time);
1944
1945         return dfixed_trunc(bandwidth);
1946 }
1947
1948 static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1949 {
1950         /* First calcualte the latency in ns */
1951         u32 mc_latency = 2000; /* 2000 ns. */
1952         u32 available_bandwidth = dce6_available_bandwidth(wm);
1953         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1954         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1955         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1956         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1957                 (wm->num_heads * cursor_line_pair_return_time);
1958         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1959         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1960         u32 tmp, dmif_size = 12288;
1961         fixed20_12 a, b, c;
1962
1963         if (wm->num_heads == 0)
1964                 return 0;
1965
1966         a.full = dfixed_const(2);
1967         b.full = dfixed_const(1);
1968         if ((wm->vsc.full > a.full) ||
1969             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1970             (wm->vtaps >= 5) ||
1971             ((wm->vsc.full >= a.full) && wm->interlaced))
1972                 max_src_lines_per_dst_line = 4;
1973         else
1974                 max_src_lines_per_dst_line = 2;
1975
1976         a.full = dfixed_const(available_bandwidth);
1977         b.full = dfixed_const(wm->num_heads);
1978         a.full = dfixed_div(a, b);
1979
1980         b.full = dfixed_const(mc_latency + 512);
1981         c.full = dfixed_const(wm->disp_clk);
1982         b.full = dfixed_div(b, c);
1983
1984         c.full = dfixed_const(dmif_size);
1985         b.full = dfixed_div(c, b);
1986
1987         tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1988
1989         b.full = dfixed_const(1000);
1990         c.full = dfixed_const(wm->disp_clk);
1991         b.full = dfixed_div(c, b);
1992         c.full = dfixed_const(wm->bytes_per_pixel);
1993         b.full = dfixed_mul(b, c);
1994
1995         lb_fill_bw = min(tmp, dfixed_trunc(b));
1996
1997         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1998         b.full = dfixed_const(1000);
1999         c.full = dfixed_const(lb_fill_bw);
2000         b.full = dfixed_div(c, b);
2001         a.full = dfixed_div(a, b);
2002         line_fill_time = dfixed_trunc(a);
2003
2004         if (line_fill_time < wm->active_time)
2005                 return latency;
2006         else
2007                 return latency + (line_fill_time - wm->active_time);
2008
2009 }
2010
2011 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2012 {
2013         if (dce6_average_bandwidth(wm) <=
2014             (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2015                 return true;
2016         else
2017                 return false;
2018 };
2019
2020 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2021 {
2022         if (dce6_average_bandwidth(wm) <=
2023             (dce6_available_bandwidth(wm) / wm->num_heads))
2024                 return true;
2025         else
2026                 return false;
2027 };
2028
2029 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2030 {
2031         u32 lb_partitions = wm->lb_size / wm->src_width;
2032         u32 line_time = wm->active_time + wm->blank_time;
2033         u32 latency_tolerant_lines;
2034         u32 latency_hiding;
2035         fixed20_12 a;
2036
2037         a.full = dfixed_const(1);
2038         if (wm->vsc.full > a.full)
2039                 latency_tolerant_lines = 1;
2040         else {
2041                 if (lb_partitions <= (wm->vtaps + 1))
2042                         latency_tolerant_lines = 1;
2043                 else
2044                         latency_tolerant_lines = 2;
2045         }
2046
2047         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2048
2049         if (dce6_latency_watermark(wm) <= latency_hiding)
2050                 return true;
2051         else
2052                 return false;
2053 }
2054
2055 static void dce6_program_watermarks(struct radeon_device *rdev,
2056                                          struct radeon_crtc *radeon_crtc,
2057                                          u32 lb_size, u32 num_heads)
2058 {
2059         struct drm_display_mode *mode = &radeon_crtc->base.mode;
2060         struct dce6_wm_params wm_low, wm_high;
2061         u32 dram_channels;
2062         u32 pixel_period;
2063         u32 line_time = 0;
2064         u32 latency_watermark_a = 0, latency_watermark_b = 0;
2065         u32 priority_a_mark = 0, priority_b_mark = 0;
2066         u32 priority_a_cnt = PRIORITY_OFF;
2067         u32 priority_b_cnt = PRIORITY_OFF;
2068         u32 tmp, arb_control3;
2069         fixed20_12 a, b, c;
2070
2071         if (radeon_crtc->base.enabled && num_heads && mode) {
2072                 pixel_period = 1000000 / (u32)mode->clock;
2073                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2074                 priority_a_cnt = 0;
2075                 priority_b_cnt = 0;
2076
2077                 if (rdev->family == CHIP_ARUBA)
2078                         dram_channels = evergreen_get_number_of_dram_channels(rdev);
2079                 else
2080                         dram_channels = si_get_number_of_dram_channels(rdev);
2081
2082                 /* watermark for high clocks */
2083                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2084                         wm_high.yclk =
2085                                 radeon_dpm_get_mclk(rdev, false) * 10;
2086                         wm_high.sclk =
2087                                 radeon_dpm_get_sclk(rdev, false) * 10;
2088                 } else {
2089                         wm_high.yclk = rdev->pm.current_mclk * 10;
2090                         wm_high.sclk = rdev->pm.current_sclk * 10;
2091                 }
2092
2093                 wm_high.disp_clk = mode->clock;
2094                 wm_high.src_width = mode->crtc_hdisplay;
2095                 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2096                 wm_high.blank_time = line_time - wm_high.active_time;
2097                 wm_high.interlaced = false;
2098                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2099                         wm_high.interlaced = true;
2100                 wm_high.vsc = radeon_crtc->vsc;
2101                 wm_high.vtaps = 1;
2102                 if (radeon_crtc->rmx_type != RMX_OFF)
2103                         wm_high.vtaps = 2;
2104                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2105                 wm_high.lb_size = lb_size;
2106                 wm_high.dram_channels = dram_channels;
2107                 wm_high.num_heads = num_heads;
2108
2109                 /* watermark for low clocks */
2110                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2111                         wm_low.yclk =
2112                                 radeon_dpm_get_mclk(rdev, true) * 10;
2113                         wm_low.sclk =
2114                                 radeon_dpm_get_sclk(rdev, true) * 10;
2115                 } else {
2116                         wm_low.yclk = rdev->pm.current_mclk * 10;
2117                         wm_low.sclk = rdev->pm.current_sclk * 10;
2118                 }
2119
2120                 wm_low.disp_clk = mode->clock;
2121                 wm_low.src_width = mode->crtc_hdisplay;
2122                 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2123                 wm_low.blank_time = line_time - wm_low.active_time;
2124                 wm_low.interlaced = false;
2125                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2126                         wm_low.interlaced = true;
2127                 wm_low.vsc = radeon_crtc->vsc;
2128                 wm_low.vtaps = 1;
2129                 if (radeon_crtc->rmx_type != RMX_OFF)
2130                         wm_low.vtaps = 2;
2131                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2132                 wm_low.lb_size = lb_size;
2133                 wm_low.dram_channels = dram_channels;
2134                 wm_low.num_heads = num_heads;
2135
2136                 /* set for high clocks */
2137                 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2138                 /* set for low clocks */
2139                 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2140
2141                 /* possibly force display priority to high */
2142                 /* should really do this at mode validation time... */
2143                 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2144                     !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2145                     !dce6_check_latency_hiding(&wm_high) ||
2146                     (rdev->disp_priority == 2)) {
2147                         DRM_DEBUG_KMS("force priority to high\n");
2148                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2149                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2150                 }
2151                 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2152                     !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2153                     !dce6_check_latency_hiding(&wm_low) ||
2154                     (rdev->disp_priority == 2)) {
2155                         DRM_DEBUG_KMS("force priority to high\n");
2156                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2157                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2158                 }
2159
2160                 a.full = dfixed_const(1000);
2161                 b.full = dfixed_const(mode->clock);
2162                 b.full = dfixed_div(b, a);
2163                 c.full = dfixed_const(latency_watermark_a);
2164                 c.full = dfixed_mul(c, b);
2165                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2166                 c.full = dfixed_div(c, a);
2167                 a.full = dfixed_const(16);
2168                 c.full = dfixed_div(c, a);
2169                 priority_a_mark = dfixed_trunc(c);
2170                 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2171
2172                 a.full = dfixed_const(1000);
2173                 b.full = dfixed_const(mode->clock);
2174                 b.full = dfixed_div(b, a);
2175                 c.full = dfixed_const(latency_watermark_b);
2176                 c.full = dfixed_mul(c, b);
2177                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2178                 c.full = dfixed_div(c, a);
2179                 a.full = dfixed_const(16);
2180                 c.full = dfixed_div(c, a);
2181                 priority_b_mark = dfixed_trunc(c);
2182                 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2183         }
2184
2185         /* select wm A */
2186         arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2187         tmp = arb_control3;
2188         tmp &= ~LATENCY_WATERMARK_MASK(3);
2189         tmp |= LATENCY_WATERMARK_MASK(1);
2190         WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2191         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2192                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2193                 LATENCY_HIGH_WATERMARK(line_time)));
2194         /* select wm B */
2195         tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2196         tmp &= ~LATENCY_WATERMARK_MASK(3);
2197         tmp |= LATENCY_WATERMARK_MASK(2);
2198         WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2199         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2200                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2201                 LATENCY_HIGH_WATERMARK(line_time)));
2202         /* restore original selection */
2203         WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2204
2205         /* write the priority marks */
2206         WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2207         WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2208
2209         /* save values for DPM */
2210         radeon_crtc->line_time = line_time;
2211         radeon_crtc->wm_high = latency_watermark_a;
2212         radeon_crtc->wm_low = latency_watermark_b;
2213 }
2214
2215 void dce6_bandwidth_update(struct radeon_device *rdev)
2216 {
2217         struct drm_display_mode *mode0 = NULL;
2218         struct drm_display_mode *mode1 = NULL;
2219         u32 num_heads = 0, lb_size;
2220         int i;
2221
2222         radeon_update_display_priority(rdev);
2223
2224         for (i = 0; i < rdev->num_crtc; i++) {
2225                 if (rdev->mode_info.crtcs[i]->base.enabled)
2226                         num_heads++;
2227         }
2228         for (i = 0; i < rdev->num_crtc; i += 2) {
2229                 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2230                 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2231                 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2232                 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2233                 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2234                 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2235         }
2236 }
2237
2238 /*
2239  * Core functions
2240  */
2241 static void si_tiling_mode_table_init(struct radeon_device *rdev)
2242 {
2243         const u32 num_tile_mode_states = 32;
2244         u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2245
2246         switch (rdev->config.si.mem_row_size_in_kb) {
2247         case 1:
2248                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2249                 break;
2250         case 2:
2251         default:
2252                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2253                 break;
2254         case 4:
2255                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2256                 break;
2257         }
2258
2259         if ((rdev->family == CHIP_TAHITI) ||
2260             (rdev->family == CHIP_PITCAIRN)) {
2261                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2262                         switch (reg_offset) {
2263                         case 0:  /* non-AA compressed depth or any compressed stencil */
2264                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2265                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2266                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2267                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2268                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2269                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2270                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2271                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2272                                 break;
2273                         case 1:  /* 2xAA/4xAA compressed depth only */
2274                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2275                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2276                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2277                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2278                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2279                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2280                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2281                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2282                                 break;
2283                         case 2:  /* 8xAA compressed depth only */
2284                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2285                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2286                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2287                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2288                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2289                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2290                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2291                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2292                                 break;
2293                         case 3:  /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2294                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2295                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2296                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2297                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2298                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2299                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2300                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2301                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2302                                 break;
2303                         case 4:  /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2304                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2305                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2306                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2307                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2308                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2309                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2310                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2311                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2312                                 break;
2313                         case 5:  /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2314                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2315                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2316                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2317                                                  TILE_SPLIT(split_equal_to_row_size) |
2318                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2319                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2320                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2321                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2322                                 break;
2323                         case 6:  /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2324                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2325                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2326                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2327                                                  TILE_SPLIT(split_equal_to_row_size) |
2328                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2329                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2330                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2331                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2332                                 break;
2333                         case 7:  /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2334                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2335                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2336                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2337                                                  TILE_SPLIT(split_equal_to_row_size) |
2338                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2339                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2340                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2341                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2342                                 break;
2343                         case 8:  /* 1D and 1D Array Surfaces */
2344                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2345                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2346                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2347                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2348                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2349                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2350                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2351                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2352                                 break;
2353                         case 9:  /* Displayable maps. */
2354                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2355                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2356                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2357                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2358                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2359                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2360                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2361                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2362                                 break;
2363                         case 10:  /* Display 8bpp. */
2364                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2365                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2366                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2367                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2368                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2369                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2370                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2371                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2372                                 break;
2373                         case 11:  /* Display 16bpp. */
2374                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2375                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2376                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2377                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2378                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2379                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2380                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2381                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2382                                 break;
2383                         case 12:  /* Display 32bpp. */
2384                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2385                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2386                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2387                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2388                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2389                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2390                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2391                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2392                                 break;
2393                         case 13:  /* Thin. */
2394                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2395                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2396                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2397                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2398                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2399                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2400                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2401                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2402                                 break;
2403                         case 14:  /* Thin 8 bpp. */
2404                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2405                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2406                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2407                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2408                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2409                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2410                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2411                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2412                                 break;
2413                         case 15:  /* Thin 16 bpp. */
2414                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2415                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2416                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2417                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2418                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2419                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2420                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2421                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2422                                 break;
2423                         case 16:  /* Thin 32 bpp. */
2424                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2425                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2426                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2427                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2428                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2429                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2430                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2431                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2432                                 break;
2433                         case 17:  /* Thin 64 bpp. */
2434                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2435                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2436                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2437                                                  TILE_SPLIT(split_equal_to_row_size) |
2438                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2439                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2440                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2441                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2442                                 break;
2443                         case 21:  /* 8 bpp PRT. */
2444                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2445                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2446                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2447                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2448                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2449                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2450                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2451                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2452                                 break;
2453                         case 22:  /* 16 bpp PRT */
2454                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2455                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2456                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2457                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2458                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2459                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2460                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2461                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2462                                 break;
2463                         case 23:  /* 32 bpp PRT */
2464                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2465                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2466                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2467                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2468                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2469                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2470                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2471                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2472                                 break;
2473                         case 24:  /* 64 bpp PRT */
2474                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2475                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2476                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2477                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2478                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2479                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2480                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2481                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2482                                 break;
2483                         case 25:  /* 128 bpp PRT */
2484                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2485                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2486                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2487                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2488                                                  NUM_BANKS(ADDR_SURF_8_BANK) |
2489                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2490                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2491                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2492                                 break;
2493                         default:
2494                                 gb_tile_moden = 0;
2495                                 break;
2496                         }
2497                         rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2498                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2499                 }
2500         } else if ((rdev->family == CHIP_VERDE) ||
2501                    (rdev->family == CHIP_OLAND) ||
2502                    (rdev->family == CHIP_HAINAN)) {
2503                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2504                         switch (reg_offset) {
2505                         case 0:  /* non-AA compressed depth or any compressed stencil */
2506                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2507                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2508                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2509                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2510                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2511                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2512                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2513                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2514                                 break;
2515                         case 1:  /* 2xAA/4xAA compressed depth only */
2516                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2517                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2518                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2519                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2520                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2521                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2522                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2523                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2524                                 break;
2525                         case 2:  /* 8xAA compressed depth only */
2526                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2527                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2528                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2529                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2530                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2531                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2532                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2533                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2534                                 break;
2535                         case 3:  /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2536                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2537                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2538                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2539                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2540                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2541                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2542                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2543                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2544                                 break;
2545                         case 4:  /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2546                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2547                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2548                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2549                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2550                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2551                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2552                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2553                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2554                                 break;
2555                         case 5:  /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2556                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2557                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2558                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2559                                                  TILE_SPLIT(split_equal_to_row_size) |
2560                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2561                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2562                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2563                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2564                                 break;
2565                         case 6:  /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2566                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2567                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2568                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2569                                                  TILE_SPLIT(split_equal_to_row_size) |
2570                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2571                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2572                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2573                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2574                                 break;
2575                         case 7:  /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2576                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2577                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2578                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2579                                                  TILE_SPLIT(split_equal_to_row_size) |
2580                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2581                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2582                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2583                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2584                                 break;
2585                         case 8:  /* 1D and 1D Array Surfaces */
2586                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2587                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2588                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2589                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2590                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2591                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2592                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2593                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2594                                 break;
2595                         case 9:  /* Displayable maps. */
2596                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2597                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2598                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2599                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2600                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2601                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2602                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2603                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2604                                 break;
2605                         case 10:  /* Display 8bpp. */
2606                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2607                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2608                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2609                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2610                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2611                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2612                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2613                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2614                                 break;
2615                         case 11:  /* Display 16bpp. */
2616                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2617                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2618                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2619                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2620                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2621                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2622                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2623                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2624                                 break;
2625                         case 12:  /* Display 32bpp. */
2626                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2627                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2628                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2629                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2630                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2631                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2632                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2633                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2634                                 break;
2635                         case 13:  /* Thin. */
2636                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2637                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2638                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2639                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2640                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2641                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2642                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2643                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2644                                 break;
2645                         case 14:  /* Thin 8 bpp. */
2646                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2647                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2648                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2649                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2650                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2651                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2652                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2653                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2654                                 break;
2655                         case 15:  /* Thin 16 bpp. */
2656                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2657                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2658                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2659                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2660                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2661                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2662                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2663                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2664                                 break;
2665                         case 16:  /* Thin 32 bpp. */
2666                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2667                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2668                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2669                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2670                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2671                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2672                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2673                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2674                                 break;
2675                         case 17:  /* Thin 64 bpp. */
2676                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2677                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2678                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2679                                                  TILE_SPLIT(split_equal_to_row_size) |
2680                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2681                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2682                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2683                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2684                                 break;
2685                         case 21:  /* 8 bpp PRT. */
2686                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2687                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2688                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2689                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2690                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2691                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2692                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2693                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2694                                 break;
2695                         case 22:  /* 16 bpp PRT */
2696                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2697                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2698                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2699                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2700                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2701                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2702                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2703                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2704                                 break;
2705                         case 23:  /* 32 bpp PRT */
2706                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2707                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2708                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2709                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2710                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2711                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2712                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2713                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2714                                 break;
2715                         case 24:  /* 64 bpp PRT */
2716                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2717                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2718                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2719                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2720                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2721                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2722                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2723                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2724                                 break;
2725                         case 25:  /* 128 bpp PRT */
2726                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2727                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2728                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2729                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2730                                                  NUM_BANKS(ADDR_SURF_8_BANK) |
2731                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2732                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2733                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2734                                 break;
2735                         default:
2736                                 gb_tile_moden = 0;
2737                                 break;
2738                         }
2739                         rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2740                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2741                 }
2742         } else
2743                 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2744 }
2745
2746 static void si_select_se_sh(struct radeon_device *rdev,
2747                             u32 se_num, u32 sh_num)
2748 {
2749         u32 data = INSTANCE_BROADCAST_WRITES;
2750
2751         if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2752                 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2753         else if (se_num == 0xffffffff)
2754                 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2755         else if (sh_num == 0xffffffff)
2756                 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2757         else
2758                 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2759         WREG32(GRBM_GFX_INDEX, data);
2760 }
2761
2762 static u32 si_create_bitmask(u32 bit_width)
2763 {
2764         u32 i, mask = 0;
2765
2766         for (i = 0; i < bit_width; i++) {
2767                 mask <<= 1;
2768                 mask |= 1;
2769         }
2770         return mask;
2771 }
2772
2773 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2774 {
2775         u32 data, mask;
2776
2777         data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2778         if (data & 1)
2779                 data &= INACTIVE_CUS_MASK;
2780         else
2781                 data = 0;
2782         data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2783
2784         data >>= INACTIVE_CUS_SHIFT;
2785
2786         mask = si_create_bitmask(cu_per_sh);
2787
2788         return ~data & mask;
2789 }
2790
2791 static void si_setup_spi(struct radeon_device *rdev,
2792                          u32 se_num, u32 sh_per_se,
2793                          u32 cu_per_sh)
2794 {
2795         int i, j, k;
2796         u32 data, mask, active_cu;
2797
2798         for (i = 0; i < se_num; i++) {
2799                 for (j = 0; j < sh_per_se; j++) {
2800                         si_select_se_sh(rdev, i, j);
2801                         data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2802                         active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2803
2804                         mask = 1;
2805                         for (k = 0; k < 16; k++) {
2806                                 mask <<= k;
2807                                 if (active_cu & mask) {
2808                                         data &= ~mask;
2809                                         WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2810                                         break;
2811                                 }
2812                         }
2813                 }
2814         }
2815         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2816 }
2817
2818 static u32 si_get_rb_disabled(struct radeon_device *rdev,
2819                               u32 max_rb_num, u32 se_num,
2820                               u32 sh_per_se)
2821 {
2822         u32 data, mask;
2823
2824         data = RREG32(CC_RB_BACKEND_DISABLE);
2825         if (data & 1)
2826                 data &= BACKEND_DISABLE_MASK;
2827         else
2828                 data = 0;
2829         data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2830
2831         data >>= BACKEND_DISABLE_SHIFT;
2832
2833         mask = si_create_bitmask(max_rb_num / se_num / sh_per_se);
2834
2835         return data & mask;
2836 }
2837
2838 static void si_setup_rb(struct radeon_device *rdev,
2839                         u32 se_num, u32 sh_per_se,
2840                         u32 max_rb_num)
2841 {
2842         int i, j;
2843         u32 data, mask;
2844         u32 disabled_rbs = 0;
2845         u32 enabled_rbs = 0;
2846
2847         for (i = 0; i < se_num; i++) {
2848                 for (j = 0; j < sh_per_se; j++) {
2849                         si_select_se_sh(rdev, i, j);
2850                         data = si_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2851                         disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2852                 }
2853         }
2854         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2855
2856         mask = 1;
2857         for (i = 0; i < max_rb_num; i++) {
2858                 if (!(disabled_rbs & mask))
2859                         enabled_rbs |= mask;
2860                 mask <<= 1;
2861         }
2862
2863         for (i = 0; i < se_num; i++) {
2864                 si_select_se_sh(rdev, i, 0xffffffff);
2865                 data = 0;
2866                 for (j = 0; j < sh_per_se; j++) {
2867                         switch (enabled_rbs & 3) {
2868                         case 1:
2869                                 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2870                                 break;
2871                         case 2:
2872                                 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2873                                 break;
2874                         case 3:
2875                         default:
2876                                 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2877                                 break;
2878                         }
2879                         enabled_rbs >>= 2;
2880                 }
2881                 WREG32(PA_SC_RASTER_CONFIG, data);
2882         }
2883         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2884 }
2885
2886 static void si_gpu_init(struct radeon_device *rdev)
2887 {
2888         u32 gb_addr_config = 0;
2889         u32 mc_shared_chmap, mc_arb_ramcfg;
2890         u32 sx_debug_1;
2891         u32 hdp_host_path_cntl;
2892         u32 tmp;
2893         int i, j;
2894
2895         switch (rdev->family) {
2896         case CHIP_TAHITI:
2897                 rdev->config.si.max_shader_engines = 2;
2898                 rdev->config.si.max_tile_pipes = 12;
2899                 rdev->config.si.max_cu_per_sh = 8;
2900                 rdev->config.si.max_sh_per_se = 2;
2901                 rdev->config.si.max_backends_per_se = 4;
2902                 rdev->config.si.max_texture_channel_caches = 12;
2903                 rdev->config.si.max_gprs = 256;
2904                 rdev->config.si.max_gs_threads = 32;
2905                 rdev->config.si.max_hw_contexts = 8;
2906
2907                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2908                 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2909                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2910                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2911                 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2912                 break;
2913         case CHIP_PITCAIRN:
2914                 rdev->config.si.max_shader_engines = 2;
2915                 rdev->config.si.max_tile_pipes = 8;
2916                 rdev->config.si.max_cu_per_sh = 5;
2917                 rdev->config.si.max_sh_per_se = 2;
2918                 rdev->config.si.max_backends_per_se = 4;
2919                 rdev->config.si.max_texture_channel_caches = 8;
2920                 rdev->config.si.max_gprs = 256;
2921                 rdev->config.si.max_gs_threads = 32;
2922                 rdev->config.si.max_hw_contexts = 8;
2923
2924                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2925                 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2926                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2927                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2928                 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2929                 break;
2930         case CHIP_VERDE:
2931         default:
2932                 rdev->config.si.max_shader_engines = 1;
2933                 rdev->config.si.max_tile_pipes = 4;
2934                 rdev->config.si.max_cu_per_sh = 5;
2935                 rdev->config.si.max_sh_per_se = 2;
2936                 rdev->config.si.max_backends_per_se = 4;
2937                 rdev->config.si.max_texture_channel_caches = 4;
2938                 rdev->config.si.max_gprs = 256;
2939                 rdev->config.si.max_gs_threads = 32;
2940                 rdev->config.si.max_hw_contexts = 8;
2941
2942                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2943                 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2944                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2945                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2946                 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2947                 break;
2948         case CHIP_OLAND:
2949                 rdev->config.si.max_shader_engines = 1;
2950                 rdev->config.si.max_tile_pipes = 4;
2951                 rdev->config.si.max_cu_per_sh = 6;
2952                 rdev->config.si.max_sh_per_se = 1;
2953                 rdev->config.si.max_backends_per_se = 2;
2954                 rdev->config.si.max_texture_channel_caches = 4;
2955                 rdev->config.si.max_gprs = 256;
2956                 rdev->config.si.max_gs_threads = 16;
2957                 rdev->config.si.max_hw_contexts = 8;
2958
2959                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2960                 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2961                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2962                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2963                 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2964                 break;
2965         case CHIP_HAINAN:
2966                 rdev->config.si.max_shader_engines = 1;
2967                 rdev->config.si.max_tile_pipes = 4;
2968                 rdev->config.si.max_cu_per_sh = 5;
2969                 rdev->config.si.max_sh_per_se = 1;
2970                 rdev->config.si.max_backends_per_se = 1;
2971                 rdev->config.si.max_texture_channel_caches = 2;
2972                 rdev->config.si.max_gprs = 256;
2973                 rdev->config.si.max_gs_threads = 16;
2974                 rdev->config.si.max_hw_contexts = 8;
2975
2976                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2977                 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2978                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2979                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2980                 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
2981                 break;
2982         }
2983
2984         /* Initialize HDP */
2985         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2986                 WREG32((0x2c14 + j), 0x00000000);
2987                 WREG32((0x2c18 + j), 0x00000000);
2988                 WREG32((0x2c1c + j), 0x00000000);
2989                 WREG32((0x2c20 + j), 0x00000000);
2990                 WREG32((0x2c24 + j), 0x00000000);
2991         }
2992
2993         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2994
2995         evergreen_fix_pci_max_read_req_size(rdev);
2996
2997         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2998
2999         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3000         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3001
3002         rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
3003         rdev->config.si.mem_max_burst_length_bytes = 256;
3004         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3005         rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3006         if (rdev->config.si.mem_row_size_in_kb > 4)
3007                 rdev->config.si.mem_row_size_in_kb = 4;
3008         /* XXX use MC settings? */
3009         rdev->config.si.shader_engine_tile_size = 32;
3010         rdev->config.si.num_gpus = 1;
3011         rdev->config.si.multi_gpu_tile_size = 64;
3012
3013         /* fix up row size */
3014         gb_addr_config &= ~ROW_SIZE_MASK;
3015         switch (rdev->config.si.mem_row_size_in_kb) {
3016         case 1:
3017         default:
3018                 gb_addr_config |= ROW_SIZE(0);
3019                 break;
3020         case 2:
3021                 gb_addr_config |= ROW_SIZE(1);
3022                 break;
3023         case 4:
3024                 gb_addr_config |= ROW_SIZE(2);
3025                 break;
3026         }
3027
3028         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3029          * not have bank info, so create a custom tiling dword.
3030          * bits 3:0   num_pipes
3031          * bits 7:4   num_banks
3032          * bits 11:8  group_size
3033          * bits 15:12 row_size
3034          */
3035         rdev->config.si.tile_config = 0;
3036         switch (rdev->config.si.num_tile_pipes) {
3037         case 1:
3038                 rdev->config.si.tile_config |= (0 << 0);
3039                 break;
3040         case 2:
3041                 rdev->config.si.tile_config |= (1 << 0);
3042                 break;
3043         case 4:
3044                 rdev->config.si.tile_config |= (2 << 0);
3045                 break;
3046         case 8:
3047         default:
3048                 /* XXX what about 12? */
3049                 rdev->config.si.tile_config |= (3 << 0);
3050                 break;
3051         }
3052         switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3053         case 0: /* four banks */
3054                 rdev->config.si.tile_config |= 0 << 4;
3055                 break;
3056         case 1: /* eight banks */
3057                 rdev->config.si.tile_config |= 1 << 4;
3058                 break;
3059         case 2: /* sixteen banks */
3060         default:
3061                 rdev->config.si.tile_config |= 2 << 4;
3062                 break;
3063         }
3064         rdev->config.si.tile_config |=
3065                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3066         rdev->config.si.tile_config |=
3067                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3068
3069         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3070         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3071         WREG32(DMIF_ADDR_CALC, gb_addr_config);
3072         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3073         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3074         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3075         if (rdev->has_uvd) {
3076                 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3077                 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3078                 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3079         }
3080
3081         si_tiling_mode_table_init(rdev);
3082
3083         si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3084                     rdev->config.si.max_sh_per_se,
3085                     rdev->config.si.max_backends_per_se);
3086
3087         si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3088                      rdev->config.si.max_sh_per_se,
3089                      rdev->config.si.max_cu_per_sh);
3090
3091
3092         /* set HW defaults for 3D engine */
3093         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3094                                      ROQ_IB2_START(0x2b)));
3095         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3096
3097         sx_debug_1 = RREG32(SX_DEBUG_1);
3098         WREG32(SX_DEBUG_1, sx_debug_1);
3099
3100         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3101
3102         WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3103                                  SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3104                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3105                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3106
3107         WREG32(VGT_NUM_INSTANCES, 1);
3108
3109         WREG32(CP_PERFMON_CNTL, 0);
3110
3111         WREG32(SQ_CONFIG, 0);
3112
3113         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3114                                           FORCE_EOV_MAX_REZ_CNT(255)));
3115
3116         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3117                AUTO_INVLD_EN(ES_AND_GS_AUTO));
3118
3119         WREG32(VGT_GS_VERTEX_REUSE, 16);
3120         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3121
3122         WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3123         WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3124         WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3125         WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3126         WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3127         WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3128         WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3129         WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3130
3131         tmp = RREG32(HDP_MISC_CNTL);
3132         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3133         WREG32(HDP_MISC_CNTL, tmp);
3134
3135         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3136         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3137
3138         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3139
3140         udelay(50);
3141 }
3142
3143 /*
3144  * GPU scratch registers helpers function.
3145  */
3146 static void si_scratch_init(struct radeon_device *rdev)
3147 {
3148         int i;
3149
3150         rdev->scratch.num_reg = 7;
3151         rdev->scratch.reg_base = SCRATCH_REG0;
3152         for (i = 0; i < rdev->scratch.num_reg; i++) {
3153                 rdev->scratch.free[i] = true;
3154                 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3155         }
3156 }
3157
3158 void si_fence_ring_emit(struct radeon_device *rdev,
3159                         struct radeon_fence *fence)
3160 {
3161         struct radeon_ring *ring = &rdev->ring[fence->ring];
3162         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3163
3164         /* flush read cache over gart */
3165         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3166         radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3167         radeon_ring_write(ring, 0);
3168         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3169         radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3170                           PACKET3_TC_ACTION_ENA |
3171                           PACKET3_SH_KCACHE_ACTION_ENA |
3172                           PACKET3_SH_ICACHE_ACTION_ENA);
3173         radeon_ring_write(ring, 0xFFFFFFFF);
3174         radeon_ring_write(ring, 0);
3175         radeon_ring_write(ring, 10); /* poll interval */
3176         /* EVENT_WRITE_EOP - flush caches, send int */
3177         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3178         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3179         radeon_ring_write(ring, addr & 0xffffffff);
3180         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3181         radeon_ring_write(ring, fence->seq);
3182         radeon_ring_write(ring, 0);
3183 }
3184
3185 /*
3186  * IB stuff
3187  */
3188 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3189 {
3190         struct radeon_ring *ring = &rdev->ring[ib->ring];
3191         u32 header;
3192
3193         if (ib->is_const_ib) {
3194                 /* set switch buffer packet before const IB */
3195                 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3196                 radeon_ring_write(ring, 0);
3197
3198                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3199         } else {
3200                 u32 next_rptr;
3201                 if (ring->rptr_save_reg) {
3202                         next_rptr = ring->wptr + 3 + 4 + 8;
3203                         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3204                         radeon_ring_write(ring, ((ring->rptr_save_reg -
3205                                                   PACKET3_SET_CONFIG_REG_START) >> 2));
3206                         radeon_ring_write(ring, next_rptr);
3207                 } else if (rdev->wb.enabled) {
3208                         next_rptr = ring->wptr + 5 + 4 + 8;
3209                         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3210                         radeon_ring_write(ring, (1 << 8));
3211                         radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3212                         radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3213                         radeon_ring_write(ring, next_rptr);
3214                 }
3215
3216                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3217         }
3218
3219         radeon_ring_write(ring, header);
3220         radeon_ring_write(ring,
3221 #ifdef __BIG_ENDIAN
3222                           (2 << 0) |
3223 #endif
3224                           (ib->gpu_addr & 0xFFFFFFFC));
3225         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3226         radeon_ring_write(ring, ib->length_dw |
3227                           (ib->vm ? (ib->vm->id << 24) : 0));
3228
3229         if (!ib->is_const_ib) {
3230                 /* flush read cache over gart for this vmid */
3231                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3232                 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3233                 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
3234                 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3235                 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3236                                   PACKET3_TC_ACTION_ENA |
3237                                   PACKET3_SH_KCACHE_ACTION_ENA |
3238                                   PACKET3_SH_ICACHE_ACTION_ENA);
3239                 radeon_ring_write(ring, 0xFFFFFFFF);
3240                 radeon_ring_write(ring, 0);
3241                 radeon_ring_write(ring, 10); /* poll interval */
3242         }
3243 }
3244
3245 /*
3246  * CP.
3247  */
3248 static void si_cp_enable(struct radeon_device *rdev, bool enable)
3249 {
3250         if (enable)
3251                 WREG32(CP_ME_CNTL, 0);
3252         else {
3253                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3254                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3255                 WREG32(SCRATCH_UMSK, 0);
3256                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3257                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3258                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3259         }
3260         udelay(50);
3261 }
3262
3263 static int si_cp_load_microcode(struct radeon_device *rdev)
3264 {
3265         const __be32 *fw_data;
3266         int i;
3267
3268         if (!rdev->me_fw || !rdev->pfp_fw)
3269                 return -EINVAL;
3270
3271         si_cp_enable(rdev, false);
3272
3273         /* PFP */
3274         fw_data = (const __be32 *)rdev->pfp_fw->data;
3275         WREG32(CP_PFP_UCODE_ADDR, 0);
3276         for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3277                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3278         WREG32(CP_PFP_UCODE_ADDR, 0);
3279
3280         /* CE */
3281         fw_data = (const __be32 *)rdev->ce_fw->data;
3282         WREG32(CP_CE_UCODE_ADDR, 0);
3283         for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3284                 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3285         WREG32(CP_CE_UCODE_ADDR, 0);
3286
3287         /* ME */
3288         fw_data = (const __be32 *)rdev->me_fw->data;
3289         WREG32(CP_ME_RAM_WADDR, 0);
3290         for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3291                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3292         WREG32(CP_ME_RAM_WADDR, 0);
3293
3294         WREG32(CP_PFP_UCODE_ADDR, 0);
3295         WREG32(CP_CE_UCODE_ADDR, 0);
3296         WREG32(CP_ME_RAM_WADDR, 0);
3297         WREG32(CP_ME_RAM_RADDR, 0);
3298         return 0;
3299 }
3300
3301 static int si_cp_start(struct radeon_device *rdev)
3302 {
3303         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3304         int r, i;
3305
3306         r = radeon_ring_lock(rdev, ring, 7 + 4);
3307         if (r) {
3308                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3309                 return r;
3310         }
3311         /* init the CP */
3312         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3313         radeon_ring_write(ring, 0x1);
3314         radeon_ring_write(ring, 0x0);
3315         radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3316         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3317         radeon_ring_write(ring, 0);
3318         radeon_ring_write(ring, 0);
3319
3320         /* init the CE partitions */
3321         radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3322         radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3323         radeon_ring_write(ring, 0xc000);
3324         radeon_ring_write(ring, 0xe000);
3325         radeon_ring_unlock_commit(rdev, ring);
3326
3327         si_cp_enable(rdev, true);
3328
3329         r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3330         if (r) {
3331                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3332                 return r;
3333         }
3334
3335         /* setup clear context state */
3336         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3337         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3338
3339         for (i = 0; i < si_default_size; i++)
3340                 radeon_ring_write(ring, si_default_state[i]);
3341
3342         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3343         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3344
3345         /* set clear context state */
3346         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3347         radeon_ring_write(ring, 0);
3348
3349         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3350         radeon_ring_write(ring, 0x00000316);
3351         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3352         radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3353
3354         radeon_ring_unlock_commit(rdev, ring);
3355
3356         for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3357                 ring = &rdev->ring[i];
3358                 r = radeon_ring_lock(rdev, ring, 2);
3359
3360                 /* clear the compute context state */
3361                 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3362                 radeon_ring_write(ring, 0);
3363
3364                 radeon_ring_unlock_commit(rdev, ring);
3365         }
3366
3367         return 0;
3368 }
3369
3370 static void si_cp_fini(struct radeon_device *rdev)
3371 {
3372         struct radeon_ring *ring;
3373         si_cp_enable(rdev, false);
3374
3375         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3376         radeon_ring_fini(rdev, ring);
3377         radeon_scratch_free(rdev, ring->rptr_save_reg);
3378
3379         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3380         radeon_ring_fini(rdev, ring);
3381         radeon_scratch_free(rdev, ring->rptr_save_reg);
3382
3383         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3384         radeon_ring_fini(rdev, ring);
3385         radeon_scratch_free(rdev, ring->rptr_save_reg);
3386 }
3387
3388 static int si_cp_resume(struct radeon_device *rdev)
3389 {
3390         struct radeon_ring *ring;
3391         u32 tmp;
3392         u32 rb_bufsz;
3393         int r;
3394
3395         si_enable_gui_idle_interrupt(rdev, false);
3396
3397         WREG32(CP_SEM_WAIT_TIMER, 0x0);
3398         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3399
3400         /* Set the write pointer delay */
3401         WREG32(CP_RB_WPTR_DELAY, 0);
3402
3403         WREG32(CP_DEBUG, 0);
3404         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3405
3406         /* ring 0 - compute and gfx */
3407         /* Set ring buffer size */
3408         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3409         rb_bufsz = order_base_2(ring->ring_size / 8);
3410         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3411 #ifdef __BIG_ENDIAN
3412         tmp |= BUF_SWAP_32BIT;
3413 #endif
3414         WREG32(CP_RB0_CNTL, tmp);
3415
3416         /* Initialize the ring buffer's read and write pointers */
3417         WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3418         ring->wptr = 0;
3419         WREG32(CP_RB0_WPTR, ring->wptr);
3420
3421         /* set the wb address whether it's enabled or not */
3422         WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3423         WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3424
3425         if (rdev->wb.enabled)
3426                 WREG32(SCRATCH_UMSK, 0xff);
3427         else {
3428                 tmp |= RB_NO_UPDATE;
3429                 WREG32(SCRATCH_UMSK, 0);
3430         }
3431
3432         mdelay(1);
3433         WREG32(CP_RB0_CNTL, tmp);
3434
3435         WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3436
3437         ring->rptr = RREG32(CP_RB0_RPTR);
3438
3439         /* ring1  - compute only */
3440         /* Set ring buffer size */
3441         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3442         rb_bufsz = order_base_2(ring->ring_size / 8);
3443         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3444 #ifdef __BIG_ENDIAN
3445         tmp |= BUF_SWAP_32BIT;
3446 #endif
3447         WREG32(CP_RB1_CNTL, tmp);
3448
3449         /* Initialize the ring buffer's read and write pointers */
3450         WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3451         ring->wptr = 0;
3452         WREG32(CP_RB1_WPTR, ring->wptr);
3453
3454         /* set the wb address whether it's enabled or not */
3455         WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3456         WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3457
3458         mdelay(1);
3459         WREG32(CP_RB1_CNTL, tmp);
3460
3461         WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3462
3463         ring->rptr = RREG32(CP_RB1_RPTR);
3464
3465         /* ring2 - compute only */
3466         /* Set ring buffer size */
3467         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3468         rb_bufsz = order_base_2(ring->ring_size / 8);
3469         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3470 #ifdef __BIG_ENDIAN
3471         tmp |= BUF_SWAP_32BIT;
3472 #endif
3473         WREG32(CP_RB2_CNTL, tmp);
3474
3475         /* Initialize the ring buffer's read and write pointers */
3476         WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3477         ring->wptr = 0;
3478         WREG32(CP_RB2_WPTR, ring->wptr);
3479
3480         /* set the wb address whether it's enabled or not */
3481         WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3482         WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3483
3484         mdelay(1);
3485         WREG32(CP_RB2_CNTL, tmp);
3486
3487         WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3488
3489         ring->rptr = RREG32(CP_RB2_RPTR);
3490
3491         /* start the rings */
3492         si_cp_start(rdev);
3493         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3494         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3495         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3496         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3497         if (r) {
3498                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3499                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3500                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3501                 return r;
3502         }
3503         r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3504         if (r) {
3505                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3506         }
3507         r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3508         if (r) {
3509                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3510         }
3511
3512         si_enable_gui_idle_interrupt(rdev, true);
3513
3514         return 0;
3515 }
3516
3517 u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3518 {
3519         u32 reset_mask = 0;
3520         u32 tmp;
3521
3522         /* GRBM_STATUS */
3523         tmp = RREG32(GRBM_STATUS);
3524         if (tmp & (PA_BUSY | SC_BUSY |
3525                    BCI_BUSY | SX_BUSY |
3526                    TA_BUSY | VGT_BUSY |
3527                    DB_BUSY | CB_BUSY |
3528                    GDS_BUSY | SPI_BUSY |
3529                    IA_BUSY | IA_BUSY_NO_DMA))
3530                 reset_mask |= RADEON_RESET_GFX;
3531
3532         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3533                    CP_BUSY | CP_COHERENCY_BUSY))
3534                 reset_mask |= RADEON_RESET_CP;
3535
3536         if (tmp & GRBM_EE_BUSY)
3537                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3538
3539         /* GRBM_STATUS2 */
3540         tmp = RREG32(GRBM_STATUS2);
3541         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3542                 reset_mask |= RADEON_RESET_RLC;
3543
3544         /* DMA_STATUS_REG 0 */
3545         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3546         if (!(tmp & DMA_IDLE))
3547                 reset_mask |= RADEON_RESET_DMA;
3548
3549         /* DMA_STATUS_REG 1 */
3550         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3551         if (!(tmp & DMA_IDLE))
3552                 reset_mask |= RADEON_RESET_DMA1;
3553
3554         /* SRBM_STATUS2 */
3555         tmp = RREG32(SRBM_STATUS2);
3556         if (tmp & DMA_BUSY)
3557                 reset_mask |= RADEON_RESET_DMA;
3558
3559         if (tmp & DMA1_BUSY)
3560                 reset_mask |= RADEON_RESET_DMA1;
3561
3562         /* SRBM_STATUS */
3563         tmp = RREG32(SRBM_STATUS);
3564
3565         if (tmp & IH_BUSY)
3566                 reset_mask |= RADEON_RESET_IH;
3567
3568         if (tmp & SEM_BUSY)
3569                 reset_mask |= RADEON_RESET_SEM;
3570
3571         if (tmp & GRBM_RQ_PENDING)
3572                 reset_mask |= RADEON_RESET_GRBM;
3573
3574         if (tmp & VMC_BUSY)
3575                 reset_mask |= RADEON_RESET_VMC;
3576
3577         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3578                    MCC_BUSY | MCD_BUSY))
3579                 reset_mask |= RADEON_RESET_MC;
3580
3581         if (evergreen_is_display_hung(rdev))
3582                 reset_mask |= RADEON_RESET_DISPLAY;
3583
3584         /* VM_L2_STATUS */
3585         tmp = RREG32(VM_L2_STATUS);
3586         if (tmp & L2_BUSY)
3587                 reset_mask |= RADEON_RESET_VMC;
3588
3589         /* Skip MC reset as it's mostly likely not hung, just busy */
3590         if (reset_mask & RADEON_RESET_MC) {
3591                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3592                 reset_mask &= ~RADEON_RESET_MC;
3593         }
3594
3595         return reset_mask;
3596 }
3597
3598 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3599 {
3600         struct evergreen_mc_save save;
3601         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3602         u32 tmp;
3603
3604         if (reset_mask == 0)
3605                 return;
3606
3607         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3608
3609         evergreen_print_gpu_status_regs(rdev);
3610         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
3611                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3612         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3613                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3614
3615         /* disable PG/CG */
3616         si_fini_pg(rdev);
3617         si_fini_cg(rdev);
3618
3619         /* stop the rlc */
3620         si_rlc_stop(rdev);
3621
3622         /* Disable CP parsing/prefetching */
3623         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3624
3625         if (reset_mask & RADEON_RESET_DMA) {
3626                 /* dma0 */
3627                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3628                 tmp &= ~DMA_RB_ENABLE;
3629                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3630         }
3631         if (reset_mask & RADEON_RESET_DMA1) {
3632                 /* dma1 */
3633                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3634                 tmp &= ~DMA_RB_ENABLE;
3635                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3636         }
3637
3638         udelay(50);
3639
3640         evergreen_mc_stop(rdev, &save);
3641         if (evergreen_mc_wait_for_idle(rdev)) {
3642                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3643         }
3644
3645         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3646                 grbm_soft_reset = SOFT_RESET_CB |
3647                         SOFT_RESET_DB |
3648                         SOFT_RESET_GDS |
3649                         SOFT_RESET_PA |
3650                         SOFT_RESET_SC |
3651                         SOFT_RESET_BCI |
3652                         SOFT_RESET_SPI |
3653                         SOFT_RESET_SX |
3654                         SOFT_RESET_TC |
3655                         SOFT_RESET_TA |
3656                         SOFT_RESET_VGT |
3657                         SOFT_RESET_IA;
3658         }
3659
3660         if (reset_mask & RADEON_RESET_CP) {
3661                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3662
3663                 srbm_soft_reset |= SOFT_RESET_GRBM;
3664         }
3665
3666         if (reset_mask & RADEON_RESET_DMA)
3667                 srbm_soft_reset |= SOFT_RESET_DMA;
3668
3669         if (reset_mask & RADEON_RESET_DMA1)
3670                 srbm_soft_reset |= SOFT_RESET_DMA1;
3671
3672         if (reset_mask & RADEON_RESET_DISPLAY)
3673                 srbm_soft_reset |= SOFT_RESET_DC;
3674
3675         if (reset_mask & RADEON_RESET_RLC)
3676                 grbm_soft_reset |= SOFT_RESET_RLC;
3677
3678         if (reset_mask & RADEON_RESET_SEM)
3679                 srbm_soft_reset |= SOFT_RESET_SEM;
3680
3681         if (reset_mask & RADEON_RESET_IH)
3682                 srbm_soft_reset |= SOFT_RESET_IH;
3683
3684         if (reset_mask & RADEON_RESET_GRBM)
3685                 srbm_soft_reset |= SOFT_RESET_GRBM;
3686
3687         if (reset_mask & RADEON_RESET_VMC)
3688                 srbm_soft_reset |= SOFT_RESET_VMC;
3689
3690         if (reset_mask & RADEON_RESET_MC)
3691                 srbm_soft_reset |= SOFT_RESET_MC;
3692
3693         if (grbm_soft_reset) {
3694                 tmp = RREG32(GRBM_SOFT_RESET);
3695                 tmp |= grbm_soft_reset;
3696                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3697                 WREG32(GRBM_SOFT_RESET, tmp);
3698                 tmp = RREG32(GRBM_SOFT_RESET);
3699
3700                 udelay(50);
3701
3702                 tmp &= ~grbm_soft_reset;
3703                 WREG32(GRBM_SOFT_RESET, tmp);
3704                 tmp = RREG32(GRBM_SOFT_RESET);
3705         }
3706
3707         if (srbm_soft_reset) {
3708                 tmp = RREG32(SRBM_SOFT_RESET);
3709                 tmp |= srbm_soft_reset;
3710                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3711                 WREG32(SRBM_SOFT_RESET, tmp);
3712                 tmp = RREG32(SRBM_SOFT_RESET);
3713
3714                 udelay(50);
3715
3716                 tmp &= ~srbm_soft_reset;
3717                 WREG32(SRBM_SOFT_RESET, tmp);
3718                 tmp = RREG32(SRBM_SOFT_RESET);
3719         }
3720
3721         /* Wait a little for things to settle down */
3722         udelay(50);
3723
3724         evergreen_mc_resume(rdev, &save);
3725         udelay(50);
3726
3727         evergreen_print_gpu_status_regs(rdev);
3728 }
3729
3730 int si_asic_reset(struct radeon_device *rdev)
3731 {
3732         u32 reset_mask;
3733
3734         reset_mask = si_gpu_check_soft_reset(rdev);
3735
3736         if (reset_mask)
3737                 r600_set_bios_scratch_engine_hung(rdev, true);
3738
3739         si_gpu_soft_reset(rdev, reset_mask);
3740
3741         reset_mask = si_gpu_check_soft_reset(rdev);
3742
3743         if (!reset_mask)
3744                 r600_set_bios_scratch_engine_hung(rdev, false);
3745
3746         return 0;
3747 }
3748
3749 /**
3750  * si_gfx_is_lockup - Check if the GFX engine is locked up
3751  *
3752  * @rdev: radeon_device pointer
3753  * @ring: radeon_ring structure holding ring information
3754  *
3755  * Check if the GFX engine is locked up.
3756  * Returns true if the engine appears to be locked up, false if not.
3757  */
3758 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3759 {
3760         u32 reset_mask = si_gpu_check_soft_reset(rdev);
3761
3762         if (!(reset_mask & (RADEON_RESET_GFX |
3763                             RADEON_RESET_COMPUTE |
3764                             RADEON_RESET_CP))) {
3765                 radeon_ring_lockup_update(ring);
3766                 return false;
3767         }
3768         /* force CP activities */
3769         radeon_ring_force_activity(rdev, ring);
3770         return radeon_ring_test_lockup(rdev, ring);
3771 }
3772
3773 /* MC */
3774 static void si_mc_program(struct radeon_device *rdev)
3775 {
3776         struct evergreen_mc_save save;
3777         u32 tmp;
3778         int i, j;
3779
3780         /* Initialize HDP */
3781         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3782                 WREG32((0x2c14 + j), 0x00000000);
3783                 WREG32((0x2c18 + j), 0x00000000);
3784                 WREG32((0x2c1c + j), 0x00000000);
3785                 WREG32((0x2c20 + j), 0x00000000);
3786                 WREG32((0x2c24 + j), 0x00000000);
3787         }
3788         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3789
3790         evergreen_mc_stop(rdev, &save);
3791         if (radeon_mc_wait_for_idle(rdev)) {
3792                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3793         }
3794         if (!ASIC_IS_NODCE(rdev))
3795                 /* Lockout access through VGA aperture*/
3796                 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
3797         /* Update configuration */
3798         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3799                rdev->mc.vram_start >> 12);
3800         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3801                rdev->mc.vram_end >> 12);
3802         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3803                rdev->vram_scratch.gpu_addr >> 12);
3804         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3805         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3806         WREG32(MC_VM_FB_LOCATION, tmp);
3807         /* XXX double check these! */
3808         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3809         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3810         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3811         WREG32(MC_VM_AGP_BASE, 0);
3812         WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3813         WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3814         if (radeon_mc_wait_for_idle(rdev)) {
3815                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3816         }
3817         evergreen_mc_resume(rdev, &save);
3818         if (!ASIC_IS_NODCE(rdev)) {
3819                 /* we need to own VRAM, so turn off the VGA renderer here
3820                  * to stop it overwriting our objects */
3821                 rv515_vga_render_disable(rdev);
3822         }
3823 }
3824
3825 void si_vram_gtt_location(struct radeon_device *rdev,
3826                           struct radeon_mc *mc)
3827 {
3828         if (mc->mc_vram_size > 0xFFC0000000ULL) {
3829                 /* leave room for at least 1024M GTT */
3830                 dev_warn(rdev->dev, "limiting VRAM\n");
3831                 mc->real_vram_size = 0xFFC0000000ULL;
3832                 mc->mc_vram_size = 0xFFC0000000ULL;
3833         }
3834         radeon_vram_location(rdev, &rdev->mc, 0);
3835         rdev->mc.gtt_base_align = 0;
3836         radeon_gtt_location(rdev, mc);
3837 }
3838
3839 static int si_mc_init(struct radeon_device *rdev)
3840 {
3841         u32 tmp;
3842         int chansize, numchan;
3843
3844         /* Get VRAM informations */
3845         rdev->mc.vram_is_ddr = true;
3846         tmp = RREG32(MC_ARB_RAMCFG);
3847         if (tmp & CHANSIZE_OVERRIDE) {
3848                 chansize = 16;
3849         } else if (tmp & CHANSIZE_MASK) {
3850                 chansize = 64;
3851         } else {
3852                 chansize = 32;
3853         }
3854         tmp = RREG32(MC_SHARED_CHMAP);
3855         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3856         case 0:
3857         default:
3858                 numchan = 1;
3859                 break;
3860         case 1:
3861                 numchan = 2;
3862                 break;
3863         case 2:
3864                 numchan = 4;
3865                 break;
3866         case 3:
3867                 numchan = 8;
3868                 break;
3869         case 4:
3870                 numchan = 3;
3871                 break;
3872         case 5:
3873                 numchan = 6;
3874                 break;
3875         case 6:
3876                 numchan = 10;
3877                 break;
3878         case 7:
3879                 numchan = 12;
3880                 break;
3881         case 8:
3882                 numchan = 16;
3883                 break;
3884         }
3885         rdev->mc.vram_width = numchan * chansize;
3886         /* Could aper size report 0 ? */
3887         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3888         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3889         /* size in MB on si */
3890         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3891         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3892         rdev->mc.visible_vram_size = rdev->mc.aper_size;
3893         si_vram_gtt_location(rdev, &rdev->mc);
3894         radeon_update_bandwidth_info(rdev);
3895
3896         return 0;
3897 }
3898
3899 /*
3900  * GART
3901  */
3902 void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
3903 {
3904         /* flush hdp cache */
3905         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3906
3907         /* bits 0-15 are the VM contexts0-15 */
3908         WREG32(VM_INVALIDATE_REQUEST, 1);
3909 }
3910
3911 static int si_pcie_gart_enable(struct radeon_device *rdev)
3912 {
3913         int r, i;
3914
3915         if (rdev->gart.robj == NULL) {
3916                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
3917                 return -EINVAL;
3918         }
3919         r = radeon_gart_table_vram_pin(rdev);
3920         if (r)
3921                 return r;
3922         radeon_gart_restore(rdev);
3923         /* Setup TLB control */
3924         WREG32(MC_VM_MX_L1_TLB_CNTL,
3925                (0xA << 7) |
3926                ENABLE_L1_TLB |
3927                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3928                ENABLE_ADVANCED_DRIVER_MODEL |
3929                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3930         /* Setup L2 cache */
3931         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
3932                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3933                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3934                EFFECTIVE_L2_QUEUE_SIZE(7) |
3935                CONTEXT1_IDENTITY_ACCESS_MODE(1));
3936         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
3937         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3938                L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3939         /* setup context0 */
3940         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
3941         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
3942         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
3943         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
3944                         (u32)(rdev->dummy_page.addr >> 12));
3945         WREG32(VM_CONTEXT0_CNTL2, 0);
3946         WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
3947                                   RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
3948
3949         WREG32(0x15D4, 0);
3950         WREG32(0x15D8, 0);
3951         WREG32(0x15DC, 0);
3952
3953         /* empty context1-15 */
3954         /* set vm size, must be a multiple of 4 */
3955         WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
3956         WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
3957         /* Assign the pt base to something valid for now; the pts used for
3958          * the VMs are determined by the application and setup and assigned
3959          * on the fly in the vm part of radeon_gart.c
3960          */
3961         for (i = 1; i < 16; i++) {
3962                 if (i < 8)
3963                         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
3964                                rdev->gart.table_addr >> 12);
3965                 else
3966                         WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
3967                                rdev->gart.table_addr >> 12);
3968         }
3969
3970         /* enable context1-15 */
3971         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
3972                (u32)(rdev->dummy_page.addr >> 12));
3973         WREG32(VM_CONTEXT1_CNTL2, 4);
3974         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
3975                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3976                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3977                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3978                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3979                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
3980                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
3981                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
3982                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
3983                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
3984                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
3985                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3986                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
3987
3988         si_pcie_gart_tlb_flush(rdev);
3989         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
3990                  (unsigned)(rdev->mc.gtt_size >> 20),
3991                  (unsigned long long)rdev->gart.table_addr);
3992         rdev->gart.ready = true;
3993         return 0;
3994 }
3995
3996 static void si_pcie_gart_disable(struct radeon_device *rdev)
3997 {
3998         /* Disable all tables */
3999         WREG32(VM_CONTEXT0_CNTL, 0);
4000         WREG32(VM_CONTEXT1_CNTL, 0);
4001         /* Setup TLB control */
4002         WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4003                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4004         /* Setup L2 cache */
4005         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4006                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4007                EFFECTIVE_L2_QUEUE_SIZE(7) |
4008                CONTEXT1_IDENTITY_ACCESS_MODE(1));
4009         WREG32(VM_L2_CNTL2, 0);
4010         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4011                L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4012         radeon_gart_table_vram_unpin(rdev);
4013 }
4014
4015 static void si_pcie_gart_fini(struct radeon_device *rdev)
4016 {
4017         si_pcie_gart_disable(rdev);
4018         radeon_gart_table_vram_free(rdev);
4019         radeon_gart_fini(rdev);
4020 }
4021
4022 /* vm parser */
4023 static bool si_vm_reg_valid(u32 reg)
4024 {
4025         /* context regs are fine */
4026         if (reg >= 0x28000)
4027                 return true;
4028
4029         /* check config regs */
4030         switch (reg) {
4031         case GRBM_GFX_INDEX:
4032         case CP_STRMOUT_CNTL:
4033         case VGT_VTX_VECT_EJECT_REG:
4034         case VGT_CACHE_INVALIDATION:
4035         case VGT_ESGS_RING_SIZE:
4036         case VGT_GSVS_RING_SIZE:
4037         case VGT_GS_VERTEX_REUSE:
4038         case VGT_PRIMITIVE_TYPE:
4039         case VGT_INDEX_TYPE:
4040         case VGT_NUM_INDICES:
4041         case VGT_NUM_INSTANCES:
4042         case VGT_TF_RING_SIZE:
4043         case VGT_HS_OFFCHIP_PARAM:
4044         case VGT_TF_MEMORY_BASE:
4045         case PA_CL_ENHANCE:
4046         case PA_SU_LINE_STIPPLE_VALUE:
4047         case PA_SC_LINE_STIPPLE_STATE:
4048         case PA_SC_ENHANCE:
4049         case SQC_CACHES:
4050         case SPI_STATIC_THREAD_MGMT_1:
4051         case SPI_STATIC_THREAD_MGMT_2:
4052         case SPI_STATIC_THREAD_MGMT_3:
4053         case SPI_PS_MAX_WAVE_ID:
4054         case SPI_CONFIG_CNTL:
4055         case SPI_CONFIG_CNTL_1:
4056         case TA_CNTL_AUX:
4057                 return true;
4058         default:
4059                 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4060                 return false;
4061         }
4062 }
4063
4064 static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4065                                   u32 *ib, struct radeon_cs_packet *pkt)
4066 {
4067         switch (pkt->opcode) {
4068         case PACKET3_NOP:
4069         case PACKET3_SET_BASE:
4070         case PACKET3_SET_CE_DE_COUNTERS:
4071         case PACKET3_LOAD_CONST_RAM:
4072         case PACKET3_WRITE_CONST_RAM:
4073         case PACKET3_WRITE_CONST_RAM_OFFSET:
4074         case PACKET3_DUMP_CONST_RAM:
4075         case PACKET3_INCREMENT_CE_COUNTER:
4076         case PACKET3_WAIT_ON_DE_COUNTER:
4077         case PACKET3_CE_WRITE:
4078                 break;
4079         default:
4080                 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4081                 return -EINVAL;
4082         }
4083         return 0;
4084 }
4085
4086 static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4087 {
4088         u32 start_reg, reg, i;
4089         u32 command = ib[idx + 4];
4090         u32 info = ib[idx + 1];
4091         u32 idx_value = ib[idx];
4092         if (command & PACKET3_CP_DMA_CMD_SAS) {
4093                 /* src address space is register */
4094                 if (((info & 0x60000000) >> 29) == 0) {
4095                         start_reg = idx_value << 2;
4096                         if (command & PACKET3_CP_DMA_CMD_SAIC) {
4097                                 reg = start_reg;
4098                                 if (!si_vm_reg_valid(reg)) {
4099                                         DRM_ERROR("CP DMA Bad SRC register\n");
4100                                         return -EINVAL;
4101                                 }
4102                         } else {
4103                                 for (i = 0; i < (command & 0x1fffff); i++) {
4104                                         reg = start_reg + (4 * i);
4105                                         if (!si_vm_reg_valid(reg)) {
4106                                                 DRM_ERROR("CP DMA Bad SRC register\n");
4107                                                 return -EINVAL;
4108                                         }
4109                                 }
4110                         }
4111                 }
4112         }
4113         if (command & PACKET3_CP_DMA_CMD_DAS) {
4114                 /* dst address space is register */
4115                 if (((info & 0x00300000) >> 20) == 0) {
4116                         start_reg = ib[idx + 2];
4117                         if (command & PACKET3_CP_DMA_CMD_DAIC) {
4118                                 reg = start_reg;
4119                                 if (!si_vm_reg_valid(reg)) {
4120                                         DRM_ERROR("CP DMA Bad DST register\n");
4121                                         return -EINVAL;
4122                                 }
4123                         } else {
4124                                 for (i = 0; i < (command & 0x1fffff); i++) {
4125                                         reg = start_reg + (4 * i);
4126                                 if (!si_vm_reg_valid(reg)) {
4127                                                 DRM_ERROR("CP DMA Bad DST register\n");
4128                                                 return -EINVAL;
4129                                         }
4130                                 }
4131                         }
4132                 }
4133         }
4134         return 0;
4135 }
4136
4137 static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4138                                    u32 *ib, struct radeon_cs_packet *pkt)
4139 {
4140         int r;
4141         u32 idx = pkt->idx + 1;
4142         u32 idx_value = ib[idx];
4143         u32 start_reg, end_reg, reg, i;
4144
4145         switch (pkt->opcode) {
4146         case PACKET3_NOP:
4147         case PACKET3_SET_BASE:
4148         case PACKET3_CLEAR_STATE:
4149         case PACKET3_INDEX_BUFFER_SIZE:
4150         case PACKET3_DISPATCH_DIRECT:
4151         case PACKET3_DISPATCH_INDIRECT:
4152         case PACKET3_ALLOC_GDS:
4153         case PACKET3_WRITE_GDS_RAM:
4154         case PACKET3_ATOMIC_GDS:
4155         case PACKET3_ATOMIC:
4156         case PACKET3_OCCLUSION_QUERY:
4157         case PACKET3_SET_PREDICATION:
4158         case PACKET3_COND_EXEC:
4159         case PACKET3_PRED_EXEC:
4160         case PACKET3_DRAW_INDIRECT:
4161         case PACKET3_DRAW_INDEX_INDIRECT:
4162         case PACKET3_INDEX_BASE:
4163         case PACKET3_DRAW_INDEX_2:
4164         case PACKET3_CONTEXT_CONTROL:
4165         case PACKET3_INDEX_TYPE:
4166         case PACKET3_DRAW_INDIRECT_MULTI:
4167         case PACKET3_DRAW_INDEX_AUTO:
4168         case PACKET3_DRAW_INDEX_IMMD:
4169         case PACKET3_NUM_INSTANCES:
4170         case PACKET3_DRAW_INDEX_MULTI_AUTO:
4171         case PACKET3_STRMOUT_BUFFER_UPDATE:
4172         case PACKET3_DRAW_INDEX_OFFSET_2:
4173         case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4174         case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4175         case PACKET3_MPEG_INDEX:
4176         case PACKET3_WAIT_REG_MEM:
4177         case PACKET3_MEM_WRITE:
4178         case PACKET3_PFP_SYNC_ME:
4179         case PACKET3_SURFACE_SYNC:
4180         case PACKET3_EVENT_WRITE:
4181         case PACKET3_EVENT_WRITE_EOP:
4182         case PACKET3_EVENT_WRITE_EOS:
4183         case PACKET3_SET_CONTEXT_REG:
4184         case PACKET3_SET_CONTEXT_REG_INDIRECT:
4185         case PACKET3_SET_SH_REG:
4186         case PACKET3_SET_SH_REG_OFFSET:
4187         case PACKET3_INCREMENT_DE_COUNTER:
4188         case PACKET3_WAIT_ON_CE_COUNTER:
4189         case PACKET3_WAIT_ON_AVAIL_BUFFER:
4190         case PACKET3_ME_WRITE:
4191                 break;
4192         case PACKET3_COPY_DATA:
4193                 if ((idx_value & 0xf00) == 0) {
4194                         reg = ib[idx + 3] * 4;
4195                         if (!si_vm_reg_valid(reg))
4196                                 return -EINVAL;
4197                 }
4198                 break;
4199         case PACKET3_WRITE_DATA:
4200                 if ((idx_value & 0xf00) == 0) {
4201                         start_reg = ib[idx + 1] * 4;
4202                         if (idx_value & 0x10000) {
4203                                 if (!si_vm_reg_valid(start_reg))
4204                                         return -EINVAL;
4205                         } else {
4206                                 for (i = 0; i < (pkt->count - 2); i++) {
4207                                         reg = start_reg + (4 * i);
4208                                         if (!si_vm_reg_valid(reg))
4209                                                 return -EINVAL;
4210                                 }
4211                         }
4212                 }
4213                 break;
4214         case PACKET3_COND_WRITE:
4215                 if (idx_value & 0x100) {
4216                         reg = ib[idx + 5] * 4;
4217                         if (!si_vm_reg_valid(reg))
4218                                 return -EINVAL;
4219                 }
4220                 break;
4221         case PACKET3_COPY_DW:
4222                 if (idx_value & 0x2) {
4223                         reg = ib[idx + 3] * 4;
4224                         if (!si_vm_reg_valid(reg))
4225                                 return -EINVAL;
4226                 }
4227                 break;
4228         case PACKET3_SET_CONFIG_REG:
4229                 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4230                 end_reg = 4 * pkt->count + start_reg - 4;
4231                 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4232                     (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4233                     (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4234                         DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4235                         return -EINVAL;
4236                 }
4237                 for (i = 0; i < pkt->count; i++) {
4238                         reg = start_reg + (4 * i);
4239                         if (!si_vm_reg_valid(reg))
4240                                 return -EINVAL;
4241                 }
4242                 break;
4243         case PACKET3_CP_DMA:
4244                 r = si_vm_packet3_cp_dma_check(ib, idx);
4245                 if (r)
4246                         return r;
4247                 break;
4248         default:
4249                 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4250                 return -EINVAL;
4251         }
4252         return 0;
4253 }
4254
4255 static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4256                                        u32 *ib, struct radeon_cs_packet *pkt)
4257 {
4258         int r;
4259         u32 idx = pkt->idx + 1;
4260         u32 idx_value = ib[idx];
4261         u32 start_reg, reg, i;
4262
4263         switch (pkt->opcode) {
4264         case PACKET3_NOP:
4265         case PACKET3_SET_BASE:
4266         case PACKET3_CLEAR_STATE:
4267         case PACKET3_DISPATCH_DIRECT:
4268         case PACKET3_DISPATCH_INDIRECT:
4269         case PACKET3_ALLOC_GDS:
4270         case PACKET3_WRITE_GDS_RAM:
4271         case PACKET3_ATOMIC_GDS:
4272         case PACKET3_ATOMIC:
4273         case PACKET3_OCCLUSION_QUERY:
4274         case PACKET3_SET_PREDICATION:
4275         case PACKET3_COND_EXEC:
4276         case PACKET3_PRED_EXEC:
4277         case PACKET3_CONTEXT_CONTROL:
4278         case PACKET3_STRMOUT_BUFFER_UPDATE:
4279         case PACKET3_WAIT_REG_MEM:
4280         case PACKET3_MEM_WRITE:
4281         case PACKET3_PFP_SYNC_ME:
4282         case PACKET3_SURFACE_SYNC:
4283         case PACKET3_EVENT_WRITE:
4284         case PACKET3_EVENT_WRITE_EOP:
4285         case PACKET3_EVENT_WRITE_EOS:
4286         case PACKET3_SET_CONTEXT_REG:
4287         case PACKET3_SET_CONTEXT_REG_INDIRECT:
4288         case PACKET3_SET_SH_REG:
4289         case PACKET3_SET_SH_REG_OFFSET:
4290         case PACKET3_INCREMENT_DE_COUNTER:
4291         case PACKET3_WAIT_ON_CE_COUNTER:
4292         case PACKET3_WAIT_ON_AVAIL_BUFFER:
4293         case PACKET3_ME_WRITE:
4294                 break;
4295         case PACKET3_COPY_DATA:
4296                 if ((idx_value & 0xf00) == 0) {
4297                         reg = ib[idx + 3] * 4;
4298                         if (!si_vm_reg_valid(reg))
4299                                 return -EINVAL;
4300                 }
4301                 break;
4302         case PACKET3_WRITE_DATA:
4303                 if ((idx_value & 0xf00) == 0) {
4304                         start_reg = ib[idx + 1] * 4;
4305                         if (idx_value & 0x10000) {
4306                                 if (!si_vm_reg_valid(start_reg))
4307                                         return -EINVAL;
4308                         } else {
4309                                 for (i = 0; i < (pkt->count - 2); i++) {
4310                                         reg = start_reg + (4 * i);
4311                                         if (!si_vm_reg_valid(reg))
4312                                                 return -EINVAL;
4313                                 }
4314                         }
4315                 }
4316                 break;
4317         case PACKET3_COND_WRITE:
4318                 if (idx_value & 0x100) {
4319                         reg = ib[idx + 5] * 4;
4320                         if (!si_vm_reg_valid(reg))
4321                                 return -EINVAL;
4322                 }
4323                 break;
4324         case PACKET3_COPY_DW:
4325                 if (idx_value & 0x2) {
4326                         reg = ib[idx + 3] * 4;
4327                         if (!si_vm_reg_valid(reg))
4328                                 return -EINVAL;
4329                 }
4330                 break;
4331         case PACKET3_CP_DMA:
4332                 r = si_vm_packet3_cp_dma_check(ib, idx);
4333                 if (r)
4334                         return r;
4335                 break;
4336         default:
4337                 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4338                 return -EINVAL;
4339         }
4340         return 0;
4341 }
4342
4343 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4344 {
4345         int ret = 0;
4346         u32 idx = 0;
4347         struct radeon_cs_packet pkt;
4348
4349         do {
4350                 pkt.idx = idx;
4351                 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4352                 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
4353                 pkt.one_reg_wr = 0;
4354                 switch (pkt.type) {
4355                 case RADEON_PACKET_TYPE0:
4356                         dev_err(rdev->dev, "Packet0 not allowed!\n");
4357                         ret = -EINVAL;
4358                         break;
4359                 case RADEON_PACKET_TYPE2:
4360                         idx += 1;
4361                         break;
4362                 case RADEON_PACKET_TYPE3:
4363                         pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
4364                         if (ib->is_const_ib)
4365                                 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4366                         else {
4367                                 switch (ib->ring) {
4368                                 case RADEON_RING_TYPE_GFX_INDEX:
4369                                         ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4370                                         break;
4371                                 case CAYMAN_RING_TYPE_CP1_INDEX:
4372                                 case CAYMAN_RING_TYPE_CP2_INDEX:
4373                                         ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4374                                         break;
4375                                 default:
4376                                         dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
4377                                         ret = -EINVAL;
4378                                         break;
4379                                 }
4380                         }
4381                         idx += pkt.count + 2;
4382                         break;
4383                 default:
4384                         dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4385                         ret = -EINVAL;
4386                         break;
4387                 }
4388                 if (ret)
4389                         break;
4390         } while (idx < ib->length_dw);
4391
4392         return ret;
4393 }
4394
4395 /*
4396  * vm
4397  */
4398 int si_vm_init(struct radeon_device *rdev)
4399 {
4400         /* number of VMs */
4401         rdev->vm_manager.nvm = 16;
4402         /* base offset of vram pages */
4403         rdev->vm_manager.vram_base_offset = 0;
4404
4405         return 0;
4406 }
4407
4408 void si_vm_fini(struct radeon_device *rdev)
4409 {
4410 }
4411
4412 /**
4413  * si_vm_decode_fault - print human readable fault info
4414  *
4415  * @rdev: radeon_device pointer
4416  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4417  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4418  *
4419  * Print human readable fault information (SI).
4420  */
4421 static void si_vm_decode_fault(struct radeon_device *rdev,
4422                                u32 status, u32 addr)
4423 {
4424         u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4425         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4426         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4427         char *block;
4428
4429         if (rdev->family == CHIP_TAHITI) {
4430                 switch (mc_id) {
4431                 case 160:
4432                 case 144:
4433                 case 96:
4434                 case 80:
4435                 case 224:
4436                 case 208:
4437                 case 32:
4438                 case 16:
4439                         block = "CB";
4440                         break;
4441                 case 161:
4442                 case 145:
4443                 case 97:
4444                 case 81:
4445                 case 225:
4446                 case 209:
4447                 case 33:
4448                 case 17:
4449                         block = "CB_FMASK";
4450                         break;
4451                 case 162:
4452                 case 146:
4453                 case 98:
4454                 case 82:
4455                 case 226:
4456                 case 210:
4457                 case 34:
4458                 case 18:
4459                         block = "CB_CMASK";
4460                         break;
4461                 case 163:
4462                 case 147:
4463                 case 99:
4464                 case 83:
4465                 case 227:
4466                 case 211:
4467                 case 35:
4468                 case 19:
4469                         block = "CB_IMMED";
4470                         break;
4471                 case 164:
4472                 case 148:
4473                 case 100:
4474                 case 84:
4475                 case 228:
4476                 case 212:
4477                 case 36:
4478                 case 20:
4479                         block = "DB";
4480                         break;
4481                 case 165:
4482                 case 149:
4483                 case 101:
4484                 case 85:
4485                 case 229:
4486                 case 213:
4487                 case 37:
4488                 case 21:
4489                         block = "DB_HTILE";
4490                         break;
4491                 case 167:
4492                 case 151:
4493                 case 103:
4494                 case 87:
4495                 case 231:
4496                 case 215:
4497                 case 39:
4498                 case 23:
4499                         block = "DB_STEN";
4500                         break;
4501                 case 72:
4502                 case 68:
4503                 case 64:
4504                 case 8:
4505                 case 4:
4506                 case 0:
4507                 case 136:
4508                 case 132:
4509                 case 128:
4510                 case 200:
4511                 case 196:
4512                 case 192:
4513                         block = "TC";
4514                         break;
4515                 case 112:
4516                 case 48:
4517                         block = "CP";
4518                         break;
4519                 case 49:
4520                 case 177:
4521                 case 50:
4522                 case 178:
4523                         block = "SH";
4524                         break;
4525                 case 53:
4526                 case 190:
4527                         block = "VGT";
4528                         break;
4529                 case 117:
4530                         block = "IH";
4531                         break;
4532                 case 51:
4533                 case 115:
4534                         block = "RLC";
4535                         break;
4536                 case 119:
4537                 case 183:
4538                         block = "DMA0";
4539                         break;
4540                 case 61:
4541                         block = "DMA1";
4542                         break;
4543                 case 248:
4544                 case 120:
4545                         block = "HDP";
4546                         break;
4547                 default:
4548                         block = "unknown";
4549                         break;
4550                 }
4551         } else {
4552                 switch (mc_id) {
4553                 case 32:
4554                 case 16:
4555                 case 96:
4556                 case 80:
4557                 case 160:
4558                 case 144:
4559                 case 224:
4560                 case 208:
4561                         block = "CB";
4562                         break;
4563                 case 33:
4564                 case 17:
4565                 case 97:
4566                 case 81:
4567                 case 161:
4568                 case 145:
4569                 case 225:
4570                 case 209:
4571                         block = "CB_FMASK";
4572                         break;
4573                 case 34:
4574                 case 18:
4575                 case 98:
4576                 case 82:
4577                 case 162:
4578                 case 146:
4579                 case 226:
4580                 case 210:
4581                         block = "CB_CMASK";
4582                         break;
4583                 case 35:
4584                 case 19:
4585                 case 99:
4586                 case 83:
4587                 case 163:
4588                 case 147:
4589                 case 227:
4590                 case 211:
4591                         block = "CB_IMMED";
4592                         break;
4593                 case 36:
4594                 case 20:
4595                 case 100:
4596                 case 84:
4597                 case 164:
4598                 case 148:
4599                 case 228:
4600                 case 212:
4601                         block = "DB";
4602                         break;
4603                 case 37:
4604                 case 21:
4605                 case 101:
4606                 case 85:
4607                 case 165:
4608                 case 149:
4609                 case 229:
4610                 case 213:
4611                         block = "DB_HTILE";
4612                         break;
4613                 case 39:
4614                 case 23:
4615                 case 103:
4616                 case 87:
4617                 case 167:
4618                 case 151:
4619                 case 231:
4620                 case 215:
4621                         block = "DB_STEN";
4622                         break;
4623                 case 72:
4624                 case 68:
4625                 case 8:
4626                 case 4:
4627                 case 136:
4628                 case 132:
4629                 case 200:
4630                 case 196:
4631                         block = "TC";
4632                         break;
4633                 case 112:
4634                 case 48:
4635                         block = "CP";
4636                         break;
4637                 case 49:
4638                 case 177:
4639                 case 50:
4640                 case 178:
4641                         block = "SH";
4642                         break;
4643                 case 53:
4644                         block = "VGT";
4645                         break;
4646                 case 117:
4647                         block = "IH";
4648                         break;
4649                 case 51:
4650                 case 115:
4651                         block = "RLC";
4652                         break;
4653                 case 119:
4654                 case 183:
4655                         block = "DMA0";
4656                         break;
4657                 case 61:
4658                         block = "DMA1";
4659                         break;
4660                 case 248:
4661                 case 120:
4662                         block = "HDP";
4663                         break;
4664                 default:
4665                         block = "unknown";
4666                         break;
4667                 }
4668         }
4669
4670         printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4671                protections, vmid, addr,
4672                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
4673                block, mc_id);
4674 }
4675
4676 /**
4677  * si_vm_set_page - update the page tables using the CP
4678  *
4679  * @rdev: radeon_device pointer
4680  * @ib: indirect buffer to fill with commands
4681  * @pe: addr of the page entry
4682  * @addr: dst addr to write into pe
4683  * @count: number of page entries to update
4684  * @incr: increase next addr by incr bytes
4685  * @flags: access flags
4686  *
4687  * Update the page tables using the CP (SI).
4688  */
4689 void si_vm_set_page(struct radeon_device *rdev,
4690                     struct radeon_ib *ib,
4691                     uint64_t pe,
4692                     uint64_t addr, unsigned count,
4693                     uint32_t incr, uint32_t flags)
4694 {
4695         uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
4696         uint64_t value;
4697         unsigned ndw;
4698
4699         if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
4700                 while (count) {
4701                         ndw = 2 + count * 2;
4702                         if (ndw > 0x3FFE)
4703                                 ndw = 0x3FFE;
4704
4705                         ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
4706                         ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
4707                                         WRITE_DATA_DST_SEL(1));
4708                         ib->ptr[ib->length_dw++] = pe;
4709                         ib->ptr[ib->length_dw++] = upper_32_bits(pe);
4710                         for (; ndw > 2; ndw -= 2, --count, pe += 8) {
4711                                 if (flags & RADEON_VM_PAGE_SYSTEM) {
4712                                         value = radeon_vm_map_gart(rdev, addr);
4713                                         value &= 0xFFFFFFFFFFFFF000ULL;
4714                                 } else if (flags & RADEON_VM_PAGE_VALID) {
4715                                         value = addr;
4716                                 } else {
4717                                         value = 0;
4718                                 }
4719                                 addr += incr;
4720                                 value |= r600_flags;
4721                                 ib->ptr[ib->length_dw++] = value;
4722                                 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4723                         }
4724                 }
4725         } else {
4726                 /* DMA */
4727                 si_dma_vm_set_page(rdev, ib, pe, addr, count, incr, flags);
4728         }
4729 }
4730
4731 void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4732 {
4733         struct radeon_ring *ring = &rdev->ring[ridx];
4734
4735         if (vm == NULL)
4736                 return;
4737
4738         /* write new base address */
4739         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4740         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4741                                  WRITE_DATA_DST_SEL(0)));
4742
4743         if (vm->id < 8) {
4744                 radeon_ring_write(ring,
4745                                   (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
4746         } else {
4747                 radeon_ring_write(ring,
4748                                   (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
4749         }
4750         radeon_ring_write(ring, 0);
4751         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4752
4753         /* flush hdp cache */
4754         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4755         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4756                                  WRITE_DATA_DST_SEL(0)));
4757         radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4758         radeon_ring_write(ring, 0);
4759         radeon_ring_write(ring, 0x1);
4760
4761         /* bits 0-15 are the VM contexts0-15 */
4762         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4763         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4764                                  WRITE_DATA_DST_SEL(0)));
4765         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4766         radeon_ring_write(ring, 0);
4767         radeon_ring_write(ring, 1 << vm->id);
4768
4769         /* sync PFP to ME, otherwise we might get invalid PFP reads */
4770         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4771         radeon_ring_write(ring, 0x0);
4772 }
4773
4774 /*
4775  *  Power and clock gating
4776  */
4777 static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
4778 {
4779         int i;
4780
4781         for (i = 0; i < rdev->usec_timeout; i++) {
4782                 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
4783                         break;
4784                 udelay(1);
4785         }
4786
4787         for (i = 0; i < rdev->usec_timeout; i++) {
4788                 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
4789                         break;
4790                 udelay(1);
4791         }
4792 }
4793
4794 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
4795                                          bool enable)
4796 {
4797         u32 tmp = RREG32(CP_INT_CNTL_RING0);
4798         u32 mask;
4799         int i;
4800
4801         if (enable)
4802                 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4803         else
4804                 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4805         WREG32(CP_INT_CNTL_RING0, tmp);
4806
4807         if (!enable) {
4808                 /* read a gfx register */
4809                 tmp = RREG32(DB_DEPTH_INFO);
4810
4811                 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
4812                 for (i = 0; i < rdev->usec_timeout; i++) {
4813                         if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
4814                                 break;
4815                         udelay(1);
4816                 }
4817         }
4818 }
4819
4820 static void si_set_uvd_dcm(struct radeon_device *rdev,
4821                            bool sw_mode)
4822 {
4823         u32 tmp, tmp2;
4824
4825         tmp = RREG32(UVD_CGC_CTRL);
4826         tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
4827         tmp |= DCM | CG_DT(1) | CLK_OD(4);
4828
4829         if (sw_mode) {
4830                 tmp &= ~0x7ffff800;
4831                 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
4832         } else {
4833                 tmp |= 0x7ffff800;
4834                 tmp2 = 0;
4835         }
4836
4837         WREG32(UVD_CGC_CTRL, tmp);
4838         WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
4839 }
4840
4841 void si_init_uvd_internal_cg(struct radeon_device *rdev)
4842 {
4843         bool hw_mode = true;
4844
4845         if (hw_mode) {
4846                 si_set_uvd_dcm(rdev, false);
4847         } else {
4848                 u32 tmp = RREG32(UVD_CGC_CTRL);
4849                 tmp &= ~DCM;
4850                 WREG32(UVD_CGC_CTRL, tmp);
4851         }
4852 }
4853
4854 static u32 si_halt_rlc(struct radeon_device *rdev)
4855 {
4856         u32 data, orig;
4857
4858         orig = data = RREG32(RLC_CNTL);
4859
4860         if (data & RLC_ENABLE) {
4861                 data &= ~RLC_ENABLE;
4862                 WREG32(RLC_CNTL, data);
4863
4864                 si_wait_for_rlc_serdes(rdev);
4865         }
4866
4867         return orig;
4868 }
4869
4870 static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
4871 {
4872         u32 tmp;
4873
4874         tmp = RREG32(RLC_CNTL);
4875         if (tmp != rlc)
4876                 WREG32(RLC_CNTL, rlc);
4877 }
4878
4879 static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
4880 {
4881         u32 data, orig;
4882
4883         orig = data = RREG32(DMA_PG);
4884         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
4885                 data |= PG_CNTL_ENABLE;
4886         else
4887                 data &= ~PG_CNTL_ENABLE;
4888         if (orig != data)
4889                 WREG32(DMA_PG, data);
4890 }
4891
4892 static void si_init_dma_pg(struct radeon_device *rdev)
4893 {
4894         u32 tmp;
4895
4896         WREG32(DMA_PGFSM_WRITE,  0x00002000);
4897         WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
4898
4899         for (tmp = 0; tmp < 5; tmp++)
4900                 WREG32(DMA_PGFSM_WRITE, 0);
4901 }
4902
4903 static void si_enable_gfx_cgpg(struct radeon_device *rdev,
4904                                bool enable)
4905 {
4906         u32 tmp;
4907
4908         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
4909                 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
4910                 WREG32(RLC_TTOP_D, tmp);
4911
4912                 tmp = RREG32(RLC_PG_CNTL);
4913                 tmp |= GFX_PG_ENABLE;
4914                 WREG32(RLC_PG_CNTL, tmp);
4915
4916                 tmp = RREG32(RLC_AUTO_PG_CTRL);
4917                 tmp |= AUTO_PG_EN;
4918                 WREG32(RLC_AUTO_PG_CTRL, tmp);
4919         } else {
4920                 tmp = RREG32(RLC_AUTO_PG_CTRL);
4921                 tmp &= ~AUTO_PG_EN;
4922                 WREG32(RLC_AUTO_PG_CTRL, tmp);
4923
4924                 tmp = RREG32(DB_RENDER_CONTROL);
4925         }
4926 }
4927
4928 static void si_init_gfx_cgpg(struct radeon_device *rdev)
4929 {
4930         u32 tmp;
4931
4932         WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4933
4934         tmp = RREG32(RLC_PG_CNTL);
4935         tmp |= GFX_PG_SRC;
4936         WREG32(RLC_PG_CNTL, tmp);
4937
4938         WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4939
4940         tmp = RREG32(RLC_AUTO_PG_CTRL);
4941
4942         tmp &= ~GRBM_REG_SGIT_MASK;
4943         tmp |= GRBM_REG_SGIT(0x700);
4944         tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
4945         WREG32(RLC_AUTO_PG_CTRL, tmp);
4946 }
4947
4948 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
4949 {
4950         u32 mask = 0, tmp, tmp1;
4951         int i;
4952
4953         si_select_se_sh(rdev, se, sh);
4954         tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
4955         tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
4956         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
4957
4958         tmp &= 0xffff0000;
4959
4960         tmp |= tmp1;
4961         tmp >>= 16;
4962
4963         for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
4964                 mask <<= 1;
4965                 mask |= 1;
4966         }
4967
4968         return (~tmp) & mask;
4969 }
4970
4971 static void si_init_ao_cu_mask(struct radeon_device *rdev)
4972 {
4973         u32 i, j, k, active_cu_number = 0;
4974         u32 mask, counter, cu_bitmap;
4975         u32 tmp = 0;
4976
4977         for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
4978                 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
4979                         mask = 1;
4980                         cu_bitmap = 0;
4981                         counter  = 0;
4982                         for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
4983                                 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
4984                                         if (counter < 2)
4985                                                 cu_bitmap |= mask;
4986                                         counter++;
4987                                 }
4988                                 mask <<= 1;
4989                         }
4990
4991                         active_cu_number += counter;
4992                         tmp |= (cu_bitmap << (i * 16 + j * 8));
4993                 }
4994         }
4995
4996         WREG32(RLC_PG_AO_CU_MASK, tmp);
4997
4998         tmp = RREG32(RLC_MAX_PG_CU);
4999         tmp &= ~MAX_PU_CU_MASK;
5000         tmp |= MAX_PU_CU(active_cu_number);
5001         WREG32(RLC_MAX_PG_CU, tmp);
5002 }
5003
5004 static void si_enable_cgcg(struct radeon_device *rdev,
5005                            bool enable)
5006 {
5007         u32 data, orig, tmp;
5008
5009         orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5010
5011         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5012                 si_enable_gui_idle_interrupt(rdev, true);
5013
5014                 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5015
5016                 tmp = si_halt_rlc(rdev);
5017
5018                 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5019                 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5020                 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5021
5022                 si_wait_for_rlc_serdes(rdev);
5023
5024                 si_update_rlc(rdev, tmp);
5025
5026                 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5027
5028                 data |= CGCG_EN | CGLS_EN;
5029         } else {
5030                 si_enable_gui_idle_interrupt(rdev, false);
5031
5032                 RREG32(CB_CGTT_SCLK_CTRL);
5033                 RREG32(CB_CGTT_SCLK_CTRL);
5034                 RREG32(CB_CGTT_SCLK_CTRL);
5035                 RREG32(CB_CGTT_SCLK_CTRL);
5036
5037                 data &= ~(CGCG_EN | CGLS_EN);
5038         }
5039
5040         if (orig != data)
5041                 WREG32(RLC_CGCG_CGLS_CTRL, data);
5042 }
5043
5044 static void si_enable_mgcg(struct radeon_device *rdev,
5045                            bool enable)
5046 {
5047         u32 data, orig, tmp = 0;
5048
5049         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5050                 orig = data = RREG32(CGTS_SM_CTRL_REG);
5051                 data = 0x96940200;
5052                 if (orig != data)
5053                         WREG32(CGTS_SM_CTRL_REG, data);
5054
5055                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5056                         orig = data = RREG32(CP_MEM_SLP_CNTL);
5057                         data |= CP_MEM_LS_EN;
5058                         if (orig != data)
5059                                 WREG32(CP_MEM_SLP_CNTL, data);
5060                 }
5061
5062                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5063                 data &= 0xffffffc0;
5064                 if (orig != data)
5065                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5066
5067                 tmp = si_halt_rlc(rdev);
5068
5069                 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5070                 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5071                 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5072
5073                 si_update_rlc(rdev, tmp);
5074         } else {
5075                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5076                 data |= 0x00000003;
5077                 if (orig != data)
5078                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5079
5080                 data = RREG32(CP_MEM_SLP_CNTL);
5081                 if (data & CP_MEM_LS_EN) {
5082                         data &= ~CP_MEM_LS_EN;
5083                         WREG32(CP_MEM_SLP_CNTL, data);
5084                 }
5085                 orig = data = RREG32(CGTS_SM_CTRL_REG);
5086                 data |= LS_OVERRIDE | OVERRIDE;
5087                 if (orig != data)
5088                         WREG32(CGTS_SM_CTRL_REG, data);
5089
5090                 tmp = si_halt_rlc(rdev);
5091
5092                 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5093                 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5094                 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5095
5096                 si_update_rlc(rdev, tmp);
5097         }
5098 }
5099
5100 static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5101                                bool enable)
5102 {
5103         u32 orig, data, tmp;
5104
5105         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5106                 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5107                 tmp |= 0x3fff;
5108                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5109
5110                 orig = data = RREG32(UVD_CGC_CTRL);
5111                 data |= DCM;
5112                 if (orig != data)
5113                         WREG32(UVD_CGC_CTRL, data);
5114
5115                 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5116                 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5117         } else {
5118                 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5119                 tmp &= ~0x3fff;
5120                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5121
5122                 orig = data = RREG32(UVD_CGC_CTRL);
5123                 data &= ~DCM;
5124                 if (orig != data)
5125                         WREG32(UVD_CGC_CTRL, data);
5126
5127                 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5128                 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5129         }
5130 }
5131
5132 static const u32 mc_cg_registers[] =
5133 {
5134         MC_HUB_MISC_HUB_CG,
5135         MC_HUB_MISC_SIP_CG,
5136         MC_HUB_MISC_VM_CG,
5137         MC_XPB_CLK_GAT,
5138         ATC_MISC_CG,
5139         MC_CITF_MISC_WR_CG,
5140         MC_CITF_MISC_RD_CG,
5141         MC_CITF_MISC_VM_CG,
5142         VM_L2_CG,
5143 };
5144
5145 static void si_enable_mc_ls(struct radeon_device *rdev,
5146                             bool enable)
5147 {
5148         int i;
5149         u32 orig, data;
5150
5151         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5152                 orig = data = RREG32(mc_cg_registers[i]);
5153                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5154                         data |= MC_LS_ENABLE;
5155                 else
5156                         data &= ~MC_LS_ENABLE;
5157                 if (data != orig)
5158                         WREG32(mc_cg_registers[i], data);
5159         }
5160 }
5161
5162 static void si_enable_mc_mgcg(struct radeon_device *rdev,
5163                                bool enable)
5164 {
5165         int i;
5166         u32 orig, data;
5167
5168         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5169                 orig = data = RREG32(mc_cg_registers[i]);
5170                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5171                         data |= MC_CG_ENABLE;
5172                 else
5173                         data &= ~MC_CG_ENABLE;
5174                 if (data != orig)
5175                         WREG32(mc_cg_registers[i], data);
5176         }
5177 }
5178
5179 static void si_enable_dma_mgcg(struct radeon_device *rdev,
5180                                bool enable)
5181 {
5182         u32 orig, data, offset;
5183         int i;
5184
5185         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5186                 for (i = 0; i < 2; i++) {
5187                         if (i == 0)
5188                                 offset = DMA0_REGISTER_OFFSET;
5189                         else
5190                                 offset = DMA1_REGISTER_OFFSET;
5191                         orig = data = RREG32(DMA_POWER_CNTL + offset);
5192                         data &= ~MEM_POWER_OVERRIDE;
5193                         if (data != orig)
5194                                 WREG32(DMA_POWER_CNTL + offset, data);
5195                         WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5196                 }
5197         } else {
5198                 for (i = 0; i < 2; i++) {
5199                         if (i == 0)
5200                                 offset = DMA0_REGISTER_OFFSET;
5201                         else
5202                                 offset = DMA1_REGISTER_OFFSET;
5203                         orig = data = RREG32(DMA_POWER_CNTL + offset);
5204                         data |= MEM_POWER_OVERRIDE;
5205                         if (data != orig)
5206                                 WREG32(DMA_POWER_CNTL + offset, data);
5207
5208                         orig = data = RREG32(DMA_CLK_CTRL + offset);
5209                         data = 0xff000000;
5210                         if (data != orig)
5211                                 WREG32(DMA_CLK_CTRL + offset, data);
5212                 }
5213         }
5214 }
5215
5216 static void si_enable_bif_mgls(struct radeon_device *rdev,
5217                                bool enable)
5218 {
5219         u32 orig, data;
5220
5221         orig = data = RREG32_PCIE(PCIE_CNTL2);
5222
5223         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5224                 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5225                         REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5226         else
5227                 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5228                           REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5229
5230         if (orig != data)
5231                 WREG32_PCIE(PCIE_CNTL2, data);
5232 }
5233
5234 static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5235                                bool enable)
5236 {
5237         u32 orig, data;
5238
5239         orig = data = RREG32(HDP_HOST_PATH_CNTL);
5240
5241         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5242                 data &= ~CLOCK_GATING_DIS;
5243         else
5244                 data |= CLOCK_GATING_DIS;
5245
5246         if (orig != data)
5247                 WREG32(HDP_HOST_PATH_CNTL, data);
5248 }
5249
5250 static void si_enable_hdp_ls(struct radeon_device *rdev,
5251                              bool enable)
5252 {
5253         u32 orig, data;
5254
5255         orig = data = RREG32(HDP_MEM_POWER_LS);
5256
5257         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5258                 data |= HDP_LS_ENABLE;
5259         else
5260                 data &= ~HDP_LS_ENABLE;
5261
5262         if (orig != data)
5263                 WREG32(HDP_MEM_POWER_LS, data);
5264 }
5265
5266 void si_update_cg(struct radeon_device *rdev,
5267                   u32 block, bool enable)
5268 {
5269         if (block & RADEON_CG_BLOCK_GFX) {
5270                 si_enable_gui_idle_interrupt(rdev, false);
5271                 /* order matters! */
5272                 if (enable) {
5273                         si_enable_mgcg(rdev, true);
5274                         si_enable_cgcg(rdev, true);
5275                 } else {
5276                         si_enable_cgcg(rdev, false);
5277                         si_enable_mgcg(rdev, false);
5278                 }
5279                 si_enable_gui_idle_interrupt(rdev, true);
5280         }
5281
5282         if (block & RADEON_CG_BLOCK_MC) {
5283                 si_enable_mc_mgcg(rdev, enable);
5284                 si_enable_mc_ls(rdev, enable);
5285         }
5286
5287         if (block & RADEON_CG_BLOCK_SDMA) {
5288                 si_enable_dma_mgcg(rdev, enable);
5289         }
5290
5291         if (block & RADEON_CG_BLOCK_BIF) {
5292                 si_enable_bif_mgls(rdev, enable);
5293         }
5294
5295         if (block & RADEON_CG_BLOCK_UVD) {
5296                 if (rdev->has_uvd) {
5297                         si_enable_uvd_mgcg(rdev, enable);
5298                 }
5299         }
5300
5301         if (block & RADEON_CG_BLOCK_HDP) {
5302                 si_enable_hdp_mgcg(rdev, enable);
5303                 si_enable_hdp_ls(rdev, enable);
5304         }
5305 }
5306
5307 static void si_init_cg(struct radeon_device *rdev)
5308 {
5309         si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5310                             RADEON_CG_BLOCK_MC |
5311                             RADEON_CG_BLOCK_SDMA |
5312                             RADEON_CG_BLOCK_BIF |
5313                             RADEON_CG_BLOCK_HDP), true);
5314         if (rdev->has_uvd) {
5315                 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
5316                 si_init_uvd_internal_cg(rdev);
5317         }
5318 }
5319
5320 static void si_fini_cg(struct radeon_device *rdev)
5321 {
5322         if (rdev->has_uvd) {
5323                 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
5324         }
5325         si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5326                             RADEON_CG_BLOCK_MC |
5327                             RADEON_CG_BLOCK_SDMA |
5328                             RADEON_CG_BLOCK_BIF |
5329                             RADEON_CG_BLOCK_HDP), false);
5330 }
5331
5332 u32 si_get_csb_size(struct radeon_device *rdev)
5333 {
5334         u32 count = 0;
5335         const struct cs_section_def *sect = NULL;
5336         const struct cs_extent_def *ext = NULL;
5337
5338         if (rdev->rlc.cs_data == NULL)
5339                 return 0;
5340
5341         /* begin clear state */
5342         count += 2;
5343         /* context control state */
5344         count += 3;
5345
5346         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5347                 for (ext = sect->section; ext->extent != NULL; ++ext) {
5348                         if (sect->id == SECT_CONTEXT)
5349                                 count += 2 + ext->reg_count;
5350                         else
5351                                 return 0;
5352                 }
5353         }
5354         /* pa_sc_raster_config */
5355         count += 3;
5356         /* end clear state */
5357         count += 2;
5358         /* clear state */
5359         count += 2;
5360
5361         return count;
5362 }
5363
5364 void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5365 {
5366         u32 count = 0, i;
5367         const struct cs_section_def *sect = NULL;
5368         const struct cs_extent_def *ext = NULL;
5369
5370         if (rdev->rlc.cs_data == NULL)
5371                 return;
5372         if (buffer == NULL)
5373                 return;
5374
5375         buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
5376         buffer[count++] = PACKET3_PREAMBLE_BEGIN_CLEAR_STATE;
5377
5378         buffer[count++] = PACKET3(PACKET3_CONTEXT_CONTROL, 1);
5379         buffer[count++] = 0x80000000;
5380         buffer[count++] = 0x80000000;
5381
5382         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5383                 for (ext = sect->section; ext->extent != NULL; ++ext) {
5384                         if (sect->id == SECT_CONTEXT) {
5385                                 buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count);
5386                                 buffer[count++] = ext->reg_index - 0xa000;
5387                                 for (i = 0; i < ext->reg_count; i++)
5388                                         buffer[count++] = ext->extent[i];
5389                         } else {
5390                                 return;
5391                         }
5392                 }
5393         }
5394
5395         buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, 1);
5396         buffer[count++] = PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START;
5397         switch (rdev->family) {
5398         case CHIP_TAHITI:
5399         case CHIP_PITCAIRN:
5400                 buffer[count++] = 0x2a00126a;
5401                 break;
5402         case CHIP_VERDE:
5403                 buffer[count++] = 0x0000124a;
5404                 break;
5405         case CHIP_OLAND:
5406                 buffer[count++] = 0x00000082;
5407                 break;
5408         case CHIP_HAINAN:
5409                 buffer[count++] = 0x00000000;
5410                 break;
5411         default:
5412                 buffer[count++] = 0x00000000;
5413                 break;
5414         }
5415
5416         buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
5417         buffer[count++] = PACKET3_PREAMBLE_END_CLEAR_STATE;
5418
5419         buffer[count++] = PACKET3(PACKET3_CLEAR_STATE, 0);
5420         buffer[count++] = 0;
5421 }
5422
5423 static void si_init_pg(struct radeon_device *rdev)
5424 {
5425         if (rdev->pg_flags) {
5426                 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5427                         si_init_dma_pg(rdev);
5428                 }
5429                 si_init_ao_cu_mask(rdev);
5430                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5431                         si_init_gfx_cgpg(rdev);
5432                 }
5433                 si_enable_dma_pg(rdev, true);
5434                 si_enable_gfx_cgpg(rdev, true);
5435         } else {
5436                 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5437                 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5438         }
5439 }
5440
5441 static void si_fini_pg(struct radeon_device *rdev)
5442 {
5443         if (rdev->pg_flags) {
5444                 si_enable_dma_pg(rdev, false);
5445                 si_enable_gfx_cgpg(rdev, false);
5446         }
5447 }
5448
5449 /*
5450  * RLC
5451  */
5452 void si_rlc_reset(struct radeon_device *rdev)
5453 {
5454         u32 tmp = RREG32(GRBM_SOFT_RESET);
5455
5456         tmp |= SOFT_RESET_RLC;
5457         WREG32(GRBM_SOFT_RESET, tmp);
5458         udelay(50);
5459         tmp &= ~SOFT_RESET_RLC;
5460         WREG32(GRBM_SOFT_RESET, tmp);
5461         udelay(50);
5462 }
5463
5464 static void si_rlc_stop(struct radeon_device *rdev)
5465 {
5466         WREG32(RLC_CNTL, 0);
5467
5468         si_enable_gui_idle_interrupt(rdev, false);
5469
5470         si_wait_for_rlc_serdes(rdev);
5471 }
5472
5473 static void si_rlc_start(struct radeon_device *rdev)
5474 {
5475         WREG32(RLC_CNTL, RLC_ENABLE);
5476
5477         si_enable_gui_idle_interrupt(rdev, true);
5478
5479         udelay(50);
5480 }
5481
5482 static bool si_lbpw_supported(struct radeon_device *rdev)
5483 {
5484         u32 tmp;
5485
5486         /* Enable LBPW only for DDR3 */
5487         tmp = RREG32(MC_SEQ_MISC0);
5488         if ((tmp & 0xF0000000) == 0xB0000000)
5489                 return true;
5490         return false;
5491 }
5492
5493 static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5494 {
5495         u32 tmp;
5496
5497         tmp = RREG32(RLC_LB_CNTL);
5498         if (enable)
5499                 tmp |= LOAD_BALANCE_ENABLE;
5500         else
5501                 tmp &= ~LOAD_BALANCE_ENABLE;
5502         WREG32(RLC_LB_CNTL, tmp);
5503
5504         if (!enable) {
5505                 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5506                 WREG32(SPI_LB_CU_MASK, 0x00ff);
5507         }
5508 }
5509
5510 static int si_rlc_resume(struct radeon_device *rdev)
5511 {
5512         u32 i;
5513         const __be32 *fw_data;
5514
5515         if (!rdev->rlc_fw)
5516                 return -EINVAL;
5517
5518         si_rlc_stop(rdev);
5519
5520         si_rlc_reset(rdev);
5521
5522         si_init_pg(rdev);
5523
5524         si_init_cg(rdev);
5525
5526         WREG32(RLC_RL_BASE, 0);
5527         WREG32(RLC_RL_SIZE, 0);
5528         WREG32(RLC_LB_CNTL, 0);
5529         WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5530         WREG32(RLC_LB_CNTR_INIT, 0);
5531         WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5532
5533         WREG32(RLC_MC_CNTL, 0);
5534         WREG32(RLC_UCODE_CNTL, 0);
5535
5536         fw_data = (const __be32 *)rdev->rlc_fw->data;
5537         for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5538                 WREG32(RLC_UCODE_ADDR, i);
5539                 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5540         }
5541         WREG32(RLC_UCODE_ADDR, 0);
5542
5543         si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5544
5545         si_rlc_start(rdev);
5546
5547         return 0;
5548 }
5549
5550 static void si_enable_interrupts(struct radeon_device *rdev)
5551 {
5552         u32 ih_cntl = RREG32(IH_CNTL);
5553         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5554
5555         ih_cntl |= ENABLE_INTR;
5556         ih_rb_cntl |= IH_RB_ENABLE;
5557         WREG32(IH_CNTL, ih_cntl);
5558         WREG32(IH_RB_CNTL, ih_rb_cntl);
5559         rdev->ih.enabled = true;
5560 }
5561
5562 static void si_disable_interrupts(struct radeon_device *rdev)
5563 {
5564         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5565         u32 ih_cntl = RREG32(IH_CNTL);
5566
5567         ih_rb_cntl &= ~IH_RB_ENABLE;
5568         ih_cntl &= ~ENABLE_INTR;
5569         WREG32(IH_RB_CNTL, ih_rb_cntl);
5570         WREG32(IH_CNTL, ih_cntl);
5571         /* set rptr, wptr to 0 */
5572         WREG32(IH_RB_RPTR, 0);
5573         WREG32(IH_RB_WPTR, 0);
5574         rdev->ih.enabled = false;
5575         rdev->ih.rptr = 0;
5576 }
5577
5578 static void si_disable_interrupt_state(struct radeon_device *rdev)
5579 {
5580         u32 tmp;
5581
5582         tmp = RREG32(CP_INT_CNTL_RING0) &
5583                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5584         WREG32(CP_INT_CNTL_RING0, tmp);
5585         WREG32(CP_INT_CNTL_RING1, 0);
5586         WREG32(CP_INT_CNTL_RING2, 0);
5587         tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5588         WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5589         tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5590         WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
5591         WREG32(GRBM_INT_CNTL, 0);
5592         if (rdev->num_crtc >= 2) {
5593                 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5594                 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5595         }
5596         if (rdev->num_crtc >= 4) {
5597                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5598                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5599         }
5600         if (rdev->num_crtc >= 6) {
5601                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5602                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5603         }
5604
5605         if (rdev->num_crtc >= 2) {
5606                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5607                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5608         }
5609         if (rdev->num_crtc >= 4) {
5610                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5611                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5612         }
5613         if (rdev->num_crtc >= 6) {
5614                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5615                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5616         }
5617
5618         if (!ASIC_IS_NODCE(rdev)) {
5619                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
5620
5621                 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5622                 WREG32(DC_HPD1_INT_CONTROL, tmp);
5623                 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5624                 WREG32(DC_HPD2_INT_CONTROL, tmp);
5625                 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5626                 WREG32(DC_HPD3_INT_CONTROL, tmp);
5627                 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5628                 WREG32(DC_HPD4_INT_CONTROL, tmp);
5629                 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5630                 WREG32(DC_HPD5_INT_CONTROL, tmp);
5631                 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5632                 WREG32(DC_HPD6_INT_CONTROL, tmp);
5633         }
5634 }
5635
5636 static int si_irq_init(struct radeon_device *rdev)
5637 {
5638         int ret = 0;
5639         int rb_bufsz;
5640         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5641
5642         /* allocate ring */
5643         ret = r600_ih_ring_alloc(rdev);
5644         if (ret)
5645                 return ret;
5646
5647         /* disable irqs */
5648         si_disable_interrupts(rdev);
5649
5650         /* init rlc */
5651         ret = si_rlc_resume(rdev);
5652         if (ret) {
5653                 r600_ih_ring_fini(rdev);
5654                 return ret;
5655         }
5656
5657         /* setup interrupt control */
5658         /* set dummy read address to ring address */
5659         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5660         interrupt_cntl = RREG32(INTERRUPT_CNTL);
5661         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5662          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5663          */
5664         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5665         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5666         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5667         WREG32(INTERRUPT_CNTL, interrupt_cntl);
5668
5669         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5670         rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
5671
5672         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5673                       IH_WPTR_OVERFLOW_CLEAR |
5674                       (rb_bufsz << 1));
5675
5676         if (rdev->wb.enabled)
5677                 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5678
5679         /* set the writeback address whether it's enabled or not */
5680         WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5681         WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5682
5683         WREG32(IH_RB_CNTL, ih_rb_cntl);
5684
5685         /* set rptr, wptr to 0 */
5686         WREG32(IH_RB_RPTR, 0);
5687         WREG32(IH_RB_WPTR, 0);
5688
5689         /* Default settings for IH_CNTL (disabled at first) */
5690         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5691         /* RPTR_REARM only works if msi's are enabled */
5692         if (rdev->msi_enabled)
5693                 ih_cntl |= RPTR_REARM;
5694         WREG32(IH_CNTL, ih_cntl);
5695
5696         /* force the active interrupt state to all disabled */
5697         si_disable_interrupt_state(rdev);
5698
5699         pci_set_master(rdev->pdev);
5700
5701         /* enable irqs */
5702         si_enable_interrupts(rdev);
5703
5704         return ret;
5705 }
5706
5707 int si_irq_set(struct radeon_device *rdev)
5708 {
5709         u32 cp_int_cntl;
5710         u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
5711         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5712         u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
5713         u32 grbm_int_cntl = 0;
5714         u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
5715         u32 dma_cntl, dma_cntl1;
5716         u32 thermal_int = 0;
5717
5718         if (!rdev->irq.installed) {
5719                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5720                 return -EINVAL;
5721         }
5722         /* don't enable anything if the ih is disabled */
5723         if (!rdev->ih.enabled) {
5724                 si_disable_interrupts(rdev);
5725                 /* force the active interrupt state to all disabled */
5726                 si_disable_interrupt_state(rdev);
5727                 return 0;
5728         }
5729
5730         cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
5731                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5732
5733         if (!ASIC_IS_NODCE(rdev)) {
5734                 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
5735                 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
5736                 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
5737                 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
5738                 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
5739                 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
5740         }
5741
5742         dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5743         dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5744
5745         thermal_int = RREG32(CG_THERMAL_INT) &
5746                 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
5747
5748         /* enable CP interrupts on all rings */
5749         if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
5750                 DRM_DEBUG("si_irq_set: sw int gfx\n");
5751                 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
5752         }
5753         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
5754                 DRM_DEBUG("si_irq_set: sw int cp1\n");
5755                 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
5756         }
5757         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
5758                 DRM_DEBUG("si_irq_set: sw int cp2\n");
5759                 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
5760         }
5761         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
5762                 DRM_DEBUG("si_irq_set: sw int dma\n");
5763                 dma_cntl |= TRAP_ENABLE;
5764         }
5765
5766         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
5767                 DRM_DEBUG("si_irq_set: sw int dma1\n");
5768                 dma_cntl1 |= TRAP_ENABLE;
5769         }
5770         if (rdev->irq.crtc_vblank_int[0] ||
5771             atomic_read(&rdev->irq.pflip[0])) {
5772                 DRM_DEBUG("si_irq_set: vblank 0\n");
5773                 crtc1 |= VBLANK_INT_MASK;
5774         }
5775         if (rdev->irq.crtc_vblank_int[1] ||
5776             atomic_read(&rdev->irq.pflip[1])) {
5777                 DRM_DEBUG("si_irq_set: vblank 1\n");
5778                 crtc2 |= VBLANK_INT_MASK;
5779         }
5780         if (rdev->irq.crtc_vblank_int[2] ||
5781             atomic_read(&rdev->irq.pflip[2])) {
5782                 DRM_DEBUG("si_irq_set: vblank 2\n");
5783                 crtc3 |= VBLANK_INT_MASK;
5784         }
5785         if (rdev->irq.crtc_vblank_int[3] ||
5786             atomic_read(&rdev->irq.pflip[3])) {
5787                 DRM_DEBUG("si_irq_set: vblank 3\n");
5788                 crtc4 |= VBLANK_INT_MASK;
5789         }
5790         if (rdev->irq.crtc_vblank_int[4] ||
5791             atomic_read(&rdev->irq.pflip[4])) {
5792                 DRM_DEBUG("si_irq_set: vblank 4\n");
5793                 crtc5 |= VBLANK_INT_MASK;
5794         }
5795         if (rdev->irq.crtc_vblank_int[5] ||
5796             atomic_read(&rdev->irq.pflip[5])) {
5797                 DRM_DEBUG("si_irq_set: vblank 5\n");
5798                 crtc6 |= VBLANK_INT_MASK;
5799         }
5800         if (rdev->irq.hpd[0]) {
5801                 DRM_DEBUG("si_irq_set: hpd 1\n");
5802                 hpd1 |= DC_HPDx_INT_EN;
5803         }
5804         if (rdev->irq.hpd[1]) {
5805                 DRM_DEBUG("si_irq_set: hpd 2\n");
5806                 hpd2 |= DC_HPDx_INT_EN;
5807         }
5808         if (rdev->irq.hpd[2]) {
5809                 DRM_DEBUG("si_irq_set: hpd 3\n");
5810                 hpd3 |= DC_HPDx_INT_EN;
5811         }
5812         if (rdev->irq.hpd[3]) {
5813                 DRM_DEBUG("si_irq_set: hpd 4\n");
5814                 hpd4 |= DC_HPDx_INT_EN;
5815         }
5816         if (rdev->irq.hpd[4]) {
5817                 DRM_DEBUG("si_irq_set: hpd 5\n");
5818                 hpd5 |= DC_HPDx_INT_EN;
5819         }
5820         if (rdev->irq.hpd[5]) {
5821                 DRM_DEBUG("si_irq_set: hpd 6\n");
5822                 hpd6 |= DC_HPDx_INT_EN;
5823         }
5824
5825         WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
5826         WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
5827         WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
5828
5829         WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
5830         WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
5831
5832         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
5833
5834         if (rdev->irq.dpm_thermal) {
5835                 DRM_DEBUG("dpm thermal\n");
5836                 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
5837         }
5838
5839         if (rdev->num_crtc >= 2) {
5840                 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
5841                 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
5842         }
5843         if (rdev->num_crtc >= 4) {
5844                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
5845                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
5846         }
5847         if (rdev->num_crtc >= 6) {
5848                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
5849                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
5850         }
5851
5852         if (rdev->num_crtc >= 2) {
5853                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
5854                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
5855         }
5856         if (rdev->num_crtc >= 4) {
5857                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
5858                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
5859         }
5860         if (rdev->num_crtc >= 6) {
5861                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
5862                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
5863         }
5864
5865         if (!ASIC_IS_NODCE(rdev)) {
5866                 WREG32(DC_HPD1_INT_CONTROL, hpd1);
5867                 WREG32(DC_HPD2_INT_CONTROL, hpd2);
5868                 WREG32(DC_HPD3_INT_CONTROL, hpd3);
5869                 WREG32(DC_HPD4_INT_CONTROL, hpd4);
5870                 WREG32(DC_HPD5_INT_CONTROL, hpd5);
5871                 WREG32(DC_HPD6_INT_CONTROL, hpd6);
5872         }
5873
5874         WREG32(CG_THERMAL_INT, thermal_int);
5875
5876         return 0;
5877 }
5878
5879 static inline void si_irq_ack(struct radeon_device *rdev)
5880 {
5881         u32 tmp;
5882
5883         if (ASIC_IS_NODCE(rdev))
5884                 return;
5885
5886         rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
5887         rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
5888         rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
5889         rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
5890         rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
5891         rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
5892         rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
5893         rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
5894         if (rdev->num_crtc >= 4) {
5895                 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
5896                 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
5897         }
5898         if (rdev->num_crtc >= 6) {
5899                 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
5900                 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
5901         }
5902
5903         if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
5904                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5905         if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
5906                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5907         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
5908                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
5909         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
5910                 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
5911         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
5912                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
5913         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
5914                 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
5915
5916         if (rdev->num_crtc >= 4) {
5917                 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
5918                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5919                 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
5920                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5921                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
5922                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
5923                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
5924                         WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
5925                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
5926                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
5927                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
5928                         WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
5929         }
5930
5931         if (rdev->num_crtc >= 6) {
5932                 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
5933                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5934                 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
5935                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5936                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
5937                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
5938                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
5939                         WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
5940                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
5941                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
5942                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
5943                         WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
5944         }
5945
5946         if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
5947                 tmp = RREG32(DC_HPD1_INT_CONTROL);
5948                 tmp |= DC_HPDx_INT_ACK;
5949                 WREG32(DC_HPD1_INT_CONTROL, tmp);
5950         }
5951         if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
5952                 tmp = RREG32(DC_HPD2_INT_CONTROL);
5953                 tmp |= DC_HPDx_INT_ACK;
5954                 WREG32(DC_HPD2_INT_CONTROL, tmp);
5955         }
5956         if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5957                 tmp = RREG32(DC_HPD3_INT_CONTROL);
5958                 tmp |= DC_HPDx_INT_ACK;
5959                 WREG32(DC_HPD3_INT_CONTROL, tmp);
5960         }
5961         if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5962                 tmp = RREG32(DC_HPD4_INT_CONTROL);
5963                 tmp |= DC_HPDx_INT_ACK;
5964                 WREG32(DC_HPD4_INT_CONTROL, tmp);
5965         }
5966         if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5967                 tmp = RREG32(DC_HPD5_INT_CONTROL);
5968                 tmp |= DC_HPDx_INT_ACK;
5969                 WREG32(DC_HPD5_INT_CONTROL, tmp);
5970         }
5971         if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5972                 tmp = RREG32(DC_HPD5_INT_CONTROL);
5973                 tmp |= DC_HPDx_INT_ACK;
5974                 WREG32(DC_HPD6_INT_CONTROL, tmp);
5975         }
5976 }
5977
5978 static void si_irq_disable(struct radeon_device *rdev)
5979 {
5980         si_disable_interrupts(rdev);
5981         /* Wait and acknowledge irq */
5982         mdelay(1);
5983         si_irq_ack(rdev);
5984         si_disable_interrupt_state(rdev);
5985 }
5986
5987 static void si_irq_suspend(struct radeon_device *rdev)
5988 {
5989         si_irq_disable(rdev);
5990         si_rlc_stop(rdev);
5991 }
5992
5993 static void si_irq_fini(struct radeon_device *rdev)
5994 {
5995         si_irq_suspend(rdev);
5996         r600_ih_ring_fini(rdev);
5997 }
5998
5999 static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6000 {
6001         u32 wptr, tmp;
6002
6003         if (rdev->wb.enabled)
6004                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6005         else
6006                 wptr = RREG32(IH_RB_WPTR);
6007
6008         if (wptr & RB_OVERFLOW) {
6009                 /* When a ring buffer overflow happen start parsing interrupt
6010                  * from the last not overwritten vector (wptr + 16). Hopefully
6011                  * this should allow us to catchup.
6012                  */
6013                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6014                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6015                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6016                 tmp = RREG32(IH_RB_CNTL);
6017                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6018                 WREG32(IH_RB_CNTL, tmp);
6019         }
6020         return (wptr & rdev->ih.ptr_mask);
6021 }
6022
6023 /*        SI IV Ring
6024  * Each IV ring entry is 128 bits:
6025  * [7:0]    - interrupt source id
6026  * [31:8]   - reserved
6027  * [59:32]  - interrupt source data
6028  * [63:60]  - reserved
6029  * [71:64]  - RINGID
6030  * [79:72]  - VMID
6031  * [127:80] - reserved
6032  */
6033 int si_irq_process(struct radeon_device *rdev)
6034 {
6035         u32 wptr;
6036         u32 rptr;
6037         u32 src_id, src_data, ring_id;
6038         u32 ring_index;
6039         bool queue_hotplug = false;
6040         bool queue_thermal = false;
6041         u32 status, addr;
6042
6043         if (!rdev->ih.enabled || rdev->shutdown)
6044                 return IRQ_NONE;
6045
6046         wptr = si_get_ih_wptr(rdev);
6047
6048 restart_ih:
6049         /* is somebody else already processing irqs? */
6050         if (atomic_xchg(&rdev->ih.lock, 1))
6051                 return IRQ_NONE;
6052
6053         rptr = rdev->ih.rptr;
6054         DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6055
6056         /* Order reading of wptr vs. reading of IH ring data */
6057         rmb();
6058
6059         /* display interrupts */
6060         si_irq_ack(rdev);
6061
6062         while (rptr != wptr) {
6063                 /* wptr/rptr are in bytes! */
6064                 ring_index = rptr / 4;
6065                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6066                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6067                 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6068
6069                 switch (src_id) {
6070                 case 1: /* D1 vblank/vline */
6071                         switch (src_data) {
6072                         case 0: /* D1 vblank */
6073                                 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6074                                         if (rdev->irq.crtc_vblank_int[0]) {
6075                                                 drm_handle_vblank(rdev->ddev, 0);
6076                                                 rdev->pm.vblank_sync = true;
6077                                                 wake_up(&rdev->irq.vblank_queue);
6078                                         }
6079                                         if (atomic_read(&rdev->irq.pflip[0]))
6080                                                 radeon_crtc_handle_flip(rdev, 0);
6081                                         rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6082                                         DRM_DEBUG("IH: D1 vblank\n");
6083                                 }
6084                                 break;
6085                         case 1: /* D1 vline */
6086                                 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
6087                                         rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6088                                         DRM_DEBUG("IH: D1 vline\n");
6089                                 }
6090                                 break;
6091                         default:
6092                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6093                                 break;
6094                         }
6095                         break;
6096                 case 2: /* D2 vblank/vline */
6097                         switch (src_data) {
6098                         case 0: /* D2 vblank */
6099                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6100                                         if (rdev->irq.crtc_vblank_int[1]) {
6101                                                 drm_handle_vblank(rdev->ddev, 1);
6102                                                 rdev->pm.vblank_sync = true;
6103                                                 wake_up(&rdev->irq.vblank_queue);
6104                                         }
6105                                         if (atomic_read(&rdev->irq.pflip[1]))
6106                                                 radeon_crtc_handle_flip(rdev, 1);
6107                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6108                                         DRM_DEBUG("IH: D2 vblank\n");
6109                                 }
6110                                 break;
6111                         case 1: /* D2 vline */
6112                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6113                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6114                                         DRM_DEBUG("IH: D2 vline\n");
6115                                 }
6116                                 break;
6117                         default:
6118                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6119                                 break;
6120                         }
6121                         break;
6122                 case 3: /* D3 vblank/vline */
6123                         switch (src_data) {
6124                         case 0: /* D3 vblank */
6125                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6126                                         if (rdev->irq.crtc_vblank_int[2]) {
6127                                                 drm_handle_vblank(rdev->ddev, 2);
6128                                                 rdev->pm.vblank_sync = true;
6129                                                 wake_up(&rdev->irq.vblank_queue);
6130                                         }
6131                                         if (atomic_read(&rdev->irq.pflip[2]))
6132                                                 radeon_crtc_handle_flip(rdev, 2);
6133                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6134                                         DRM_DEBUG("IH: D3 vblank\n");
6135                                 }
6136                                 break;
6137                         case 1: /* D3 vline */
6138                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6139                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6140                                         DRM_DEBUG("IH: D3 vline\n");
6141                                 }
6142                                 break;
6143                         default:
6144                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6145                                 break;
6146                         }
6147                         break;
6148                 case 4: /* D4 vblank/vline */
6149                         switch (src_data) {
6150                         case 0: /* D4 vblank */
6151                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6152                                         if (rdev->irq.crtc_vblank_int[3]) {
6153                                                 drm_handle_vblank(rdev->ddev, 3);
6154                                                 rdev->pm.vblank_sync = true;
6155                                                 wake_up(&rdev->irq.vblank_queue);
6156                                         }
6157                                         if (atomic_read(&rdev->irq.pflip[3]))
6158                                                 radeon_crtc_handle_flip(rdev, 3);
6159                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6160                                         DRM_DEBUG("IH: D4 vblank\n");
6161                                 }
6162                                 break;
6163                         case 1: /* D4 vline */
6164                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6165                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6166                                         DRM_DEBUG("IH: D4 vline\n");
6167                                 }
6168                                 break;
6169                         default:
6170                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6171                                 break;
6172                         }
6173                         break;
6174                 case 5: /* D5 vblank/vline */
6175                         switch (src_data) {
6176                         case 0: /* D5 vblank */
6177                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6178                                         if (rdev->irq.crtc_vblank_int[4]) {
6179                                                 drm_handle_vblank(rdev->ddev, 4);
6180                                                 rdev->pm.vblank_sync = true;
6181                                                 wake_up(&rdev->irq.vblank_queue);
6182                                         }
6183                                         if (atomic_read(&rdev->irq.pflip[4]))
6184                                                 radeon_crtc_handle_flip(rdev, 4);
6185                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6186                                         DRM_DEBUG("IH: D5 vblank\n");
6187                                 }
6188                                 break;
6189                         case 1: /* D5 vline */
6190                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6191                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6192                                         DRM_DEBUG("IH: D5 vline\n");
6193                                 }
6194                                 break;
6195                         default:
6196                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6197                                 break;
6198                         }
6199                         break;
6200                 case 6: /* D6 vblank/vline */
6201                         switch (src_data) {
6202                         case 0: /* D6 vblank */
6203                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6204                                         if (rdev->irq.crtc_vblank_int[5]) {
6205                                                 drm_handle_vblank(rdev->ddev, 5);
6206                                                 rdev->pm.vblank_sync = true;
6207                                                 wake_up(&rdev->irq.vblank_queue);
6208                                         }
6209                                         if (atomic_read(&rdev->irq.pflip[5]))
6210                                                 radeon_crtc_handle_flip(rdev, 5);
6211                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6212                                         DRM_DEBUG("IH: D6 vblank\n");
6213                                 }
6214                                 break;
6215                         case 1: /* D6 vline */
6216                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6217                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6218                                         DRM_DEBUG("IH: D6 vline\n");
6219                                 }
6220                                 break;
6221                         default:
6222                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6223                                 break;
6224                         }
6225                         break;
6226                 case 42: /* HPD hotplug */
6227                         switch (src_data) {
6228                         case 0:
6229                                 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6230                                         rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6231                                         queue_hotplug = true;
6232                                         DRM_DEBUG("IH: HPD1\n");
6233                                 }
6234                                 break;
6235                         case 1:
6236                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6237                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6238                                         queue_hotplug = true;
6239                                         DRM_DEBUG("IH: HPD2\n");
6240                                 }
6241                                 break;
6242                         case 2:
6243                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6244                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6245                                         queue_hotplug = true;
6246                                         DRM_DEBUG("IH: HPD3\n");
6247                                 }
6248                                 break;
6249                         case 3:
6250                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6251                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6252                                         queue_hotplug = true;
6253                                         DRM_DEBUG("IH: HPD4\n");
6254                                 }
6255                                 break;
6256                         case 4:
6257                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6258                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6259                                         queue_hotplug = true;
6260                                         DRM_DEBUG("IH: HPD5\n");
6261                                 }
6262                                 break;
6263                         case 5:
6264                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6265                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6266                                         queue_hotplug = true;
6267                                         DRM_DEBUG("IH: HPD6\n");
6268                                 }
6269                                 break;
6270                         default:
6271                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6272                                 break;
6273                         }
6274                         break;
6275                 case 146:
6276                 case 147:
6277                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6278                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6279                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6280                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
6281                                 addr);
6282                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6283                                 status);
6284                         si_vm_decode_fault(rdev, status, addr);
6285                         /* reset addr and status */
6286                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6287                         break;
6288                 case 176: /* RINGID0 CP_INT */
6289                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6290                         break;
6291                 case 177: /* RINGID1 CP_INT */
6292                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6293                         break;
6294                 case 178: /* RINGID2 CP_INT */
6295                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6296                         break;
6297                 case 181: /* CP EOP event */
6298                         DRM_DEBUG("IH: CP EOP\n");
6299                         switch (ring_id) {
6300                         case 0:
6301                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6302                                 break;
6303                         case 1:
6304                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6305                                 break;
6306                         case 2:
6307                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6308                                 break;
6309                         }
6310                         break;
6311                 case 224: /* DMA trap event */
6312                         DRM_DEBUG("IH: DMA trap\n");
6313                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6314                         break;
6315                 case 230: /* thermal low to high */
6316                         DRM_DEBUG("IH: thermal low to high\n");
6317                         rdev->pm.dpm.thermal.high_to_low = false;
6318                         queue_thermal = true;
6319                         break;
6320                 case 231: /* thermal high to low */
6321                         DRM_DEBUG("IH: thermal high to low\n");
6322                         rdev->pm.dpm.thermal.high_to_low = true;
6323                         queue_thermal = true;
6324                         break;
6325                 case 233: /* GUI IDLE */
6326                         DRM_DEBUG("IH: GUI idle\n");
6327                         break;
6328                 case 244: /* DMA trap event */
6329                         DRM_DEBUG("IH: DMA1 trap\n");
6330                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6331                         break;
6332                 default:
6333                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6334                         break;
6335                 }
6336
6337                 /* wptr/rptr are in bytes! */
6338                 rptr += 16;
6339                 rptr &= rdev->ih.ptr_mask;
6340         }
6341         if (queue_hotplug)
6342                 schedule_work(&rdev->hotplug_work);
6343         if (queue_thermal && rdev->pm.dpm_enabled)
6344                 schedule_work(&rdev->pm.dpm.thermal.work);
6345         rdev->ih.rptr = rptr;
6346         WREG32(IH_RB_RPTR, rdev->ih.rptr);
6347         atomic_set(&rdev->ih.lock, 0);
6348
6349         /* make sure wptr hasn't changed while processing */
6350         wptr = si_get_ih_wptr(rdev);
6351         if (wptr != rptr)
6352                 goto restart_ih;
6353
6354         return IRQ_HANDLED;
6355 }
6356
6357 /*
6358  * startup/shutdown callbacks
6359  */
6360 static int si_startup(struct radeon_device *rdev)
6361 {
6362         struct radeon_ring *ring;
6363         int r;
6364
6365         /* enable pcie gen2/3 link */
6366         si_pcie_gen3_enable(rdev);
6367         /* enable aspm */
6368         si_program_aspm(rdev);
6369
6370         /* scratch needs to be initialized before MC */
6371         r = r600_vram_scratch_init(rdev);
6372         if (r)
6373                 return r;
6374
6375         si_mc_program(rdev);
6376
6377         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6378             !rdev->rlc_fw || !rdev->mc_fw) {
6379                 r = si_init_microcode(rdev);
6380                 if (r) {
6381                         DRM_ERROR("Failed to load firmware!\n");
6382                         return r;
6383                 }
6384         }
6385
6386         r = si_mc_load_microcode(rdev);
6387         if (r) {
6388                 DRM_ERROR("Failed to load MC firmware!\n");
6389                 return r;
6390         }
6391
6392         r = si_pcie_gart_enable(rdev);
6393         if (r)
6394                 return r;
6395         si_gpu_init(rdev);
6396
6397         /* allocate rlc buffers */
6398         if (rdev->family == CHIP_VERDE) {
6399                 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6400                 rdev->rlc.reg_list_size =
6401                         (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6402         }
6403         rdev->rlc.cs_data = si_cs_data;
6404         r = sumo_rlc_init(rdev);
6405         if (r) {
6406                 DRM_ERROR("Failed to init rlc BOs!\n");
6407                 return r;
6408         }
6409
6410         /* allocate wb buffer */
6411         r = radeon_wb_init(rdev);
6412         if (r)
6413                 return r;
6414
6415         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6416         if (r) {
6417                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6418                 return r;
6419         }
6420
6421         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6422         if (r) {
6423                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6424                 return r;
6425         }
6426
6427         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6428         if (r) {
6429                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6430                 return r;
6431         }
6432
6433         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6434         if (r) {
6435                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6436                 return r;
6437         }
6438
6439         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6440         if (r) {
6441                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6442                 return r;
6443         }
6444
6445         if (rdev->has_uvd) {
6446                 r = uvd_v2_2_resume(rdev);
6447                 if (!r) {
6448                         r = radeon_fence_driver_start_ring(rdev,
6449                                                            R600_RING_TYPE_UVD_INDEX);
6450                         if (r)
6451                                 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6452                 }
6453                 if (r)
6454                         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6455         }
6456
6457         /* Enable IRQ */
6458         if (!rdev->irq.installed) {
6459                 r = radeon_irq_kms_init(rdev);
6460                 if (r)
6461                         return r;
6462         }
6463
6464         r = si_irq_init(rdev);
6465         if (r) {
6466                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6467                 radeon_irq_kms_fini(rdev);
6468                 return r;
6469         }
6470         si_irq_set(rdev);
6471
6472         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6473         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6474                              CP_RB0_RPTR, CP_RB0_WPTR,
6475                              RADEON_CP_PACKET2);
6476         if (r)
6477                 return r;
6478
6479         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6480         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6481                              CP_RB1_RPTR, CP_RB1_WPTR,
6482                              RADEON_CP_PACKET2);
6483         if (r)
6484                 return r;
6485
6486         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6487         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6488                              CP_RB2_RPTR, CP_RB2_WPTR,
6489                              RADEON_CP_PACKET2);
6490         if (r)
6491                 return r;
6492
6493         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6494         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6495                              DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
6496                              DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
6497                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6498         if (r)
6499                 return r;
6500
6501         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6502         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6503                              DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
6504                              DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
6505                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6506         if (r)
6507                 return r;
6508
6509         r = si_cp_load_microcode(rdev);
6510         if (r)
6511                 return r;
6512         r = si_cp_resume(rdev);
6513         if (r)
6514                 return r;
6515
6516         r = cayman_dma_resume(rdev);
6517         if (r)
6518                 return r;
6519
6520         if (rdev->has_uvd) {
6521                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6522                 if (ring->ring_size) {
6523                         r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
6524                                              UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
6525                                              RADEON_CP_PACKET2);
6526                         if (!r)
6527                                 r = uvd_v1_0_init(rdev);
6528                         if (r)
6529                                 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6530                 }
6531         }
6532
6533         r = radeon_ib_pool_init(rdev);
6534         if (r) {
6535                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
6536                 return r;
6537         }
6538
6539         r = radeon_vm_manager_init(rdev);
6540         if (r) {
6541                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
6542                 return r;
6543         }
6544
6545         r = dce6_audio_init(rdev);
6546         if (r)
6547                 return r;
6548
6549         return 0;
6550 }
6551
6552 int si_resume(struct radeon_device *rdev)
6553 {
6554         int r;
6555
6556         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6557          * posting will perform necessary task to bring back GPU into good
6558          * shape.
6559          */
6560         /* post card */
6561         atom_asic_init(rdev->mode_info.atom_context);
6562
6563         /* init golden registers */
6564         si_init_golden_registers(rdev);
6565
6566         rdev->accel_working = true;
6567         r = si_startup(rdev);
6568         if (r) {
6569                 DRM_ERROR("si startup failed on resume\n");
6570                 rdev->accel_working = false;
6571                 return r;
6572         }
6573
6574         return r;
6575
6576 }
6577
6578 int si_suspend(struct radeon_device *rdev)
6579 {
6580         dce6_audio_fini(rdev);
6581         radeon_vm_manager_fini(rdev);
6582         si_cp_enable(rdev, false);
6583         cayman_dma_stop(rdev);
6584         if (rdev->has_uvd) {
6585                 uvd_v1_0_fini(rdev);
6586                 radeon_uvd_suspend(rdev);
6587         }
6588         si_fini_pg(rdev);
6589         si_fini_cg(rdev);
6590         si_irq_suspend(rdev);
6591         radeon_wb_disable(rdev);
6592         si_pcie_gart_disable(rdev);
6593         return 0;
6594 }
6595
6596 /* Plan is to move initialization in that function and use
6597  * helper function so that radeon_device_init pretty much
6598  * do nothing more than calling asic specific function. This
6599  * should also allow to remove a bunch of callback function
6600  * like vram_info.
6601  */
6602 int si_init(struct radeon_device *rdev)
6603 {
6604         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6605         int r;
6606
6607         /* Read BIOS */
6608         if (!radeon_get_bios(rdev)) {
6609                 if (ASIC_IS_AVIVO(rdev))
6610                         return -EINVAL;
6611         }
6612         /* Must be an ATOMBIOS */
6613         if (!rdev->is_atom_bios) {
6614                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6615                 return -EINVAL;
6616         }
6617         r = radeon_atombios_init(rdev);
6618         if (r)
6619                 return r;
6620
6621         /* Post card if necessary */
6622         if (!radeon_card_posted(rdev)) {
6623                 if (!rdev->bios) {
6624                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6625                         return -EINVAL;
6626                 }
6627                 DRM_INFO("GPU not posted. posting now...\n");
6628                 atom_asic_init(rdev->mode_info.atom_context);
6629         }
6630         /* init golden registers */
6631         si_init_golden_registers(rdev);
6632         /* Initialize scratch registers */
6633         si_scratch_init(rdev);
6634         /* Initialize surface registers */
6635         radeon_surface_init(rdev);
6636         /* Initialize clocks */
6637         radeon_get_clock_info(rdev->ddev);
6638
6639         /* Fence driver */
6640         r = radeon_fence_driver_init(rdev);
6641         if (r)
6642                 return r;
6643
6644         /* initialize memory controller */
6645         r = si_mc_init(rdev);
6646         if (r)
6647                 return r;
6648         /* Memory manager */
6649         r = radeon_bo_init(rdev);
6650         if (r)
6651                 return r;
6652
6653         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6654         ring->ring_obj = NULL;
6655         r600_ring_init(rdev, ring, 1024 * 1024);
6656
6657         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6658         ring->ring_obj = NULL;
6659         r600_ring_init(rdev, ring, 1024 * 1024);
6660
6661         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6662         ring->ring_obj = NULL;
6663         r600_ring_init(rdev, ring, 1024 * 1024);
6664
6665         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6666         ring->ring_obj = NULL;
6667         r600_ring_init(rdev, ring, 64 * 1024);
6668
6669         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6670         ring->ring_obj = NULL;
6671         r600_ring_init(rdev, ring, 64 * 1024);
6672
6673         if (rdev->has_uvd) {
6674                 r = radeon_uvd_init(rdev);
6675                 if (!r) {
6676                         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6677                         ring->ring_obj = NULL;
6678                         r600_ring_init(rdev, ring, 4096);
6679                 }
6680         }
6681
6682         rdev->ih.ring_obj = NULL;
6683         r600_ih_ring_init(rdev, 64 * 1024);
6684
6685         r = r600_pcie_gart_init(rdev);
6686         if (r)
6687                 return r;
6688
6689         rdev->accel_working = true;
6690         r = si_startup(rdev);
6691         if (r) {
6692                 dev_err(rdev->dev, "disabling GPU acceleration\n");
6693                 si_cp_fini(rdev);
6694                 cayman_dma_fini(rdev);
6695                 si_irq_fini(rdev);
6696                 sumo_rlc_fini(rdev);
6697                 radeon_wb_fini(rdev);
6698                 radeon_ib_pool_fini(rdev);
6699                 radeon_vm_manager_fini(rdev);
6700                 radeon_irq_kms_fini(rdev);
6701                 si_pcie_gart_fini(rdev);
6702                 rdev->accel_working = false;
6703         }
6704
6705         /* Don't start up if the MC ucode is missing.
6706          * The default clocks and voltages before the MC ucode
6707          * is loaded are not suffient for advanced operations.
6708          */
6709         if (!rdev->mc_fw) {
6710                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6711                 return -EINVAL;
6712         }
6713
6714         return 0;
6715 }
6716
6717 void si_fini(struct radeon_device *rdev)
6718 {
6719         si_cp_fini(rdev);
6720         cayman_dma_fini(rdev);
6721         si_fini_pg(rdev);
6722         si_fini_cg(rdev);
6723         si_irq_fini(rdev);
6724         sumo_rlc_fini(rdev);
6725         radeon_wb_fini(rdev);
6726         radeon_vm_manager_fini(rdev);
6727         radeon_ib_pool_fini(rdev);
6728         radeon_irq_kms_fini(rdev);
6729         if (rdev->has_uvd) {
6730                 uvd_v1_0_fini(rdev);
6731                 radeon_uvd_fini(rdev);
6732         }
6733         si_pcie_gart_fini(rdev);
6734         r600_vram_scratch_fini(rdev);
6735         radeon_gem_fini(rdev);
6736         radeon_fence_driver_fini(rdev);
6737         radeon_bo_fini(rdev);
6738         radeon_atombios_fini(rdev);
6739         kfree(rdev->bios);
6740         rdev->bios = NULL;
6741 }
6742
6743 /**
6744  * si_get_gpu_clock_counter - return GPU clock counter snapshot
6745  *
6746  * @rdev: radeon_device pointer
6747  *
6748  * Fetches a GPU clock counter snapshot (SI).
6749  * Returns the 64 bit clock counter snapshot.
6750  */
6751 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6752 {
6753         uint64_t clock;
6754
6755         mutex_lock(&rdev->gpu_clock_mutex);
6756         WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6757         clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6758                 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6759         mutex_unlock(&rdev->gpu_clock_mutex);
6760         return clock;
6761 }
6762
6763 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6764 {
6765         unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
6766         int r;
6767
6768         /* bypass vclk and dclk with bclk */
6769         WREG32_P(CG_UPLL_FUNC_CNTL_2,
6770                 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6771                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6772
6773         /* put PLL in bypass mode */
6774         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
6775
6776         if (!vclk || !dclk) {
6777                 /* keep the Bypass mode, put PLL to sleep */
6778                 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6779                 return 0;
6780         }
6781
6782         r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
6783                                           16384, 0x03FFFFFF, 0, 128, 5,
6784                                           &fb_div, &vclk_div, &dclk_div);
6785         if (r)
6786                 return r;
6787
6788         /* set RESET_ANTI_MUX to 0 */
6789         WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
6790
6791         /* set VCO_MODE to 1 */
6792         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
6793
6794         /* toggle UPLL_SLEEP to 1 then back to 0 */
6795         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6796         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
6797
6798         /* deassert UPLL_RESET */
6799         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6800
6801         mdelay(1);
6802
6803         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
6804         if (r)
6805                 return r;
6806
6807         /* assert UPLL_RESET again */
6808         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
6809
6810         /* disable spread spectrum. */
6811         WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
6812
6813         /* set feedback divider */
6814         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
6815
6816         /* set ref divider to 0 */
6817         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
6818
6819         if (fb_div < 307200)
6820                 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
6821         else
6822                 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
6823
6824         /* set PDIV_A and PDIV_B */
6825         WREG32_P(CG_UPLL_FUNC_CNTL_2,
6826                 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
6827                 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
6828
6829         /* give the PLL some time to settle */
6830         mdelay(15);
6831
6832         /* deassert PLL_RESET */
6833         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6834
6835         mdelay(15);
6836
6837         /* switch from bypass mode to normal mode */
6838         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
6839
6840         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
6841         if (r)
6842                 return r;
6843
6844         /* switch VCLK and DCLK selection */
6845         WREG32_P(CG_UPLL_FUNC_CNTL_2,
6846                 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
6847                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6848
6849         mdelay(100);
6850
6851         return 0;
6852 }
6853
6854 static void si_pcie_gen3_enable(struct radeon_device *rdev)
6855 {
6856         struct pci_dev *root = rdev->pdev->bus->self;
6857         int bridge_pos, gpu_pos;
6858         u32 speed_cntl, mask, current_data_rate;
6859         int ret, i;
6860         u16 tmp16;
6861
6862         if (radeon_pcie_gen2 == 0)
6863                 return;
6864
6865         if (rdev->flags & RADEON_IS_IGP)
6866                 return;
6867
6868         if (!(rdev->flags & RADEON_IS_PCIE))
6869                 return;
6870
6871         ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
6872         if (ret != 0)
6873                 return;
6874
6875         if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
6876                 return;
6877
6878         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6879         current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
6880                 LC_CURRENT_DATA_RATE_SHIFT;
6881         if (mask & DRM_PCIE_SPEED_80) {
6882                 if (current_data_rate == 2) {
6883                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
6884                         return;
6885                 }
6886                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
6887         } else if (mask & DRM_PCIE_SPEED_50) {
6888                 if (current_data_rate == 1) {
6889                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
6890                         return;
6891                 }
6892                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
6893         }
6894
6895         bridge_pos = pci_pcie_cap(root);
6896         if (!bridge_pos)
6897                 return;
6898
6899         gpu_pos = pci_pcie_cap(rdev->pdev);
6900         if (!gpu_pos)
6901                 return;
6902
6903         if (mask & DRM_PCIE_SPEED_80) {
6904                 /* re-try equalization if gen3 is not already enabled */
6905                 if (current_data_rate != 2) {
6906                         u16 bridge_cfg, gpu_cfg;
6907                         u16 bridge_cfg2, gpu_cfg2;
6908                         u32 max_lw, current_lw, tmp;
6909
6910                         pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6911                         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6912
6913                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
6914                         pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6915
6916                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
6917                         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6918
6919                         tmp = RREG32_PCIE(PCIE_LC_STATUS1);
6920                         max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
6921                         current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
6922
6923                         if (current_lw < max_lw) {
6924                                 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6925                                 if (tmp & LC_RENEGOTIATION_SUPPORT) {
6926                                         tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
6927                                         tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
6928                                         tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
6929                                         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
6930                                 }
6931                         }
6932
6933                         for (i = 0; i < 10; i++) {
6934                                 /* check status */
6935                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
6936                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
6937                                         break;
6938
6939                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6940                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6941
6942                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
6943                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
6944
6945                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6946                                 tmp |= LC_SET_QUIESCE;
6947                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6948
6949                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6950                                 tmp |= LC_REDO_EQ;
6951                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6952
6953                                 mdelay(100);
6954
6955                                 /* linkctl */
6956                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
6957                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6958                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
6959                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6960
6961                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
6962                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6963                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
6964                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6965
6966                                 /* linkctl2 */
6967                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
6968                                 tmp16 &= ~((1 << 4) | (7 << 9));
6969                                 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
6970                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
6971
6972                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6973                                 tmp16 &= ~((1 << 4) | (7 << 9));
6974                                 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
6975                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6976
6977                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6978                                 tmp &= ~LC_SET_QUIESCE;
6979                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6980                         }
6981                 }
6982         }
6983
6984         /* set the link speed */
6985         speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
6986         speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
6987         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6988
6989         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6990         tmp16 &= ~0xf;
6991         if (mask & DRM_PCIE_SPEED_80)
6992                 tmp16 |= 3; /* gen3 */
6993         else if (mask & DRM_PCIE_SPEED_50)
6994                 tmp16 |= 2; /* gen2 */
6995         else
6996                 tmp16 |= 1; /* gen1 */
6997         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6998
6999         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7000         speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7001         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7002
7003         for (i = 0; i < rdev->usec_timeout; i++) {
7004                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7005                 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7006                         break;
7007                 udelay(1);
7008         }
7009 }
7010
7011 static void si_program_aspm(struct radeon_device *rdev)
7012 {
7013         u32 data, orig;
7014         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7015         bool disable_clkreq = false;
7016
7017         if (radeon_aspm == 0)
7018                 return;
7019
7020         if (!(rdev->flags & RADEON_IS_PCIE))
7021                 return;
7022
7023         orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7024         data &= ~LC_XMIT_N_FTS_MASK;
7025         data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7026         if (orig != data)
7027                 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7028
7029         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7030         data |= LC_GO_TO_RECOVERY;
7031         if (orig != data)
7032                 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7033
7034         orig = data = RREG32_PCIE(PCIE_P_CNTL);
7035         data |= P_IGNORE_EDB_ERR;
7036         if (orig != data)
7037                 WREG32_PCIE(PCIE_P_CNTL, data);
7038
7039         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7040         data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7041         data |= LC_PMI_TO_L1_DIS;
7042         if (!disable_l0s)
7043                 data |= LC_L0S_INACTIVITY(7);
7044
7045         if (!disable_l1) {
7046                 data |= LC_L1_INACTIVITY(7);
7047                 data &= ~LC_PMI_TO_L1_DIS;
7048                 if (orig != data)
7049                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7050
7051                 if (!disable_plloff_in_l1) {
7052                         bool clk_req_support;
7053
7054                         orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7055                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7056                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7057                         if (orig != data)
7058                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7059
7060                         orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7061                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7062                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7063                         if (orig != data)
7064                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7065
7066                         orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7067                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7068                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7069                         if (orig != data)
7070                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7071
7072                         orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7073                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7074                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7075                         if (orig != data)
7076                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7077
7078                         if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7079                                 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7080                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7081                                 if (orig != data)
7082                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7083
7084                                 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7085                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7086                                 if (orig != data)
7087                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7088
7089                                 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7090                                 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7091                                 if (orig != data)
7092                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7093
7094                                 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7095                                 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7096                                 if (orig != data)
7097                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7098
7099                                 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7100                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7101                                 if (orig != data)
7102                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7103
7104                                 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7105                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7106                                 if (orig != data)
7107                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7108
7109                                 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7110                                 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7111                                 if (orig != data)
7112                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7113
7114                                 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7115                                 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7116                                 if (orig != data)
7117                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7118                         }
7119                         orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7120                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7121                         data |= LC_DYN_LANES_PWR_STATE(3);
7122                         if (orig != data)
7123                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7124
7125                         orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7126                         data &= ~LS2_EXIT_TIME_MASK;
7127                         if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7128                                 data |= LS2_EXIT_TIME(5);
7129                         if (orig != data)
7130                                 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7131
7132                         orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7133                         data &= ~LS2_EXIT_TIME_MASK;
7134                         if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7135                                 data |= LS2_EXIT_TIME(5);
7136                         if (orig != data)
7137                                 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7138
7139                         if (!disable_clkreq) {
7140                                 struct pci_dev *root = rdev->pdev->bus->self;
7141                                 u32 lnkcap;
7142
7143                                 clk_req_support = false;
7144                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7145                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7146                                         clk_req_support = true;
7147                         } else {
7148                                 clk_req_support = false;
7149                         }
7150
7151                         if (clk_req_support) {
7152                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7153                                 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7154                                 if (orig != data)
7155                                         WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7156
7157                                 orig = data = RREG32(THM_CLK_CNTL);
7158                                 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7159                                 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7160                                 if (orig != data)
7161                                         WREG32(THM_CLK_CNTL, data);
7162
7163                                 orig = data = RREG32(MISC_CLK_CNTL);
7164                                 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7165                                 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7166                                 if (orig != data)
7167                                         WREG32(MISC_CLK_CNTL, data);
7168
7169                                 orig = data = RREG32(CG_CLKPIN_CNTL);
7170                                 data &= ~BCLK_AS_XCLK;
7171                                 if (orig != data)
7172                                         WREG32(CG_CLKPIN_CNTL, data);
7173
7174                                 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7175                                 data &= ~FORCE_BIF_REFCLK_EN;
7176                                 if (orig != data)
7177                                         WREG32(CG_CLKPIN_CNTL_2, data);
7178
7179                                 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7180                                 data &= ~MPLL_CLKOUT_SEL_MASK;
7181                                 data |= MPLL_CLKOUT_SEL(4);
7182                                 if (orig != data)
7183                                         WREG32(MPLL_BYPASSCLK_SEL, data);
7184
7185                                 orig = data = RREG32(SPLL_CNTL_MODE);
7186                                 data &= ~SPLL_REFCLK_SEL_MASK;
7187                                 if (orig != data)
7188                                         WREG32(SPLL_CNTL_MODE, data);
7189                         }
7190                 }
7191         } else {
7192                 if (orig != data)
7193                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7194         }
7195
7196         orig = data = RREG32_PCIE(PCIE_CNTL2);
7197         data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7198         if (orig != data)
7199                 WREG32_PCIE(PCIE_CNTL2, data);
7200
7201         if (!disable_l0s) {
7202                 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7203                 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7204                         data = RREG32_PCIE(PCIE_LC_STATUS1);
7205                         if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7206                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7207                                 data &= ~LC_L0S_INACTIVITY_MASK;
7208                                 if (orig != data)
7209                                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7210                         }
7211                 }
7212         }
7213 }