drivers/gpu/drm/radeon/r600.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/seq_file.h>
  29 #include <linux/firmware.h>
  30 #include <linux/platform_device.h>
  31 #include "drmP.h"
  32 #include "radeon_drm.h"
  33 #include "radeon.h"
  34 #include "radeon_asic.h"
  35 #include "radeon_mode.h"
  36 #include "r600d.h"
  37 #include "atom.h"
  38 #include "avivod.h"
  39
  40 #define PFP_UCODE_SIZE 576
  41 #define PM4_UCODE_SIZE 1792
  42 #define RLC_UCODE_SIZE 768
  43 #define R700_PFP_UCODE_SIZE 848
  44 #define R700_PM4_UCODE_SIZE 1360
  45 #define R700_RLC_UCODE_SIZE 1024
  46 #define EVERGREEN_PFP_UCODE_SIZE 1120
  47 #define EVERGREEN_PM4_UCODE_SIZE 1376
  48
  49 /* Firmware Names */
  50 MODULE_FIRMWARE("radeon/R600_pfp.bin");
  51 MODULE_FIRMWARE("radeon/R600_me.bin");
  52 MODULE_FIRMWARE("radeon/RV610_pfp.bin");
  53 MODULE_FIRMWARE("radeon/RV610_me.bin");
  54 MODULE_FIRMWARE("radeon/RV630_pfp.bin");
  55 MODULE_FIRMWARE("radeon/RV630_me.bin");
  56 MODULE_FIRMWARE("radeon/RV620_pfp.bin");
  57 MODULE_FIRMWARE("radeon/RV620_me.bin");
  58 MODULE_FIRMWARE("radeon/RV635_pfp.bin");
  59 MODULE_FIRMWARE("radeon/RV635_me.bin");
  60 MODULE_FIRMWARE("radeon/RV670_pfp.bin");
  61 MODULE_FIRMWARE("radeon/RV670_me.bin");
  62 MODULE_FIRMWARE("radeon/RS780_pfp.bin");
  63 MODULE_FIRMWARE("radeon/RS780_me.bin");
  64 MODULE_FIRMWARE("radeon/RV770_pfp.bin");
  65 MODULE_FIRMWARE("radeon/RV770_me.bin");
  66 MODULE_FIRMWARE("radeon/RV730_pfp.bin");
  67 MODULE_FIRMWARE("radeon/RV730_me.bin");
  68 MODULE_FIRMWARE("radeon/RV710_pfp.bin");
  69 MODULE_FIRMWARE("radeon/RV710_me.bin");
  70 MODULE_FIRMWARE("radeon/R600_rlc.bin");
  71 MODULE_FIRMWARE("radeon/R700_rlc.bin");
  72 MODULE_FIRMWARE("radeon/CEDAR_pfp.bin");
  73 MODULE_FIRMWARE("radeon/CEDAR_me.bin");
  74 MODULE_FIRMWARE("radeon/REDWOOD_pfp.bin");
  75 MODULE_FIRMWARE("radeon/REDWOOD_me.bin");
  76 MODULE_FIRMWARE("radeon/JUNIPER_pfp.bin");
  77 MODULE_FIRMWARE("radeon/JUNIPER_me.bin");
  78 MODULE_FIRMWARE("radeon/CYRPESS_pfp.bin");
  79 MODULE_FIRMWARE("radeon/CYPRESS_me.bin");
  80
  81 int r600_debugfs_mc_info_init(struct radeon_device *rdev);
  82
  83 /* r600,rv610,rv630,rv620,rv635,rv670 */
  84 int r600_mc_wait_for_idle(struct radeon_device *rdev);
  85 void r600_gpu_init(struct radeon_device *rdev);
  86 void r600_fini(struct radeon_device *rdev);
  87
  88 /* hpd for digital panel detect/disconnect */
  89 bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
  90 {
  91         bool connected = false;
  92
  93         if (ASIC_IS_DCE3(rdev)) {
  94                 switch (hpd) {
  95                 case RADEON_HPD_1:
  96                         if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
  97                                 connected = true;
  98                         break;
  99                 case RADEON_HPD_2:
 100                         if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
 101                                 connected = true;
 102                         break;
 103                 case RADEON_HPD_3:
 104                         if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
 105                                 connected = true;
 106                         break;
 107                 case RADEON_HPD_4:
 108                         if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
 109                                 connected = true;
 110                         break;
 111                         /* DCE 3.2 */
 112                 case RADEON_HPD_5:
 113                         if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
 114                                 connected = true;
 115                         break;
 116                 case RADEON_HPD_6:
 117                         if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
 118                                 connected = true;
 119                         break;
 120                 default:
 121                         break;
 122                 }
 123         } else {
 124                 switch (hpd) {
 125                 case RADEON_HPD_1:
 126                         if (RREG32(DC_HOT_PLUG_DETECT1_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 127                                 connected = true;
 128                         break;
 129                 case RADEON_HPD_2:
 130                         if (RREG32(DC_HOT_PLUG_DETECT2_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 131                                 connected = true;
 132                         break;
 133                 case RADEON_HPD_3:
 134                         if (RREG32(DC_HOT_PLUG_DETECT3_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 135                                 connected = true;
 136                         break;
 137                 default:
 138                         break;
 139                 }
 140         }
 141         return connected;
 142 }
 143
 144 void r600_hpd_set_polarity(struct radeon_device *rdev,
 145                            enum radeon_hpd_id hpd)
 146 {
 147         u32 tmp;
 148         bool connected = r600_hpd_sense(rdev, hpd);
 149
 150         if (ASIC_IS_DCE3(rdev)) {
 151                 switch (hpd) {
 152                 case RADEON_HPD_1:
 153                         tmp = RREG32(DC_HPD1_INT_CONTROL);
 154                         if (connected)
 155                                 tmp &= ~DC_HPDx_INT_POLARITY;
 156                         else
 157                                 tmp |= DC_HPDx_INT_POLARITY;
 158                         WREG32(DC_HPD1_INT_CONTROL, tmp);
 159                         break;
 160                 case RADEON_HPD_2:
 161                         tmp = RREG32(DC_HPD2_INT_CONTROL);
 162                         if (connected)
 163                                 tmp &= ~DC_HPDx_INT_POLARITY;
 164                         else
 165                                 tmp |= DC_HPDx_INT_POLARITY;
 166                         WREG32(DC_HPD2_INT_CONTROL, tmp);
 167                         break;
 168                 case RADEON_HPD_3:
 169                         tmp = RREG32(DC_HPD3_INT_CONTROL);
 170                         if (connected)
 171                                 tmp &= ~DC_HPDx_INT_POLARITY;
 172                         else
 173                                 tmp |= DC_HPDx_INT_POLARITY;
 174                         WREG32(DC_HPD3_INT_CONTROL, tmp);
 175                         break;
 176                 case RADEON_HPD_4:
 177                         tmp = RREG32(DC_HPD4_INT_CONTROL);
 178                         if (connected)
 179                                 tmp &= ~DC_HPDx_INT_POLARITY;
 180                         else
 181                                 tmp |= DC_HPDx_INT_POLARITY;
 182                         WREG32(DC_HPD4_INT_CONTROL, tmp);
 183                         break;
 184                 case RADEON_HPD_5:
 185                         tmp = RREG32(DC_HPD5_INT_CONTROL);
 186                         if (connected)
 187                                 tmp &= ~DC_HPDx_INT_POLARITY;
 188                         else
 189                                 tmp |= DC_HPDx_INT_POLARITY;
 190                         WREG32(DC_HPD5_INT_CONTROL, tmp);
 191                         break;
 192                         /* DCE 3.2 */
 193                 case RADEON_HPD_6:
 194                         tmp = RREG32(DC_HPD6_INT_CONTROL);
 195                         if (connected)
 196                                 tmp &= ~DC_HPDx_INT_POLARITY;
 197                         else
 198                                 tmp |= DC_HPDx_INT_POLARITY;
 199                         WREG32(DC_HPD6_INT_CONTROL, tmp);
 200                         break;
 201                 default:
 202                         break;
 203                 }
 204         } else {
 205                 switch (hpd) {
 206                 case RADEON_HPD_1:
 207                         tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
 208                         if (connected)
 209                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 210                         else
 211                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 212                         WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
 213                         break;
 214                 case RADEON_HPD_2:
 215                         tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
 216                         if (connected)
 217                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 218                         else
 219                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 220                         WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
 221                         break;
 222                 case RADEON_HPD_3:
 223                         tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
 224                         if (connected)
 225                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 226                         else
 227                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 228                         WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
 229                         break;
 230                 default:
 231                         break;
 232                 }
 233         }
 234 }
 235
 236 void r600_hpd_init(struct radeon_device *rdev)
 237 {
 238         struct drm_device *dev = rdev->ddev;
 239         struct drm_connector *connector;
 240
 241         if (ASIC_IS_DCE3(rdev)) {
 242                 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) | DC_HPDx_RX_INT_TIMER(0xfa);
 243                 if (ASIC_IS_DCE32(rdev))
 244                         tmp |= DC_HPDx_EN;
 245
 246                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 247                         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 248                         switch (radeon_connector->hpd.hpd) {
 249                         case RADEON_HPD_1:
 250                                 WREG32(DC_HPD1_CONTROL, tmp);
 251                                 rdev->irq.hpd[0] = true;
 252                                 break;
 253                         case RADEON_HPD_2:
 254                                 WREG32(DC_HPD2_CONTROL, tmp);
 255                                 rdev->irq.hpd[1] = true;
 256                                 break;
 257                         case RADEON_HPD_3:
 258                                 WREG32(DC_HPD3_CONTROL, tmp);
 259                                 rdev->irq.hpd[2] = true;
 260                                 break;
 261                         case RADEON_HPD_4:
 262                                 WREG32(DC_HPD4_CONTROL, tmp);
 263                                 rdev->irq.hpd[3] = true;
 264                                 break;
 265                                 /* DCE 3.2 */
 266                         case RADEON_HPD_5:
 267                                 WREG32(DC_HPD5_CONTROL, tmp);
 268                                 rdev->irq.hpd[4] = true;
 269                                 break;
 270                         case RADEON_HPD_6:
 271                                 WREG32(DC_HPD6_CONTROL, tmp);
 272                                 rdev->irq.hpd[5] = true;
 273                                 break;
 274                         default:
 275                                 break;
 276                         }
 277                 }
 278         } else {
 279                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 280                         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 281                         switch (radeon_connector->hpd.hpd) {
 282                         case RADEON_HPD_1:
 283                                 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 284                                 rdev->irq.hpd[0] = true;
 285                                 break;
 286                         case RADEON_HPD_2:
 287                                 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 288                                 rdev->irq.hpd[1] = true;
 289                                 break;
 290                         case RADEON_HPD_3:
 291                                 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 292                                 rdev->irq.hpd[2] = true;
 293                                 break;
 294                         default:
 295                                 break;
 296                         }
 297                 }
 298         }
 299         if (rdev->irq.installed)
 300                 r600_irq_set(rdev);
 301 }
 302
 303 void r600_hpd_fini(struct radeon_device *rdev)
 304 {
 305         struct drm_device *dev = rdev->ddev;
 306         struct drm_connector *connector;
 307
 308         if (ASIC_IS_DCE3(rdev)) {
 309                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 310                         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 311                         switch (radeon_connector->hpd.hpd) {
 312                         case RADEON_HPD_1:
 313                                 WREG32(DC_HPD1_CONTROL, 0);
 314                                 rdev->irq.hpd[0] = false;
 315                                 break;
 316                         case RADEON_HPD_2:
 317                                 WREG32(DC_HPD2_CONTROL, 0);
 318                                 rdev->irq.hpd[1] = false;
 319                                 break;
 320                         case RADEON_HPD_3:
 321                                 WREG32(DC_HPD3_CONTROL, 0);
 322                                 rdev->irq.hpd[2] = false;
 323                                 break;
 324                         case RADEON_HPD_4:
 325                                 WREG32(DC_HPD4_CONTROL, 0);
 326                                 rdev->irq.hpd[3] = false;
 327                                 break;
 328                                 /* DCE 3.2 */
 329                         case RADEON_HPD_5:
 330                                 WREG32(DC_HPD5_CONTROL, 0);
 331                                 rdev->irq.hpd[4] = false;
 332                                 break;
 333                         case RADEON_HPD_6:
 334                                 WREG32(DC_HPD6_CONTROL, 0);
 335                                 rdev->irq.hpd[5] = false;
 336                                 break;
 337                         default:
 338                                 break;
 339                         }
 340                 }
 341         } else {
 342                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 343                         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 344                         switch (radeon_connector->hpd.hpd) {
 345                         case RADEON_HPD_1:
 346                                 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, 0);
 347                                 rdev->irq.hpd[0] = false;
 348                                 break;
 349                         case RADEON_HPD_2:
 350                                 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, 0);
 351                                 rdev->irq.hpd[1] = false;
 352                                 break;
 353                         case RADEON_HPD_3:
 354                                 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, 0);
 355                                 rdev->irq.hpd[2] = false;
 356                                 break;
 357                         default:
 358                                 break;
 359                         }
 360                 }
 361         }
 362 }
 363
 364 /*
 365  * R600 PCIE GART
 366  */
 367 void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
 368 {
 369         unsigned i;
 370         u32 tmp;
 371
 372         /* flush hdp cache so updates hit vram */
 373         WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
 374
 375         WREG32(VM_CONTEXT0_INVALIDATION_LOW_ADDR, rdev->mc.gtt_start >> 12);
 376         WREG32(VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (rdev->mc.gtt_end - 1) >> 12);
 377         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
 378         for (i = 0; i < rdev->usec_timeout; i++) {
 379                 /* read MC_STATUS */
 380                 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
 381                 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
 382                 if (tmp == 2) {
 383                         printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
 384                         return;
 385                 }
 386                 if (tmp) {
 387                         return;
 388                 }
 389                 udelay(1);
 390         }
 391 }
 392
 393 int r600_pcie_gart_init(struct radeon_device *rdev)
 394 {
 395         int r;
 396
 397         if (rdev->gart.table.vram.robj) {
 398                 WARN(1, "R600 PCIE GART already initialized.\n");
 399                 return 0;
 400         }
 401         /* Initialize common gart structure */
 402         r = radeon_gart_init(rdev);
 403         if (r)
 404                 return r;
 405         rdev->gart.table_size = rdev->gart.num_gpu_pages * 8;
 406         return radeon_gart_table_vram_alloc(rdev);
 407 }
 408
 409 int r600_pcie_gart_enable(struct radeon_device *rdev)
 410 {
 411         u32 tmp;
 412         int r, i;
 413
 414         if (rdev->gart.table.vram.robj == NULL) {
 415                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
 416                 return -EINVAL;
 417         }
 418         r = radeon_gart_table_vram_pin(rdev);
 419         if (r)
 420                 return r;
 421         radeon_gart_restore(rdev);
 422
 423         /* Setup L2 cache */
 424         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 425                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 426                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 427         WREG32(VM_L2_CNTL2, 0);
 428         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 429         /* Setup TLB control */
 430         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 431                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 432                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
 433                 ENABLE_WAIT_L2_QUERY;
 434         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
 435         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
 436         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
 437         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
 438         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
 439         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
 440         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
 441         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
 442         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
 443         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
 444         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
 445         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
 446         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 447         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 448         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
 449         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
 450         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
 451         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
 452                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
 453         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
 454                         (u32)(rdev->dummy_page.addr >> 12));
 455         for (i = 1; i < 7; i++)
 456                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 457
 458         r600_pcie_gart_tlb_flush(rdev);
 459         rdev->gart.ready = true;
 460         return 0;
 461 }
 462
 463 void r600_pcie_gart_disable(struct radeon_device *rdev)
 464 {
 465         u32 tmp;
 466         int i, r;
 467
 468         /* Disable all tables */
 469         for (i = 0; i < 7; i++)
 470                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 471
 472         /* Disable L2 cache */
 473         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
 474                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 475         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 476         /* Setup L1 TLB control */
 477         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
 478                 ENABLE_WAIT_L2_QUERY;
 479         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
 480         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
 481         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
 482         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
 483         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
 484         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
 485         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
 486         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
 487         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp);
 488         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp);
 489         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
 490         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
 491         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp);
 492         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
 493         if (rdev->gart.table.vram.robj) {
 494                 r = radeon_bo_reserve(rdev->gart.table.vram.robj, false);
 495                 if (likely(r == 0)) {
 496                         radeon_bo_kunmap(rdev->gart.table.vram.robj);
 497                         radeon_bo_unpin(rdev->gart.table.vram.robj);
 498                         radeon_bo_unreserve(rdev->gart.table.vram.robj);
 499                 }
 500         }
 501 }
 502
 503 void r600_pcie_gart_fini(struct radeon_device *rdev)
 504 {
 505         radeon_gart_fini(rdev);
 506         r600_pcie_gart_disable(rdev);
 507         radeon_gart_table_vram_free(rdev);
 508 }
 509
 510 void r600_agp_enable(struct radeon_device *rdev)
 511 {
 512         u32 tmp;
 513         int i;
 514
 515         /* Setup L2 cache */
 516         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 517                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 518                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 519         WREG32(VM_L2_CNTL2, 0);
 520         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 521         /* Setup TLB control */
 522         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 523                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 524                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
 525                 ENABLE_WAIT_L2_QUERY;
 526         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
 527         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
 528         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
 529         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
 530         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
 531         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
 532         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
 533         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
 534         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
 535         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
 536         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
 537         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
 538         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 539         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 540         for (i = 0; i < 7; i++)
 541                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 542 }
 543
 544 int r600_mc_wait_for_idle(struct radeon_device *rdev)
 545 {
 546         unsigned i;
 547         u32 tmp;
 548
 549         for (i = 0; i < rdev->usec_timeout; i++) {
 550                 /* read MC_STATUS */
 551                 tmp = RREG32(R_000E50_SRBM_STATUS) & 0x3F00;
 552                 if (!tmp)
 553                         return 0;
 554                 udelay(1);
 555         }
 556         return -1;
 557 }
 558
 559 static void r600_mc_program(struct radeon_device *rdev)
 560 {
 561         struct rv515_mc_save save;
 562         u32 tmp;
 563         int i, j;
 564
 565         /* Initialize HDP */
 566         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
 567                 WREG32((0x2c14 + j), 0x00000000);
 568                 WREG32((0x2c18 + j), 0x00000000);
 569                 WREG32((0x2c1c + j), 0x00000000);
 570                 WREG32((0x2c20 + j), 0x00000000);
 571                 WREG32((0x2c24 + j), 0x00000000);
 572         }
 573         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
 574
 575         rv515_mc_stop(rdev, &save);
 576         if (r600_mc_wait_for_idle(rdev)) {
 577                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
 578         }
 579         /* Lockout access through VGA aperture (doesn't exist before R600) */
 580         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
 581         /* Update configuration */
 582         if (rdev->flags & RADEON_IS_AGP) {
 583                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
 584                         /* VRAM before AGP */
 585                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
 586                                 rdev->mc.vram_start >> 12);
 587                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
 588                                 rdev->mc.gtt_end >> 12);
 589                 } else {
 590                         /* VRAM after AGP */
 591                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
 592                                 rdev->mc.gtt_start >> 12);
 593                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
 594                                 rdev->mc.vram_end >> 12);
 595                 }
 596         } else {
 597                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, rdev->mc.vram_start >> 12);
 598                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, rdev->mc.vram_end >> 12);
 599         }
 600         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
 601         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
 602         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
 603         WREG32(MC_VM_FB_LOCATION, tmp);
 604         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
 605         WREG32(HDP_NONSURFACE_INFO, (2 << 7));
 606         WREG32(HDP_NONSURFACE_SIZE, rdev->mc.mc_vram_size | 0x3FF);
 607         if (rdev->flags & RADEON_IS_AGP) {
 608                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 22);
 609                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 22);
 610                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
 611         } else {
 612                 WREG32(MC_VM_AGP_BASE, 0);
 613                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
 614                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
 615         }
 616         if (r600_mc_wait_for_idle(rdev)) {
 617                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
 618         }
 619         rv515_mc_resume(rdev, &save);
 620         /* we need to own VRAM, so turn off the VGA renderer here
 621          * to stop it overwriting our objects */
 622         rv515_vga_render_disable(rdev);
 623 }
 624
 625 /**
 626  * r600_vram_gtt_location - try to find VRAM & GTT location
 627  * @rdev: radeon device structure holding all necessary informations
 628  * @mc: memory controller structure holding memory informations
 629  *
 630  * Function will place try to place VRAM at same place as in CPU (PCI)
 631  * address space as some GPU seems to have issue when we reprogram at
 632  * different address space.
 633  *
 634  * If there is not enough space to fit the unvisible VRAM after the
 635  * aperture then we limit the VRAM size to the aperture.
 636  *
 637  * If we are using AGP then place VRAM adjacent to AGP aperture are we need
 638  * them to be in one from GPU point of view so that we can program GPU to
 639  * catch access outside them (weird GPU policy see ??).
 640  *
 641  * This function will never fails, worst case are limiting VRAM or GTT.
 642  *
 643  * Note: GTT start, end, size should be initialized before calling this
 644  * function on AGP platform.
 645  */
 646 void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
 647 {
 648         u64 size_bf, size_af;
 649
 650         if (mc->mc_vram_size > 0xE0000000) {
 651                 /* leave room for at least 512M GTT */
 652                 dev_warn(rdev->dev, "limiting VRAM\n");
 653                 mc->real_vram_size = 0xE0000000;
 654                 mc->mc_vram_size = 0xE0000000;
 655         }
 656         if (rdev->flags & RADEON_IS_AGP) {
 657                 size_bf = mc->gtt_start;
 658                 size_af = 0xFFFFFFFF - mc->gtt_end + 1;
 659                 if (size_bf > size_af) {
 660                         if (mc->mc_vram_size > size_bf) {
 661                                 dev_warn(rdev->dev, "limiting VRAM\n");
 662                                 mc->real_vram_size = size_bf;
 663                                 mc->mc_vram_size = size_bf;
 664                         }
 665                         mc->vram_start = mc->gtt_start - mc->mc_vram_size;
 666                 } else {
 667                         if (mc->mc_vram_size > size_af) {
 668                                 dev_warn(rdev->dev, "limiting VRAM\n");
 669                                 mc->real_vram_size = size_af;
 670                                 mc->mc_vram_size = size_af;
 671                         }
 672                         mc->vram_start = mc->gtt_end;
 673                 }
 674                 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
 675                 dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
 676                                 mc->mc_vram_size >> 20, mc->vram_start,
 677                                 mc->vram_end, mc->real_vram_size >> 20);
 678         } else {
 679                 u64 base = 0;
 680                 if (rdev->flags & RADEON_IS_IGP)
 681                         base = (RREG32(MC_VM_FB_LOCATION) & 0xFFFF) << 24;
 682                 radeon_vram_location(rdev, &rdev->mc, base);
 683                 radeon_gtt_location(rdev, mc);
 684         }
 685 }
 686
 687 int r600_mc_init(struct radeon_device *rdev)
 688 {
 689         u32 tmp;
 690         int chansize, numchan;
 691
 692         /* Get VRAM informations */
 693         rdev->mc.vram_is_ddr = true;
 694         tmp = RREG32(RAMCFG);
 695         if (tmp & CHANSIZE_OVERRIDE) {
 696                 chansize = 16;
 697         } else if (tmp & CHANSIZE_MASK) {
 698                 chansize = 64;
 699         } else {
 700                 chansize = 32;
 701         }
 702         tmp = RREG32(CHMAP);
 703         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
 704         case 0:
 705         default:
 706                 numchan = 1;
 707                 break;
 708         case 1:
 709                 numchan = 2;
 710                 break;
 711         case 2:
 712                 numchan = 4;
 713                 break;
 714         case 3:
 715                 numchan = 8;
 716                 break;
 717         }
 718         rdev->mc.vram_width = numchan * chansize;
 719         /* Could aper size report 0 ? */
 720         rdev->mc.aper_base = drm_get_resource_start(rdev->ddev, 0);
 721         rdev->mc.aper_size = drm_get_resource_len(rdev->ddev, 0);
 722         /* Setup GPU memory space */
 723         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
 724         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
 725         rdev->mc.visible_vram_size = rdev->mc.aper_size;
 726         /* FIXME remove this once we support unmappable VRAM */
 727         if (rdev->mc.mc_vram_size > rdev->mc.aper_size) {
 728                 rdev->mc.mc_vram_size = rdev->mc.aper_size;
 729                 rdev->mc.real_vram_size = rdev->mc.aper_size;
 730         }
 731         r600_vram_gtt_location(rdev, &rdev->mc);
 732
 733         if (rdev->flags & RADEON_IS_IGP)
 734                 rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
 735         radeon_update_bandwidth_info(rdev);
 736         return 0;
 737 }
 738
 739 /* We doesn't check that the GPU really needs a reset we simply do the
 740  * reset, it's up to the caller to determine if the GPU needs one. We
 741  * might add an helper function to check that.
 742  */
 743 int r600_gpu_soft_reset(struct radeon_device *rdev)
 744 {
 745         struct rv515_mc_save save;
 746         u32 grbm_busy_mask = S_008010_VC_BUSY(1) | S_008010_VGT_BUSY_NO_DMA(1) |
 747                                 S_008010_VGT_BUSY(1) | S_008010_TA03_BUSY(1) |
 748                                 S_008010_TC_BUSY(1) | S_008010_SX_BUSY(1) |
 749                                 S_008010_SH_BUSY(1) | S_008010_SPI03_BUSY(1) |
 750                                 S_008010_SMX_BUSY(1) | S_008010_SC_BUSY(1) |
 751                                 S_008010_PA_BUSY(1) | S_008010_DB03_BUSY(1) |
 752                                 S_008010_CR_BUSY(1) | S_008010_CB03_BUSY(1) |
 753                                 S_008010_GUI_ACTIVE(1);
 754         u32 grbm2_busy_mask = S_008014_SPI0_BUSY(1) | S_008014_SPI1_BUSY(1) |
 755                         S_008014_SPI2_BUSY(1) | S_008014_SPI3_BUSY(1) |
 756                         S_008014_TA0_BUSY(1) | S_008014_TA1_BUSY(1) |
 757                         S_008014_TA2_BUSY(1) | S_008014_TA3_BUSY(1) |
 758                         S_008014_DB0_BUSY(1) | S_008014_DB1_BUSY(1) |
 759                         S_008014_DB2_BUSY(1) | S_008014_DB3_BUSY(1) |
 760                         S_008014_CB0_BUSY(1) | S_008014_CB1_BUSY(1) |
 761                         S_008014_CB2_BUSY(1) | S_008014_CB3_BUSY(1);
 762         u32 tmp;
 763
 764         dev_info(rdev->dev, "GPU softreset \n");
 765         dev_info(rdev->dev, "  R_008010_GRBM_STATUS=0x%08X\n",
 766                 RREG32(R_008010_GRBM_STATUS));
 767         dev_info(rdev->dev, "  R_008014_GRBM_STATUS2=0x%08X\n",
 768                 RREG32(R_008014_GRBM_STATUS2));
 769         dev_info(rdev->dev, "  R_000E50_SRBM_STATUS=0x%08X\n",
 770                 RREG32(R_000E50_SRBM_STATUS));
 771         rv515_mc_stop(rdev, &save);
 772         if (r600_mc_wait_for_idle(rdev)) {
 773                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
 774         }
 775         /* Disable CP parsing/prefetching */
 776         WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
 777         /* Check if any of the rendering block is busy and reset it */
 778         if ((RREG32(R_008010_GRBM_STATUS) & grbm_busy_mask) ||
 779             (RREG32(R_008014_GRBM_STATUS2) & grbm2_busy_mask)) {
 780                 tmp = S_008020_SOFT_RESET_CR(1) |
 781                         S_008020_SOFT_RESET_DB(1) |
 782                         S_008020_SOFT_RESET_CB(1) |
 783                         S_008020_SOFT_RESET_PA(1) |
 784                         S_008020_SOFT_RESET_SC(1) |
 785                         S_008020_SOFT_RESET_SMX(1) |
 786                         S_008020_SOFT_RESET_SPI(1) |
 787                         S_008020_SOFT_RESET_SX(1) |
 788                         S_008020_SOFT_RESET_SH(1) |
 789                         S_008020_SOFT_RESET_TC(1) |
 790                         S_008020_SOFT_RESET_TA(1) |
 791                         S_008020_SOFT_RESET_VC(1) |
 792                         S_008020_SOFT_RESET_VGT(1);
 793                 dev_info(rdev->dev, "  R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
 794                 WREG32(R_008020_GRBM_SOFT_RESET, tmp);
 795                 RREG32(R_008020_GRBM_SOFT_RESET);
 796                 mdelay(15);
 797                 WREG32(R_008020_GRBM_SOFT_RESET, 0);
 798         }
 799         /* Reset CP (we always reset CP) */
 800         tmp = S_008020_SOFT_RESET_CP(1);
 801         dev_info(rdev->dev, "R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
 802         WREG32(R_008020_GRBM_SOFT_RESET, tmp);
 803         RREG32(R_008020_GRBM_SOFT_RESET);
 804         mdelay(15);
 805         WREG32(R_008020_GRBM_SOFT_RESET, 0);
 806         /* Wait a little for things to settle down */
 807         mdelay(1);
 808         dev_info(rdev->dev, "  R_008010_GRBM_STATUS=0x%08X\n",
 809                 RREG32(R_008010_GRBM_STATUS));
 810         dev_info(rdev->dev, "  R_008014_GRBM_STATUS2=0x%08X\n",
 811                 RREG32(R_008014_GRBM_STATUS2));
 812         dev_info(rdev->dev, "  R_000E50_SRBM_STATUS=0x%08X\n",
 813                 RREG32(R_000E50_SRBM_STATUS));
 814         rv515_mc_resume(rdev, &save);
 815         return 0;
 816 }
 817
 818 bool r600_gpu_is_lockup(struct radeon_device *rdev)
 819 {
 820         u32 srbm_status;
 821         u32 grbm_status;
 822         u32 grbm_status2;
 823         int r;
 824
 825         srbm_status = RREG32(R_000E50_SRBM_STATUS);
 826         grbm_status = RREG32(R_008010_GRBM_STATUS);
 827         grbm_status2 = RREG32(R_008014_GRBM_STATUS2);
 828         if (!G_008010_GUI_ACTIVE(grbm_status)) {
 829                 r100_gpu_lockup_update(&rdev->config.r300.lockup, &rdev->cp);
 830                 return false;
 831         }
 832         /* force CP activities */
 833         r = radeon_ring_lock(rdev, 2);
 834         if (!r) {
 835                 /* PACKET2 NOP */
 836                 radeon_ring_write(rdev, 0x80000000);
 837                 radeon_ring_write(rdev, 0x80000000);
 838                 radeon_ring_unlock_commit(rdev);
 839         }
 840         rdev->cp.rptr = RREG32(R600_CP_RB_RPTR);
 841         return r100_gpu_cp_is_lockup(rdev, &rdev->config.r300.lockup, &rdev->cp);
 842 }
 843
 844 int r600_asic_reset(struct radeon_device *rdev)
 845 {
 846         return r600_gpu_soft_reset(rdev);
 847 }
 848
 849 static u32 r600_get_tile_pipe_to_backend_map(u32 num_tile_pipes,
 850                                              u32 num_backends,
 851                                              u32 backend_disable_mask)
 852 {
 853         u32 backend_map = 0;
 854         u32 enabled_backends_mask;
 855         u32 enabled_backends_count;
 856         u32 cur_pipe;
 857         u32 swizzle_pipe[R6XX_MAX_PIPES];
 858         u32 cur_backend;
 859         u32 i;
 860
 861         if (num_tile_pipes > R6XX_MAX_PIPES)
 862                 num_tile_pipes = R6XX_MAX_PIPES;
 863         if (num_tile_pipes < 1)
 864                 num_tile_pipes = 1;
 865         if (num_backends > R6XX_MAX_BACKENDS)
 866                 num_backends = R6XX_MAX_BACKENDS;
 867         if (num_backends < 1)
 868                 num_backends = 1;
 869
 870         enabled_backends_mask = 0;
 871         enabled_backends_count = 0;
 872         for (i = 0; i < R6XX_MAX_BACKENDS; ++i) {
 873                 if (((backend_disable_mask >> i) & 1) == 0) {
 874                         enabled_backends_mask |= (1 << i);
 875                         ++enabled_backends_count;
 876                 }
 877                 if (enabled_backends_count == num_backends)
 878                         break;
 879         }
 880
 881         if (enabled_backends_count == 0) {
 882                 enabled_backends_mask = 1;
 883                 enabled_backends_count = 1;
 884         }
 885
 886         if (enabled_backends_count != num_backends)
 887                 num_backends = enabled_backends_count;
 888
 889         memset((uint8_t *)&swizzle_pipe[0], 0, sizeof(u32) * R6XX_MAX_PIPES);
 890         switch (num_tile_pipes) {
 891         case 1:
 892                 swizzle_pipe[0] = 0;
 893                 break;
 894         case 2:
 895                 swizzle_pipe[0] = 0;
 896                 swizzle_pipe[1] = 1;
 897                 break;
 898         case 3:
 899                 swizzle_pipe[0] = 0;
 900                 swizzle_pipe[1] = 1;
 901                 swizzle_pipe[2] = 2;
 902                 break;
 903         case 4:
 904                 swizzle_pipe[0] = 0;
 905                 swizzle_pipe[1] = 1;
 906                 swizzle_pipe[2] = 2;
 907                 swizzle_pipe[3] = 3;
 908                 break;
 909         case 5:
 910                 swizzle_pipe[0] = 0;
 911                 swizzle_pipe[1] = 1;
 912                 swizzle_pipe[2] = 2;
 913                 swizzle_pipe[3] = 3;
 914                 swizzle_pipe[4] = 4;
 915                 break;
 916         case 6:
 917                 swizzle_pipe[0] = 0;
 918                 swizzle_pipe[1] = 2;
 919                 swizzle_pipe[2] = 4;
 920                 swizzle_pipe[3] = 5;
 921                 swizzle_pipe[4] = 1;
 922                 swizzle_pipe[5] = 3;
 923                 break;
 924         case 7:
 925                 swizzle_pipe[0] = 0;
 926                 swizzle_pipe[1] = 2;
 927                 swizzle_pipe[2] = 4;
 928                 swizzle_pipe[3] = 6;
 929                 swizzle_pipe[4] = 1;
 930                 swizzle_pipe[5] = 3;
 931                 swizzle_pipe[6] = 5;
 932                 break;
 933         case 8:
 934                 swizzle_pipe[0] = 0;
 935                 swizzle_pipe[1] = 2;
 936                 swizzle_pipe[2] = 4;
 937                 swizzle_pipe[3] = 6;
 938                 swizzle_pipe[4] = 1;
 939                 swizzle_pipe[5] = 3;
 940                 swizzle_pipe[6] = 5;
 941                 swizzle_pipe[7] = 7;
 942                 break;
 943         }
 944
 945         cur_backend = 0;
 946         for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
 947                 while (((1 << cur_backend) & enabled_backends_mask) == 0)
 948                         cur_backend = (cur_backend + 1) % R6XX_MAX_BACKENDS;
 949
 950                 backend_map |= (u32)(((cur_backend & 3) << (swizzle_pipe[cur_pipe] * 2)));
 951
 952                 cur_backend = (cur_backend + 1) % R6XX_MAX_BACKENDS;
 953         }
 954
 955         return backend_map;
 956 }
 957
 958 int r600_count_pipe_bits(uint32_t val)
 959 {
 960         int i, ret = 0;
 961
 962         for (i = 0; i < 32; i++) {
 963                 ret += val & 1;
 964                 val >>= 1;
 965         }
 966         return ret;
 967 }
 968
 969 void r600_gpu_init(struct radeon_device *rdev)
 970 {
 971         u32 tiling_config;
 972         u32 ramcfg;
 973         u32 backend_map;
 974         u32 cc_rb_backend_disable;
 975         u32 cc_gc_shader_pipe_config;
 976         u32 tmp;
 977         int i, j;
 978         u32 sq_config;
 979         u32 sq_gpr_resource_mgmt_1 = 0;
 980         u32 sq_gpr_resource_mgmt_2 = 0;
 981         u32 sq_thread_resource_mgmt = 0;
 982         u32 sq_stack_resource_mgmt_1 = 0;
 983         u32 sq_stack_resource_mgmt_2 = 0;
 984
 985         /* FIXME: implement */
 986         switch (rdev->family) {
 987         case CHIP_R600:
 988                 rdev->config.r600.max_pipes = 4;
 989                 rdev->config.r600.max_tile_pipes = 8;
 990                 rdev->config.r600.max_simds = 4;
 991                 rdev->config.r600.max_backends = 4;
 992                 rdev->config.r600.max_gprs = 256;
 993                 rdev->config.r600.max_threads = 192;
 994                 rdev->config.r600.max_stack_entries = 256;
 995                 rdev->config.r600.max_hw_contexts = 8;
 996                 rdev->config.r600.max_gs_threads = 16;
 997                 rdev->config.r600.sx_max_export_size = 128;
 998                 rdev->config.r600.sx_max_export_pos_size = 16;
 999                 rdev->config.r600.sx_max_export_smx_size = 128;
1000                 rdev->config.r600.sq_num_cf_insts = 2;
1001                 break;
1002         case CHIP_RV630:
1003         case CHIP_RV635:
1004                 rdev->config.r600.max_pipes = 2;
1005                 rdev->config.r600.max_tile_pipes = 2;
1006                 rdev->config.r600.max_simds = 3;
1007                 rdev->config.r600.max_backends = 1;
1008                 rdev->config.r600.max_gprs = 128;
1009                 rdev->config.r600.max_threads = 192;
1010                 rdev->config.r600.max_stack_entries = 128;
1011                 rdev->config.r600.max_hw_contexts = 8;
1012                 rdev->config.r600.max_gs_threads = 4;
1013                 rdev->config.r600.sx_max_export_size = 128;
1014                 rdev->config.r600.sx_max_export_pos_size = 16;
1015                 rdev->config.r600.sx_max_export_smx_size = 128;
1016                 rdev->config.r600.sq_num_cf_insts = 2;
1017                 break;
1018         case CHIP_RV610:
1019         case CHIP_RV620:
1020         case CHIP_RS780:
1021         case CHIP_RS880:
1022                 rdev->config.r600.max_pipes = 1;
1023                 rdev->config.r600.max_tile_pipes = 1;
1024                 rdev->config.r600.max_simds = 2;
1025                 rdev->config.r600.max_backends = 1;
1026                 rdev->config.r600.max_gprs = 128;
1027                 rdev->config.r600.max_threads = 192;
1028                 rdev->config.r600.max_stack_entries = 128;
1029                 rdev->config.r600.max_hw_contexts = 4;
1030                 rdev->config.r600.max_gs_threads = 4;
1031                 rdev->config.r600.sx_max_export_size = 128;
1032                 rdev->config.r600.sx_max_export_pos_size = 16;
1033                 rdev->config.r600.sx_max_export_smx_size = 128;
1034                 rdev->config.r600.sq_num_cf_insts = 1;
1035                 break;
1036         case CHIP_RV670:
1037                 rdev->config.r600.max_pipes = 4;
1038                 rdev->config.r600.max_tile_pipes = 4;
1039                 rdev->config.r600.max_simds = 4;
1040                 rdev->config.r600.max_backends = 4;
1041                 rdev->config.r600.max_gprs = 192;
1042                 rdev->config.r600.max_threads = 192;
1043                 rdev->config.r600.max_stack_entries = 256;
1044                 rdev->config.r600.max_hw_contexts = 8;
1045                 rdev->config.r600.max_gs_threads = 16;
1046                 rdev->config.r600.sx_max_export_size = 128;
1047                 rdev->config.r600.sx_max_export_pos_size = 16;
1048                 rdev->config.r600.sx_max_export_smx_size = 128;
1049                 rdev->config.r600.sq_num_cf_insts = 2;
1050                 break;
1051         default:
1052                 break;
1053         }
1054
1055         /* Initialize HDP */
1056         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1057                 WREG32((0x2c14 + j), 0x00000000);
1058                 WREG32((0x2c18 + j), 0x00000000);
1059                 WREG32((0x2c1c + j), 0x00000000);
1060                 WREG32((0x2c20 + j), 0x00000000);
1061                 WREG32((0x2c24 + j), 0x00000000);
1062         }
1063
1064         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1065
1066         /* Setup tiling */
1067         tiling_config = 0;
1068         ramcfg = RREG32(RAMCFG);
1069         switch (rdev->config.r600.max_tile_pipes) {
1070         case 1:
1071                 tiling_config |= PIPE_TILING(0);
1072                 break;
1073         case 2:
1074                 tiling_config |= PIPE_TILING(1);
1075                 break;
1076         case 4:
1077                 tiling_config |= PIPE_TILING(2);
1078                 break;
1079         case 8:
1080                 tiling_config |= PIPE_TILING(3);
1081                 break;
1082         default:
1083                 break;
1084         }
1085         rdev->config.r600.tiling_npipes = rdev->config.r600.max_tile_pipes;
1086         rdev->config.r600.tiling_nbanks = 4 << ((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1087         tiling_config |= BANK_TILING((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1088         tiling_config |= GROUP_SIZE(0);
1089         rdev->config.r600.tiling_group_size = 256;
1090         tmp = (ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
1091         if (tmp > 3) {
1092                 tiling_config |= ROW_TILING(3);
1093                 tiling_config |= SAMPLE_SPLIT(3);
1094         } else {
1095                 tiling_config |= ROW_TILING(tmp);
1096                 tiling_config |= SAMPLE_SPLIT(tmp);
1097         }
1098         tiling_config |= BANK_SWAPS(1);
1099
1100         cc_rb_backend_disable = RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000;
1101         cc_rb_backend_disable |=
1102                 BACKEND_DISABLE((R6XX_MAX_BACKENDS_MASK << rdev->config.r600.max_backends) & R6XX_MAX_BACKENDS_MASK);
1103
1104         cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
1105         cc_gc_shader_pipe_config |=
1106                 INACTIVE_QD_PIPES((R6XX_MAX_PIPES_MASK << rdev->config.r600.max_pipes) & R6XX_MAX_PIPES_MASK);
1107         cc_gc_shader_pipe_config |=
1108                 INACTIVE_SIMDS((R6XX_MAX_SIMDS_MASK << rdev->config.r600.max_simds) & R6XX_MAX_SIMDS_MASK);
1109
1110         backend_map = r600_get_tile_pipe_to_backend_map(rdev->config.r600.max_tile_pipes,
1111                                                         (R6XX_MAX_BACKENDS -
1112                                                          r600_count_pipe_bits((cc_rb_backend_disable &
1113                                                                                R6XX_MAX_BACKENDS_MASK) >> 16)),
1114                                                         (cc_rb_backend_disable >> 16));
1115
1116         tiling_config |= BACKEND_MAP(backend_map);
1117         WREG32(GB_TILING_CONFIG, tiling_config);
1118         WREG32(DCP_TILING_CONFIG, tiling_config & 0xffff);
1119         WREG32(HDP_TILING_CONFIG, tiling_config & 0xffff);
1120
1121         /* Setup pipes */
1122         WREG32(CC_RB_BACKEND_DISABLE, cc_rb_backend_disable);
1123         WREG32(CC_GC_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
1124         WREG32(GC_USER_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
1125
1126         tmp = R6XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1127         WREG32(VGT_OUT_DEALLOC_CNTL, (tmp * 4) & DEALLOC_DIST_MASK);
1128         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((tmp * 4) - 2) & VTX_REUSE_DEPTH_MASK);
1129
1130         /* Setup some CP states */
1131         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | ROQ_IB2_START(0x2b)));
1132         WREG32(CP_MEQ_THRESHOLDS, (MEQ_END(0x40) | ROQ_END(0x40)));
1133
1134         WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO | SYNC_GRADIENT |
1135                              SYNC_WALKER | SYNC_ALIGNER));
1136         /* Setup various GPU states */
1137         if (rdev->family == CHIP_RV670)
1138                 WREG32(ARB_GDEC_RD_CNTL, 0x00000021);
1139
1140         tmp = RREG32(SX_DEBUG_1);
1141         tmp |= SMX_EVENT_RELEASE;
1142         if ((rdev->family > CHIP_R600))
1143                 tmp |= ENABLE_NEW_SMX_ADDRESS;
1144         WREG32(SX_DEBUG_1, tmp);
1145
1146         if (((rdev->family) == CHIP_R600) ||
1147             ((rdev->family) == CHIP_RV630) ||
1148             ((rdev->family) == CHIP_RV610) ||
1149             ((rdev->family) == CHIP_RV620) ||
1150             ((rdev->family) == CHIP_RS780) ||
1151             ((rdev->family) == CHIP_RS880)) {
1152                 WREG32(DB_DEBUG, PREZ_MUST_WAIT_FOR_POSTZ_DONE);
1153         } else {
1154                 WREG32(DB_DEBUG, 0);
1155         }
1156         WREG32(DB_WATERMARKS, (DEPTH_FREE(4) | DEPTH_CACHELINE_FREE(16) |
1157                                DEPTH_FLUSH(16) | DEPTH_PENDING_FREE(4)));
1158
1159         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1160         WREG32(VGT_NUM_INSTANCES, 0);
1161
1162         WREG32(SPI_CONFIG_CNTL, GPR_WRITE_PRIORITY(0));
1163         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(0));
1164
1165         tmp = RREG32(SQ_MS_FIFO_SIZES);
1166         if (((rdev->family) == CHIP_RV610) ||
1167             ((rdev->family) == CHIP_RV620) ||
1168             ((rdev->family) == CHIP_RS780) ||
1169             ((rdev->family) == CHIP_RS880)) {
1170                 tmp = (CACHE_FIFO_SIZE(0xa) |
1171                        FETCH_FIFO_HIWATER(0xa) |
1172                        DONE_FIFO_HIWATER(0xe0) |
1173                        ALU_UPDATE_FIFO_HIWATER(0x8));
1174         } else if (((rdev->family) == CHIP_R600) ||
1175                    ((rdev->family) == CHIP_RV630)) {
1176                 tmp &= ~DONE_FIFO_HIWATER(0xff);
1177                 tmp |= DONE_FIFO_HIWATER(0x4);
1178         }
1179         WREG32(SQ_MS_FIFO_SIZES, tmp);
1180
1181         /* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1182          * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
1183          */
1184         sq_config = RREG32(SQ_CONFIG);
1185         sq_config &= ~(PS_PRIO(3) |
1186                        VS_PRIO(3) |
1187                        GS_PRIO(3) |
1188                        ES_PRIO(3));
1189         sq_config |= (DX9_CONSTS |
1190                       VC_ENABLE |
1191                       PS_PRIO(0) |
1192                       VS_PRIO(1) |
1193                       GS_PRIO(2) |
1194                       ES_PRIO(3));
1195
1196         if ((rdev->family) == CHIP_R600) {
1197                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(124) |
1198                                           NUM_VS_GPRS(124) |
1199                                           NUM_CLAUSE_TEMP_GPRS(4));
1200                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(0) |
1201                                           NUM_ES_GPRS(0));
1202                 sq_thread_resource_mgmt = (NUM_PS_THREADS(136) |
1203                                            NUM_VS_THREADS(48) |
1204                                            NUM_GS_THREADS(4) |
1205                                            NUM_ES_THREADS(4));
1206                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(128) |
1207                                             NUM_VS_STACK_ENTRIES(128));
1208                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(0) |
1209                                             NUM_ES_STACK_ENTRIES(0));
1210         } else if (((rdev->family) == CHIP_RV610) ||
1211                    ((rdev->family) == CHIP_RV620) ||
1212                    ((rdev->family) == CHIP_RS780) ||
1213                    ((rdev->family) == CHIP_RS880)) {
1214                 /* no vertex cache */
1215                 sq_config &= ~VC_ENABLE;
1216
1217                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1218                                           NUM_VS_GPRS(44) |
1219                                           NUM_CLAUSE_TEMP_GPRS(2));
1220                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
1221                                           NUM_ES_GPRS(17));
1222                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1223                                            NUM_VS_THREADS(78) |
1224                                            NUM_GS_THREADS(4) |
1225                                            NUM_ES_THREADS(31));
1226                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
1227                                             NUM_VS_STACK_ENTRIES(40));
1228                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
1229                                             NUM_ES_STACK_ENTRIES(16));
1230         } else if (((rdev->family) == CHIP_RV630) ||
1231                    ((rdev->family) == CHIP_RV635)) {
1232                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1233                                           NUM_VS_GPRS(44) |
1234                                           NUM_CLAUSE_TEMP_GPRS(2));
1235                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(18) |
1236                                           NUM_ES_GPRS(18));
1237                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1238                                            NUM_VS_THREADS(78) |
1239                                            NUM_GS_THREADS(4) |
1240                                            NUM_ES_THREADS(31));
1241                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
1242                                             NUM_VS_STACK_ENTRIES(40));
1243                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
1244                                             NUM_ES_STACK_ENTRIES(16));
1245         } else if ((rdev->family) == CHIP_RV670) {
1246                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1247                                           NUM_VS_GPRS(44) |
1248                                           NUM_CLAUSE_TEMP_GPRS(2));
1249                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
1250                                           NUM_ES_GPRS(17));
1251                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1252                                            NUM_VS_THREADS(78) |
1253                                            NUM_GS_THREADS(4) |
1254                                            NUM_ES_THREADS(31));
1255                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(64) |
1256                                             NUM_VS_STACK_ENTRIES(64));
1257                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(64) |
1258                                             NUM_ES_STACK_ENTRIES(64));
1259         }
1260
1261         WREG32(SQ_CONFIG, sq_config);
1262         WREG32(SQ_GPR_RESOURCE_MGMT_1,  sq_gpr_resource_mgmt_1);
1263         WREG32(SQ_GPR_RESOURCE_MGMT_2,  sq_gpr_resource_mgmt_2);
1264         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1265         WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
1266         WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
1267
1268         if (((rdev->family) == CHIP_RV610) ||
1269             ((rdev->family) == CHIP_RV620) ||
1270             ((rdev->family) == CHIP_RS780) ||
1271             ((rdev->family) == CHIP_RS880)) {
1272                 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(TC_ONLY));
1273         } else {
1274                 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC));
1275         }
1276
1277         /* More default values. 2D/3D driver should adjust as needed */
1278         WREG32(PA_SC_AA_SAMPLE_LOCS_2S, (S0_X(0xc) | S0_Y(0x4) |
1279                                          S1_X(0x4) | S1_Y(0xc)));
1280         WREG32(PA_SC_AA_SAMPLE_LOCS_4S, (S0_X(0xe) | S0_Y(0xe) |
1281                                          S1_X(0x2) | S1_Y(0x2) |
1282                                          S2_X(0xa) | S2_Y(0x6) |
1283                                          S3_X(0x6) | S3_Y(0xa)));
1284         WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD0, (S0_X(0xe) | S0_Y(0xb) |
1285                                              S1_X(0x4) | S1_Y(0xc) |
1286                                              S2_X(0x1) | S2_Y(0x6) |
1287                                              S3_X(0xa) | S3_Y(0xe)));
1288         WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD1, (S4_X(0x6) | S4_Y(0x1) |
1289                                              S5_X(0x0) | S5_Y(0x0) |
1290                                              S6_X(0xb) | S6_Y(0x4) |
1291                                              S7_X(0x7) | S7_Y(0x8)));
1292
1293         WREG32(VGT_STRMOUT_EN, 0);
1294         tmp = rdev->config.r600.max_pipes * 16;
1295         switch (rdev->family) {
1296         case CHIP_RV610:
1297         case CHIP_RV620:
1298         case CHIP_RS780:
1299         case CHIP_RS880:
1300                 tmp += 32;
1301                 break;
1302         case CHIP_RV670:
1303                 tmp += 128;
1304                 break;
1305         default:
1306                 break;
1307         }
1308         if (tmp > 256) {
1309                 tmp = 256;
1310         }
1311         WREG32(VGT_ES_PER_GS, 128);
1312         WREG32(VGT_GS_PER_ES, tmp);
1313         WREG32(VGT_GS_PER_VS, 2);
1314         WREG32(VGT_GS_VERTEX_REUSE, 16);
1315
1316         /* more default values. 2D/3D driver should adjust as needed */
1317         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1318         WREG32(VGT_STRMOUT_EN, 0);
1319         WREG32(SX_MISC, 0);
1320         WREG32(PA_SC_MODE_CNTL, 0);
1321         WREG32(PA_SC_AA_CONFIG, 0);
1322         WREG32(PA_SC_LINE_STIPPLE, 0);
1323         WREG32(SPI_INPUT_Z, 0);
1324         WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
1325         WREG32(CB_COLOR7_FRAG, 0);
1326
1327         /* Clear render buffer base addresses */
1328         WREG32(CB_COLOR0_BASE, 0);
1329         WREG32(CB_COLOR1_BASE, 0);
1330         WREG32(CB_COLOR2_BASE, 0);
1331         WREG32(CB_COLOR3_BASE, 0);
1332         WREG32(CB_COLOR4_BASE, 0);
1333         WREG32(CB_COLOR5_BASE, 0);
1334         WREG32(CB_COLOR6_BASE, 0);
1335         WREG32(CB_COLOR7_BASE, 0);
1336         WREG32(CB_COLOR7_FRAG, 0);
1337
1338         switch (rdev->family) {
1339         case CHIP_RV610:
1340         case CHIP_RV620:
1341         case CHIP_RS780:
1342         case CHIP_RS880:
1343                 tmp = TC_L2_SIZE(8);
1344                 break;
1345         case CHIP_RV630:
1346         case CHIP_RV635:
1347                 tmp = TC_L2_SIZE(4);
1348                 break;
1349         case CHIP_R600:
1350                 tmp = TC_L2_SIZE(0) | L2_DISABLE_LATE_HIT;
1351                 break;
1352         default:
1353                 tmp = TC_L2_SIZE(0);
1354                 break;
1355         }
1356         WREG32(TC_CNTL, tmp);
1357
1358         tmp = RREG32(HDP_HOST_PATH_CNTL);
1359         WREG32(HDP_HOST_PATH_CNTL, tmp);
1360
1361         tmp = RREG32(ARB_POP);
1362         tmp |= ENABLE_TC128;
1363         WREG32(ARB_POP, tmp);
1364
1365         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1366         WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
1367                                NUM_CLIP_SEQ(3)));
1368         WREG32(PA_SC_ENHANCE, FORCE_EOV_MAX_CLK_CNT(4095));
1369 }
1370
1371
1372 /*
1373  * Indirect registers accessor
1374  */
1375 u32 r600_pciep_rreg(struct radeon_device *rdev, u32 reg)
1376 {
1377         u32 r;
1378
1379         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
1380         (void)RREG32(PCIE_PORT_INDEX);
1381         r = RREG32(PCIE_PORT_DATA);
1382         return r;
1383 }
1384
1385 void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
1386 {
1387         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
1388         (void)RREG32(PCIE_PORT_INDEX);
1389         WREG32(PCIE_PORT_DATA, (v));
1390         (void)RREG32(PCIE_PORT_DATA);
1391 }
1392
1393 /*
1394  * CP & Ring
1395  */
1396 void r600_cp_stop(struct radeon_device *rdev)
1397 {
1398         WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1399 }
1400
1401 int r600_init_microcode(struct radeon_device *rdev)
1402 {
1403         struct platform_device *pdev;
1404         const char *chip_name;
1405         const char *rlc_chip_name;
1406         size_t pfp_req_size, me_req_size, rlc_req_size;
1407         char fw_name[30];
1408         int err;
1409
1410         DRM_DEBUG("\n");
1411
1412         pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
1413         err = IS_ERR(pdev);
1414         if (err) {
1415                 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
1416                 return -EINVAL;
1417         }
1418
1419         switch (rdev->family) {
1420         case CHIP_R600:
1421                 chip_name = "R600";
1422                 rlc_chip_name = "R600";
1423                 break;
1424         case CHIP_RV610:
1425                 chip_name = "RV610";
1426                 rlc_chip_name = "R600";
1427                 break;
1428         case CHIP_RV630:
1429                 chip_name = "RV630";
1430                 rlc_chip_name = "R600";
1431                 break;
1432         case CHIP_RV620:
1433                 chip_name = "RV620";
1434                 rlc_chip_name = "R600";
1435                 break;
1436         case CHIP_RV635:
1437                 chip_name = "RV635";
1438                 rlc_chip_name = "R600";
1439                 break;
1440         case CHIP_RV670:
1441                 chip_name = "RV670";
1442                 rlc_chip_name = "R600";
1443                 break;
1444         case CHIP_RS780:
1445         case CHIP_RS880:
1446                 chip_name = "RS780";
1447                 rlc_chip_name = "R600";
1448                 break;
1449         case CHIP_RV770:
1450                 chip_name = "RV770";
1451                 rlc_chip_name = "R700";
1452                 break;
1453         case CHIP_RV730:
1454         case CHIP_RV740:
1455                 chip_name = "RV730";
1456                 rlc_chip_name = "R700";
1457                 break;
1458         case CHIP_RV710:
1459                 chip_name = "RV710";
1460                 rlc_chip_name = "R700";
1461                 break;
1462         case CHIP_CEDAR:
1463                 chip_name = "CEDAR";
1464                 rlc_chip_name = "";
1465                 break;
1466         case CHIP_REDWOOD:
1467                 chip_name = "REDWOOD";
1468                 rlc_chip_name = "";
1469                 break;
1470         case CHIP_JUNIPER:
1471                 chip_name = "JUNIPER";
1472                 rlc_chip_name = "";
1473                 break;
1474         case CHIP_CYPRESS:
1475         case CHIP_HEMLOCK:
1476                 chip_name = "CYPRESS";
1477                 rlc_chip_name = "";
1478                 break;
1479         default: BUG();
1480         }
1481
1482         if (rdev->family >= CHIP_CEDAR) {
1483                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
1484                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
1485                 rlc_req_size = 0;
1486         } else if (rdev->family >= CHIP_RV770) {
1487                 pfp_req_size = R700_PFP_UCODE_SIZE * 4;
1488                 me_req_size = R700_PM4_UCODE_SIZE * 4;
1489                 rlc_req_size = R700_RLC_UCODE_SIZE * 4;
1490         } else {
1491                 pfp_req_size = PFP_UCODE_SIZE * 4;
1492                 me_req_size = PM4_UCODE_SIZE * 12;
1493                 rlc_req_size = RLC_UCODE_SIZE * 4;
1494         }
1495
1496         DRM_INFO("Loading %s Microcode\n", chip_name);
1497
1498         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1499         err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
1500         if (err)
1501                 goto out;
1502         if (rdev->pfp_fw->size != pfp_req_size) {
1503                 printk(KERN_ERR
1504                        "r600_cp: Bogus length %zu in firmware \"%s\"\n",
1505                        rdev->pfp_fw->size, fw_name);
1506                 err = -EINVAL;
1507                 goto out;
1508         }
1509
1510         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1511         err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
1512         if (err)
1513                 goto out;
1514         if (rdev->me_fw->size != me_req_size) {
1515                 printk(KERN_ERR
1516                        "r600_cp: Bogus length %zu in firmware \"%s\"\n",
1517                        rdev->me_fw->size, fw_name);
1518                 err = -EINVAL;
1519         }
1520
1521         /* XXX until evergreen interrupts are supported */
1522         if (rdev->family < CHIP_CEDAR) {
1523         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
1524         err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
1525         if (err)
1526                 goto out;
1527         if (rdev->rlc_fw->size != rlc_req_size) {
1528                 printk(KERN_ERR
1529                        "r600_rlc: Bogus length %zu in firmware \"%s\"\n",
1530                        rdev->rlc_fw->size, fw_name);
1531                 err = -EINVAL;
1532         }
1533         }
1534
1535 out:
1536         platform_device_unregister(pdev);
1537
1538         if (err) {
1539                 if (err != -EINVAL)
1540                         printk(KERN_ERR
1541                                "r600_cp: Failed to load firmware \"%s\"\n",
1542                                fw_name);
1543                 release_firmware(rdev->pfp_fw);
1544                 rdev->pfp_fw = NULL;
1545                 release_firmware(rdev->me_fw);
1546                 rdev->me_fw = NULL;
1547                 release_firmware(rdev->rlc_fw);
1548                 rdev->rlc_fw = NULL;
1549         }
1550         return err;
1551 }
1552
1553 static int r600_cp_load_microcode(struct radeon_device *rdev)
1554 {
1555         const __be32 *fw_data;
1556         int i;
1557
1558         if (!rdev->me_fw || !rdev->pfp_fw)
1559                 return -EINVAL;
1560
1561         r600_cp_stop(rdev);
1562
1563         WREG32(CP_RB_CNTL, RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
1564
1565         /* Reset cp */
1566         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
1567         RREG32(GRBM_SOFT_RESET);
1568         mdelay(15);
1569         WREG32(GRBM_SOFT_RESET, 0);
1570
1571         WREG32(CP_ME_RAM_WADDR, 0);
1572
1573         fw_data = (const __be32 *)rdev->me_fw->data;
1574         WREG32(CP_ME_RAM_WADDR, 0);
1575         for (i = 0; i < PM4_UCODE_SIZE * 3; i++)
1576                 WREG32(CP_ME_RAM_DATA,
1577                        be32_to_cpup(fw_data++));
1578
1579         fw_data = (const __be32 *)rdev->pfp_fw->data;
1580         WREG32(CP_PFP_UCODE_ADDR, 0);
1581         for (i = 0; i < PFP_UCODE_SIZE; i++)
1582                 WREG32(CP_PFP_UCODE_DATA,
1583                        be32_to_cpup(fw_data++));
1584
1585         WREG32(CP_PFP_UCODE_ADDR, 0);
1586         WREG32(CP_ME_RAM_WADDR, 0);
1587         WREG32(CP_ME_RAM_RADDR, 0);
1588         return 0;
1589 }
1590
1591 int r600_cp_start(struct radeon_device *rdev)
1592 {
1593         int r;
1594         uint32_t cp_me;
1595
1596         r = radeon_ring_lock(rdev, 7);
1597         if (r) {
1598                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1599                 return r;
1600         }
1601         radeon_ring_write(rdev, PACKET3(PACKET3_ME_INITIALIZE, 5));
1602         radeon_ring_write(rdev, 0x1);
1603         if (rdev->family >= CHIP_CEDAR) {
1604                 radeon_ring_write(rdev, 0x0);
1605                 radeon_ring_write(rdev, rdev->config.evergreen.max_hw_contexts - 1);
1606         } else if (rdev->family >= CHIP_RV770) {
1607                 radeon_ring_write(rdev, 0x0);
1608                 radeon_ring_write(rdev, rdev->config.rv770.max_hw_contexts - 1);
1609         } else {
1610                 radeon_ring_write(rdev, 0x3);
1611                 radeon_ring_write(rdev, rdev->config.r600.max_hw_contexts - 1);
1612         }
1613         radeon_ring_write(rdev, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1614         radeon_ring_write(rdev, 0);
1615         radeon_ring_write(rdev, 0);
1616         radeon_ring_unlock_commit(rdev);
1617
1618         cp_me = 0xff;
1619         WREG32(R_0086D8_CP_ME_CNTL, cp_me);
1620         return 0;
1621 }
1622
1623 int r600_cp_resume(struct radeon_device *rdev)
1624 {
1625         u32 tmp;
1626         u32 rb_bufsz;
1627         int r;
1628
1629         /* Reset cp */
1630         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
1631         RREG32(GRBM_SOFT_RESET);
1632         mdelay(15);
1633         WREG32(GRBM_SOFT_RESET, 0);
1634
1635         /* Set ring buffer size */
1636         rb_bufsz = drm_order(rdev->cp.ring_size / 8);
1637         tmp = RB_NO_UPDATE | (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
1638 #ifdef __BIG_ENDIAN
1639         tmp |= BUF_SWAP_32BIT;
1640 #endif
1641         WREG32(CP_RB_CNTL, tmp);
1642         WREG32(CP_SEM_WAIT_TIMER, 0x4);
1643
1644         /* Set the write pointer delay */
1645         WREG32(CP_RB_WPTR_DELAY, 0);
1646
1647         /* Initialize the ring buffer's read and write pointers */
1648         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
1649         WREG32(CP_RB_RPTR_WR, 0);
1650         WREG32(CP_RB_WPTR, 0);
1651         WREG32(CP_RB_RPTR_ADDR, rdev->cp.gpu_addr & 0xFFFFFFFF);
1652         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->cp.gpu_addr));
1653         mdelay(1);
1654         WREG32(CP_RB_CNTL, tmp);
1655
1656         WREG32(CP_RB_BASE, rdev->cp.gpu_addr >> 8);
1657         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
1658
1659         rdev->cp.rptr = RREG32(CP_RB_RPTR);
1660         rdev->cp.wptr = RREG32(CP_RB_WPTR);
1661
1662         r600_cp_start(rdev);
1663         rdev->cp.ready = true;
1664         r = radeon_ring_test(rdev);
1665         if (r) {
1666                 rdev->cp.ready = false;
1667                 return r;
1668         }
1669         return 0;
1670 }
1671
1672 void r600_cp_commit(struct radeon_device *rdev)
1673 {
1674         WREG32(CP_RB_WPTR, rdev->cp.wptr);
1675         (void)RREG32(CP_RB_WPTR);
1676 }
1677
1678 void r600_ring_init(struct radeon_device *rdev, unsigned ring_size)
1679 {
1680         u32 rb_bufsz;
1681
1682         /* Align ring size */
1683         rb_bufsz = drm_order(ring_size / 8);
1684         ring_size = (1 << (rb_bufsz + 1)) * 4;
1685         rdev->cp.ring_size = ring_size;
1686         rdev->cp.align_mask = 16 - 1;
1687 }
1688
1689 void r600_cp_fini(struct radeon_device *rdev)
1690 {
1691         r600_cp_stop(rdev);
1692         radeon_ring_fini(rdev);
1693 }
1694
1695
1696 /*
1697  * GPU scratch registers helpers function.
1698  */
1699 void r600_scratch_init(struct radeon_device *rdev)
1700 {
1701         int i;
1702
1703         rdev->scratch.num_reg = 7;
1704         for (i = 0; i < rdev->scratch.num_reg; i++) {
1705                 rdev->scratch.free[i] = true;
1706                 rdev->scratch.reg[i] = SCRATCH_REG0 + (i * 4);
1707         }
1708 }
1709
1710 int r600_ring_test(struct radeon_device *rdev)
1711 {
1712         uint32_t scratch;
1713         uint32_t tmp = 0;
1714         unsigned i;
1715         int r;
1716
1717         r = radeon_scratch_get(rdev, &scratch);
1718         if (r) {
1719                 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
1720                 return r;
1721         }
1722         WREG32(scratch, 0xCAFEDEAD);
1723         r = radeon_ring_lock(rdev, 3);
1724         if (r) {
1725                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1726                 radeon_scratch_free(rdev, scratch);
1727                 return r;
1728         }
1729         radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1730         radeon_ring_write(rdev, ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
1731         radeon_ring_write(rdev, 0xDEADBEEF);
1732         radeon_ring_unlock_commit(rdev);
1733         for (i = 0; i < rdev->usec_timeout; i++) {
1734                 tmp = RREG32(scratch);
1735                 if (tmp == 0xDEADBEEF)
1736                         break;
1737                 DRM_UDELAY(1);
1738         }
1739         if (i < rdev->usec_timeout) {
1740                 DRM_INFO("ring test succeeded in %d usecs\n", i);
1741         } else {
1742                 DRM_ERROR("radeon: ring test failed (scratch(0x%04X)=0x%08X)\n",
1743                           scratch, tmp);
1744                 r = -EINVAL;
1745         }
1746         radeon_scratch_free(rdev, scratch);
1747         return r;
1748 }
1749
1750 void r600_wb_disable(struct radeon_device *rdev)
1751 {
1752         int r;
1753
1754         WREG32(SCRATCH_UMSK, 0);
1755         if (rdev->wb.wb_obj) {
1756                 r = radeon_bo_reserve(rdev->wb.wb_obj, false);
1757                 if (unlikely(r != 0))
1758                         return;
1759                 radeon_bo_kunmap(rdev->wb.wb_obj);
1760                 radeon_bo_unpin(rdev->wb.wb_obj);
1761                 radeon_bo_unreserve(rdev->wb.wb_obj);
1762         }
1763 }
1764
1765 void r600_wb_fini(struct radeon_device *rdev)
1766 {
1767         r600_wb_disable(rdev);
1768         if (rdev->wb.wb_obj) {
1769                 radeon_bo_unref(&rdev->wb.wb_obj);
1770                 rdev->wb.wb = NULL;
1771                 rdev->wb.wb_obj = NULL;
1772         }
1773 }
1774
1775 int r600_wb_enable(struct radeon_device *rdev)
1776 {
1777         int r;
1778
1779         if (rdev->wb.wb_obj == NULL) {
1780                 r = radeon_bo_create(rdev, NULL, RADEON_GPU_PAGE_SIZE, true,
1781                                 RADEON_GEM_DOMAIN_GTT, &rdev->wb.wb_obj);
1782                 if (r) {
1783                         dev_warn(rdev->dev, "(%d) create WB bo failed\n", r);
1784                         return r;
1785                 }
1786                 r = radeon_bo_reserve(rdev->wb.wb_obj, false);
1787                 if (unlikely(r != 0)) {
1788                         r600_wb_fini(rdev);
1789                         return r;
1790                 }
1791                 r = radeon_bo_pin(rdev->wb.wb_obj, RADEON_GEM_DOMAIN_GTT,
1792                                 &rdev->wb.gpu_addr);
1793                 if (r) {
1794                         radeon_bo_unreserve(rdev->wb.wb_obj);
1795                         dev_warn(rdev->dev, "(%d) pin WB bo failed\n", r);
1796                         r600_wb_fini(rdev);
1797                         return r;
1798                 }
1799                 r = radeon_bo_kmap(rdev->wb.wb_obj, (void **)&rdev->wb.wb);
1800                 radeon_bo_unreserve(rdev->wb.wb_obj);
1801                 if (r) {
1802                         dev_warn(rdev->dev, "(%d) map WB bo failed\n", r);
1803                         r600_wb_fini(rdev);
1804                         return r;
1805                 }
1806         }
1807         WREG32(SCRATCH_ADDR, (rdev->wb.gpu_addr >> 8) & 0xFFFFFFFF);
1808         WREG32(CP_RB_RPTR_ADDR, (rdev->wb.gpu_addr + 1024) & 0xFFFFFFFC);
1809         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + 1024) & 0xFF);
1810         WREG32(SCRATCH_UMSK, 0xff);
1811         return 0;
1812 }
1813
1814 void r600_fence_ring_emit(struct radeon_device *rdev,
1815                           struct radeon_fence *fence)
1816 {
1817         /* Also consider EVENT_WRITE_EOP.  it handles the interrupts + timestamps + events */
1818
1819         radeon_ring_write(rdev, PACKET3(PACKET3_EVENT_WRITE, 0));
1820         radeon_ring_write(rdev, CACHE_FLUSH_AND_INV_EVENT);
1821         /* wait for 3D idle clean */
1822         radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1823         radeon_ring_write(rdev, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
1824         radeon_ring_write(rdev, WAIT_3D_IDLE_bit | WAIT_3D_IDLECLEAN_bit);
1825         /* Emit fence sequence & fire IRQ */
1826         radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1827         radeon_ring_write(rdev, ((rdev->fence_drv.scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
1828         radeon_ring_write(rdev, fence->seq);
1829         /* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
1830         radeon_ring_write(rdev, PACKET0(CP_INT_STATUS, 0));
1831         radeon_ring_write(rdev, RB_INT_STAT);
1832 }
1833
1834 int r600_copy_blit(struct radeon_device *rdev,
1835                    uint64_t src_offset, uint64_t dst_offset,
1836                    unsigned num_pages, struct radeon_fence *fence)
1837 {
1838         int r;
1839
1840         mutex_lock(&rdev->r600_blit.mutex);
1841         rdev->r600_blit.vb_ib = NULL;
1842         r = r600_blit_prepare_copy(rdev, num_pages * RADEON_GPU_PAGE_SIZE);
1843         if (r) {
1844                 if (rdev->r600_blit.vb_ib)
1845                         radeon_ib_free(rdev, &rdev->r600_blit.vb_ib);
1846                 mutex_unlock(&rdev->r600_blit.mutex);
1847                 return r;
1848         }
1849         r600_kms_blit_copy(rdev, src_offset, dst_offset, num_pages * RADEON_GPU_PAGE_SIZE);
1850         r600_blit_done_copy(rdev, fence);
1851         mutex_unlock(&rdev->r600_blit.mutex);
1852         return 0;
1853 }
1854
1855 int r600_set_surface_reg(struct radeon_device *rdev, int reg,
1856                          uint32_t tiling_flags, uint32_t pitch,
1857                          uint32_t offset, uint32_t obj_size)
1858 {
1859         /* FIXME: implement */
1860         return 0;
1861 }
1862
1863 void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
1864 {
1865         /* FIXME: implement */
1866 }
1867
1868
1869 bool r600_card_posted(struct radeon_device *rdev)
1870 {
1871         uint32_t reg;
1872
1873         /* first check CRTCs */
1874         reg = RREG32(D1CRTC_CONTROL) |
1875                 RREG32(D2CRTC_CONTROL);
1876         if (reg & CRTC_EN)
1877                 return true;
1878
1879         /* then check MEM_SIZE, in case the crtcs are off */
1880         if (RREG32(CONFIG_MEMSIZE))
1881                 return true;
1882
1883         return false;
1884 }
1885
1886 int r600_startup(struct radeon_device *rdev)
1887 {
1888         int r;
1889
1890         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
1891                 r = r600_init_microcode(rdev);
1892                 if (r) {
1893                         DRM_ERROR("Failed to load firmware!\n");
1894                         return r;
1895                 }
1896         }
1897
1898         r600_mc_program(rdev);
1899         if (rdev->flags & RADEON_IS_AGP) {
1900                 r600_agp_enable(rdev);
1901         } else {
1902                 r = r600_pcie_gart_enable(rdev);
1903                 if (r)
1904                         return r;
1905         }
1906         r600_gpu_init(rdev);
1907         r = r600_blit_init(rdev);
1908         if (r) {
1909                 r600_blit_fini(rdev);
1910                 rdev->asic->copy = NULL;
1911                 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
1912         }
1913         /* pin copy shader into vram */
1914         if (rdev->r600_blit.shader_obj) {
1915                 r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false);
1916                 if (unlikely(r != 0))
1917                         return r;
1918                 r = radeon_bo_pin(rdev->r600_blit.shader_obj, RADEON_GEM_DOMAIN_VRAM,
1919                                 &rdev->r600_blit.shader_gpu_addr);
1920                 radeon_bo_unreserve(rdev->r600_blit.shader_obj);
1921                 if (r) {
1922                         dev_err(rdev->dev, "(%d) pin blit object failed\n", r);
1923                         return r;
1924                 }
1925         }
1926         /* Enable IRQ */
1927         r = r600_irq_init(rdev);
1928         if (r) {
1929                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
1930                 radeon_irq_kms_fini(rdev);
1931                 return r;
1932         }
1933         r600_irq_set(rdev);
1934
1935         r = radeon_ring_init(rdev, rdev->cp.ring_size);
1936         if (r)
1937                 return r;
1938         r = r600_cp_load_microcode(rdev);
1939         if (r)
1940                 return r;
1941         r = r600_cp_resume(rdev);
1942         if (r)
1943                 return r;
1944         /* write back buffer are not vital so don't worry about failure */
1945         r600_wb_enable(rdev);
1946         return 0;
1947 }
1948
1949 void r600_vga_set_state(struct radeon_device *rdev, bool state)
1950 {
1951         uint32_t temp;
1952
1953         temp = RREG32(CONFIG_CNTL);
1954         if (state == false) {
1955                 temp &= ~(1<<0);
1956                 temp |= (1<<1);
1957         } else {
1958                 temp &= ~(1<<1);
1959         }
1960         WREG32(CONFIG_CNTL, temp);
1961 }
1962
1963 int r600_resume(struct radeon_device *rdev)
1964 {
1965         int r;
1966
1967         /* Do not reset GPU before posting, on r600 hw unlike on r500 hw,
1968          * posting will perform necessary task to bring back GPU into good
1969          * shape.
1970          */
1971         /* post card */
1972         atom_asic_init(rdev->mode_info.atom_context);
1973         /* Initialize clocks */
1974         r = radeon_clocks_init(rdev);
1975         if (r) {
1976                 return r;
1977         }
1978
1979         r = r600_startup(rdev);
1980         if (r) {
1981                 DRM_ERROR("r600 startup failed on resume\n");
1982                 return r;
1983         }
1984
1985         r = r600_ib_test(rdev);
1986         if (r) {
1987                 DRM_ERROR("radeon: failled testing IB (%d).\n", r);
1988                 return r;
1989         }
1990
1991         r = r600_audio_init(rdev);
1992         if (r) {
1993                 DRM_ERROR("radeon: audio resume failed\n");
1994                 return r;
1995         }
1996
1997         return r;
1998 }
1999
2000 int r600_suspend(struct radeon_device *rdev)
2001 {
2002         int r;
2003
2004         r600_audio_fini(rdev);
2005         /* FIXME: we should wait for ring to be empty */
2006         r600_cp_stop(rdev);
2007         rdev->cp.ready = false;
2008         r600_irq_suspend(rdev);
2009         r600_wb_disable(rdev);
2010         r600_pcie_gart_disable(rdev);
2011         /* unpin shaders bo */
2012         if (rdev->r600_blit.shader_obj) {
2013                 r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false);
2014                 if (!r) {
2015                         radeon_bo_unpin(rdev->r600_blit.shader_obj);
2016                         radeon_bo_unreserve(rdev->r600_blit.shader_obj);
2017                 }
2018         }
2019         return 0;
2020 }
2021
2022 /* Plan is to move initialization in that function and use
2023  * helper function so that radeon_device_init pretty much
2024  * do nothing more than calling asic specific function. This
2025  * should also allow to remove a bunch of callback function
2026  * like vram_info.
2027  */
2028 int r600_init(struct radeon_device *rdev)
2029 {
2030         int r;
2031
2032         r = radeon_dummy_page_init(rdev);
2033         if (r)
2034                 return r;
2035         if (r600_debugfs_mc_info_init(rdev)) {
2036                 DRM_ERROR("Failed to register debugfs file for mc !\n");
2037         }
2038         /* This don't do much */
2039         r = radeon_gem_init(rdev);
2040         if (r)
2041                 return r;
2042         /* Read BIOS */
2043         if (!radeon_get_bios(rdev)) {
2044                 if (ASIC_IS_AVIVO(rdev))
2045                         return -EINVAL;
2046         }
2047         /* Must be an ATOMBIOS */
2048         if (!rdev->is_atom_bios) {
2049                 dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
2050                 return -EINVAL;
2051         }
2052         r = radeon_atombios_init(rdev);
2053         if (r)
2054                 return r;
2055         /* Post card if necessary */
2056         if (!r600_card_posted(rdev)) {
2057                 if (!rdev->bios) {
2058                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2059                         return -EINVAL;
2060                 }
2061                 DRM_INFO("GPU not posted. posting now...\n");
2062                 atom_asic_init(rdev->mode_info.atom_context);
2063         }
2064         /* Initialize scratch registers */
2065         r600_scratch_init(rdev);
2066         /* Initialize surface registers */
2067         radeon_surface_init(rdev);
2068         /* Initialize clocks */
2069         radeon_get_clock_info(rdev->ddev);
2070         r = radeon_clocks_init(rdev);
2071         if (r)
2072                 return r;
2073         /* Initialize power management */
2074         radeon_pm_init(rdev);
2075         /* Fence driver */
2076         r = radeon_fence_driver_init(rdev);
2077         if (r)
2078                 return r;
2079         if (rdev->flags & RADEON_IS_AGP) {
2080                 r = radeon_agp_init(rdev);
2081                 if (r)
2082                         radeon_agp_disable(rdev);
2083         }
2084         r = r600_mc_init(rdev);
2085         if (r)
2086                 return r;
2087         /* Memory manager */
2088         r = radeon_bo_init(rdev);
2089         if (r)
2090                 return r;
2091
2092         r = radeon_irq_kms_init(rdev);
2093         if (r)
2094                 return r;
2095
2096         rdev->cp.ring_obj = NULL;
2097         r600_ring_init(rdev, 1024 * 1024);
2098
2099         rdev->ih.ring_obj = NULL;
2100         r600_ih_ring_init(rdev, 64 * 1024);
2101
2102         r = r600_pcie_gart_init(rdev);
2103         if (r)
2104                 return r;
2105
2106         rdev->accel_working = true;
2107         r = r600_startup(rdev);
2108         if (r) {
2109                 dev_err(rdev->dev, "disabling GPU acceleration\n");
2110                 r600_cp_fini(rdev);
2111                 r600_wb_fini(rdev);
2112                 r600_irq_fini(rdev);
2113                 radeon_irq_kms_fini(rdev);
2114                 r600_pcie_gart_fini(rdev);
2115                 rdev->accel_working = false;
2116         }
2117         if (rdev->accel_working) {
2118                 r = radeon_ib_pool_init(rdev);
2119                 if (r) {
2120                         dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2121                         rdev->accel_working = false;
2122                 } else {
2123                         r = r600_ib_test(rdev);
2124                         if (r) {
2125                                 dev_err(rdev->dev, "IB test failed (%d).\n", r);
2126                                 rdev->accel_working = false;
2127                         }
2128                 }
2129         }
2130
2131         r = r600_audio_init(rdev);
2132         if (r)
2133                 return r; /* TODO error handling */
2134         return 0;
2135 }
2136
2137 void r600_fini(struct radeon_device *rdev)
2138 {
2139         radeon_pm_fini(rdev);
2140         r600_audio_fini(rdev);
2141         r600_blit_fini(rdev);
2142         r600_cp_fini(rdev);
2143         r600_wb_fini(rdev);
2144         r600_irq_fini(rdev);
2145         radeon_irq_kms_fini(rdev);
2146         r600_pcie_gart_fini(rdev);
2147         radeon_agp_fini(rdev);
2148         radeon_gem_fini(rdev);
2149         radeon_fence_driver_fini(rdev);
2150         radeon_clocks_fini(rdev);
2151         radeon_bo_fini(rdev);
2152         radeon_atombios_fini(rdev);
2153         kfree(rdev->bios);
2154         rdev->bios = NULL;
2155         radeon_dummy_page_fini(rdev);
2156 }
2157
2158
2159 /*
2160  * CS stuff
2161  */
2162 void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2163 {
2164         /* FIXME: implement */
2165         radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2166         radeon_ring_write(rdev, ib->gpu_addr & 0xFFFFFFFC);
2167         radeon_ring_write(rdev, upper_32_bits(ib->gpu_addr) & 0xFF);
2168         radeon_ring_write(rdev, ib->length_dw);
2169 }
2170
2171 int r600_ib_test(struct radeon_device *rdev)
2172 {
2173         struct radeon_ib *ib;
2174         uint32_t scratch;
2175         uint32_t tmp = 0;
2176         unsigned i;
2177         int r;
2178
2179         r = radeon_scratch_get(rdev, &scratch);
2180         if (r) {
2181                 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
2182                 return r;
2183         }
2184         WREG32(scratch, 0xCAFEDEAD);
2185         r = radeon_ib_get(rdev, &ib);
2186         if (r) {
2187                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
2188                 return r;
2189         }
2190         ib->ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
2191         ib->ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
2192         ib->ptr[2] = 0xDEADBEEF;
2193         ib->ptr[3] = PACKET2(0);
2194         ib->ptr[4] = PACKET2(0);
2195         ib->ptr[5] = PACKET2(0);
2196         ib->ptr[6] = PACKET2(0);
2197         ib->ptr[7] = PACKET2(0);
2198         ib->ptr[8] = PACKET2(0);
2199         ib->ptr[9] = PACKET2(0);
2200         ib->ptr[10] = PACKET2(0);
2201         ib->ptr[11] = PACKET2(0);
2202         ib->ptr[12] = PACKET2(0);
2203         ib->ptr[13] = PACKET2(0);
2204         ib->ptr[14] = PACKET2(0);
2205         ib->ptr[15] = PACKET2(0);
2206         ib->length_dw = 16;
2207         r = radeon_ib_schedule(rdev, ib);
2208         if (r) {
2209                 radeon_scratch_free(rdev, scratch);
2210                 radeon_ib_free(rdev, &ib);
2211                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
2212                 return r;
2213         }
2214         r = radeon_fence_wait(ib->fence, false);
2215         if (r) {
2216                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
2217                 return r;
2218         }
2219         for (i = 0; i < rdev->usec_timeout; i++) {
2220                 tmp = RREG32(scratch);
2221                 if (tmp == 0xDEADBEEF)
2222                         break;
2223                 DRM_UDELAY(1);
2224         }
2225         if (i < rdev->usec_timeout) {
2226                 DRM_INFO("ib test succeeded in %u usecs\n", i);
2227         } else {
2228                 DRM_ERROR("radeon: ib test failed (sracth(0x%04X)=0x%08X)\n",
2229                           scratch, tmp);
2230                 r = -EINVAL;
2231         }
2232         radeon_scratch_free(rdev, scratch);
2233         radeon_ib_free(rdev, &ib);
2234         return r;
2235 }
2236
2237 /*
2238  * Interrupts
2239  *
2240  * Interrupts use a ring buffer on r6xx/r7xx hardware.  It works pretty
2241  * the same as the CP ring buffer, but in reverse.  Rather than the CPU
2242  * writing to the ring and the GPU consuming, the GPU writes to the ring
2243  * and host consumes.  As the host irq handler processes interrupts, it
2244  * increments the rptr.  When the rptr catches up with the wptr, all the
2245  * current interrupts have been processed.
2246  */
2247
2248 void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size)
2249 {
2250         u32 rb_bufsz;
2251
2252         /* Align ring size */
2253         rb_bufsz = drm_order(ring_size / 4);
2254         ring_size = (1 << rb_bufsz) * 4;
2255         rdev->ih.ring_size = ring_size;
2256         rdev->ih.ptr_mask = rdev->ih.ring_size - 1;
2257         rdev->ih.rptr = 0;
2258 }
2259
2260 static int r600_ih_ring_alloc(struct radeon_device *rdev)
2261 {
2262         int r;
2263
2264         /* Allocate ring buffer */
2265         if (rdev->ih.ring_obj == NULL) {
2266                 r = radeon_bo_create(rdev, NULL, rdev->ih.ring_size,
2267                                      true,
2268                                      RADEON_GEM_DOMAIN_GTT,
2269                                      &rdev->ih.ring_obj);
2270                 if (r) {
2271                         DRM_ERROR("radeon: failed to create ih ring buffer (%d).\n", r);
2272                         return r;
2273                 }
2274                 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
2275                 if (unlikely(r != 0))
2276                         return r;
2277                 r = radeon_bo_pin(rdev->ih.ring_obj,
2278                                   RADEON_GEM_DOMAIN_GTT,
2279                                   &rdev->ih.gpu_addr);
2280                 if (r) {
2281                         radeon_bo_unreserve(rdev->ih.ring_obj);
2282                         DRM_ERROR("radeon: failed to pin ih ring buffer (%d).\n", r);
2283                         return r;
2284                 }
2285                 r = radeon_bo_kmap(rdev->ih.ring_obj,
2286                                    (void **)&rdev->ih.ring);
2287                 radeon_bo_unreserve(rdev->ih.ring_obj);
2288                 if (r) {
2289                         DRM_ERROR("radeon: failed to map ih ring buffer (%d).\n", r);
2290                         return r;
2291                 }
2292         }
2293         return 0;
2294 }
2295
2296 static void r600_ih_ring_fini(struct radeon_device *rdev)
2297 {
2298         int r;
2299         if (rdev->ih.ring_obj) {
2300                 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
2301                 if (likely(r == 0)) {
2302                         radeon_bo_kunmap(rdev->ih.ring_obj);
2303                         radeon_bo_unpin(rdev->ih.ring_obj);
2304                         radeon_bo_unreserve(rdev->ih.ring_obj);
2305                 }
2306                 radeon_bo_unref(&rdev->ih.ring_obj);
2307                 rdev->ih.ring = NULL;
2308                 rdev->ih.ring_obj = NULL;
2309         }
2310 }
2311
2312 static void r600_rlc_stop(struct radeon_device *rdev)
2313 {
2314
2315         if (rdev->family >= CHIP_RV770) {
2316                 /* r7xx asics need to soft reset RLC before halting */
2317                 WREG32(SRBM_SOFT_RESET, SOFT_RESET_RLC);
2318                 RREG32(SRBM_SOFT_RESET);
2319                 udelay(15000);
2320                 WREG32(SRBM_SOFT_RESET, 0);
2321                 RREG32(SRBM_SOFT_RESET);
2322         }
2323
2324         WREG32(RLC_CNTL, 0);
2325 }
2326
2327 static void r600_rlc_start(struct radeon_device *rdev)
2328 {
2329         WREG32(RLC_CNTL, RLC_ENABLE);
2330 }
2331
2332 static int r600_rlc_init(struct radeon_device *rdev)
2333 {
2334         u32 i;
2335         const __be32 *fw_data;
2336
2337         if (!rdev->rlc_fw)
2338                 return -EINVAL;
2339
2340         r600_rlc_stop(rdev);
2341
2342         WREG32(RLC_HB_BASE, 0);
2343         WREG32(RLC_HB_CNTL, 0);
2344         WREG32(RLC_HB_RPTR, 0);
2345         WREG32(RLC_HB_WPTR, 0);
2346         WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
2347         WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
2348         WREG32(RLC_MC_CNTL, 0);
2349         WREG32(RLC_UCODE_CNTL, 0);
2350
2351         fw_data = (const __be32 *)rdev->rlc_fw->data;
2352         if (rdev->family >= CHIP_RV770) {
2353                 for (i = 0; i < R700_RLC_UCODE_SIZE; i++) {
2354                         WREG32(RLC_UCODE_ADDR, i);
2355                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
2356                 }
2357         } else {
2358                 for (i = 0; i < RLC_UCODE_SIZE; i++) {
2359                         WREG32(RLC_UCODE_ADDR, i);
2360                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
2361                 }
2362         }
2363         WREG32(RLC_UCODE_ADDR, 0);
2364
2365         r600_rlc_start(rdev);
2366
2367         return 0;
2368 }
2369
2370 static void r600_enable_interrupts(struct radeon_device *rdev)
2371 {
2372         u32 ih_cntl = RREG32(IH_CNTL);
2373         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
2374
2375         ih_cntl |= ENABLE_INTR;
2376         ih_rb_cntl |= IH_RB_ENABLE;
2377         WREG32(IH_CNTL, ih_cntl);
2378         WREG32(IH_RB_CNTL, ih_rb_cntl);
2379         rdev->ih.enabled = true;
2380 }
2381
2382 static void r600_disable_interrupts(struct radeon_device *rdev)
2383 {
2384         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
2385         u32 ih_cntl = RREG32(IH_CNTL);
2386
2387         ih_rb_cntl &= ~IH_RB_ENABLE;
2388         ih_cntl &= ~ENABLE_INTR;
2389         WREG32(IH_RB_CNTL, ih_rb_cntl);
2390         WREG32(IH_CNTL, ih_cntl);
2391         /* set rptr, wptr to 0 */
2392         WREG32(IH_RB_RPTR, 0);
2393         WREG32(IH_RB_WPTR, 0);
2394         rdev->ih.enabled = false;
2395         rdev->ih.wptr = 0;
2396         rdev->ih.rptr = 0;
2397 }
2398
2399 static void r600_disable_interrupt_state(struct radeon_device *rdev)
2400 {
2401         u32 tmp;
2402
2403         WREG32(CP_INT_CNTL, 0);
2404         WREG32(GRBM_INT_CNTL, 0);
2405         WREG32(DxMODE_INT_MASK, 0);
2406         if (ASIC_IS_DCE3(rdev)) {
2407                 WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0);
2408                 WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0);
2409                 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2410                 WREG32(DC_HPD1_INT_CONTROL, tmp);
2411                 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2412                 WREG32(DC_HPD2_INT_CONTROL, tmp);
2413                 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2414                 WREG32(DC_HPD3_INT_CONTROL, tmp);
2415                 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2416                 WREG32(DC_HPD4_INT_CONTROL, tmp);
2417                 if (ASIC_IS_DCE32(rdev)) {
2418                         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2419                         WREG32(DC_HPD5_INT_CONTROL, tmp);
2420                         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2421                         WREG32(DC_HPD6_INT_CONTROL, tmp);
2422                 }
2423         } else {
2424                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
2425                 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
2426                 tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
2427                 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
2428                 tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
2429                 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
2430                 tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
2431                 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
2432         }
2433 }
2434
2435 int r600_irq_init(struct radeon_device *rdev)
2436 {
2437         int ret = 0;
2438         int rb_bufsz;
2439         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
2440
2441         /* allocate ring */
2442         ret = r600_ih_ring_alloc(rdev);
2443         if (ret)
2444                 return ret;
2445
2446         /* disable irqs */
2447         r600_disable_interrupts(rdev);
2448
2449         /* init rlc */
2450         ret = r600_rlc_init(rdev);
2451         if (ret) {
2452                 r600_ih_ring_fini(rdev);
2453                 return ret;
2454         }
2455
2456         /* setup interrupt control */
2457         /* set dummy read address to ring address */
2458         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
2459         interrupt_cntl = RREG32(INTERRUPT_CNTL);
2460         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
2461          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
2462          */
2463         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
2464         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
2465         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
2466         WREG32(INTERRUPT_CNTL, interrupt_cntl);
2467
2468         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
2469         rb_bufsz = drm_order(rdev->ih.ring_size / 4);
2470
2471         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
2472                       IH_WPTR_OVERFLOW_CLEAR |
2473                       (rb_bufsz << 1));
2474         /* WPTR writeback, not yet */
2475         /*ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;*/
2476         WREG32(IH_RB_WPTR_ADDR_LO, 0);
2477         WREG32(IH_RB_WPTR_ADDR_HI, 0);
2478
2479         WREG32(IH_RB_CNTL, ih_rb_cntl);
2480
2481         /* set rptr, wptr to 0 */
2482         WREG32(IH_RB_RPTR, 0);
2483         WREG32(IH_RB_WPTR, 0);
2484
2485         /* Default settings for IH_CNTL (disabled at first) */
2486         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10);
2487         /* RPTR_REARM only works if msi's are enabled */
2488         if (rdev->msi_enabled)
2489                 ih_cntl |= RPTR_REARM;
2490
2491 #ifdef __BIG_ENDIAN
2492         ih_cntl |= IH_MC_SWAP(IH_MC_SWAP_32BIT);
2493 #endif
2494         WREG32(IH_CNTL, ih_cntl);
2495
2496         /* force the active interrupt state to all disabled */
2497         r600_disable_interrupt_state(rdev);
2498
2499         /* enable irqs */
2500         r600_enable_interrupts(rdev);
2501
2502         return ret;
2503 }
2504
2505 void r600_irq_suspend(struct radeon_device *rdev)
2506 {
2507         r600_disable_interrupts(rdev);
2508         r600_rlc_stop(rdev);
2509 }
2510
2511 void r600_irq_fini(struct radeon_device *rdev)
2512 {
2513         r600_irq_suspend(rdev);
2514         r600_ih_ring_fini(rdev);
2515 }
2516
2517 int r600_irq_set(struct radeon_device *rdev)
2518 {
2519         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
2520         u32 mode_int = 0;
2521         u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0;
2522
2523         if (!rdev->irq.installed) {
2524                 WARN(1, "Can't enable IRQ/MSI because no handler is installed.\n");
2525                 return -EINVAL;
2526         }
2527         /* don't enable anything if the ih is disabled */
2528         if (!rdev->ih.enabled) {
2529                 r600_disable_interrupts(rdev);
2530                 /* force the active interrupt state to all disabled */
2531                 r600_disable_interrupt_state(rdev);
2532                 return 0;
2533         }
2534
2535         if (ASIC_IS_DCE3(rdev)) {
2536                 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
2537                 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
2538                 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
2539                 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
2540                 if (ASIC_IS_DCE32(rdev)) {
2541                         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
2542                         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
2543                 }
2544         } else {
2545                 hpd1 = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & ~DC_HPDx_INT_EN;
2546                 hpd2 = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & ~DC_HPDx_INT_EN;
2547                 hpd3 = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & ~DC_HPDx_INT_EN;
2548         }
2549
2550         if (rdev->irq.sw_int) {
2551                 DRM_DEBUG("r600_irq_set: sw int\n");
2552                 cp_int_cntl |= RB_INT_ENABLE;
2553         }
2554         if (rdev->irq.crtc_vblank_int[0]) {
2555                 DRM_DEBUG("r600_irq_set: vblank 0\n");
2556                 mode_int |= D1MODE_VBLANK_INT_MASK;
2557         }
2558         if (rdev->irq.crtc_vblank_int[1]) {
2559                 DRM_DEBUG("r600_irq_set: vblank 1\n");
2560                 mode_int |= D2MODE_VBLANK_INT_MASK;
2561         }
2562         if (rdev->irq.hpd[0]) {
2563                 DRM_DEBUG("r600_irq_set: hpd 1\n");
2564                 hpd1 |= DC_HPDx_INT_EN;
2565         }
2566         if (rdev->irq.hpd[1]) {
2567                 DRM_DEBUG("r600_irq_set: hpd 2\n");
2568                 hpd2 |= DC_HPDx_INT_EN;
2569         }
2570         if (rdev->irq.hpd[2]) {
2571                 DRM_DEBUG("r600_irq_set: hpd 3\n");
2572                 hpd3 |= DC_HPDx_INT_EN;
2573         }
2574         if (rdev->irq.hpd[3]) {
2575                 DRM_DEBUG("r600_irq_set: hpd 4\n");
2576                 hpd4 |= DC_HPDx_INT_EN;
2577         }
2578         if (rdev->irq.hpd[4]) {
2579                 DRM_DEBUG("r600_irq_set: hpd 5\n");
2580                 hpd5 |= DC_HPDx_INT_EN;
2581         }
2582         if (rdev->irq.hpd[5]) {
2583                 DRM_DEBUG("r600_irq_set: hpd 6\n");
2584                 hpd6 |= DC_HPDx_INT_EN;
2585         }
2586
2587         WREG32(CP_INT_CNTL, cp_int_cntl);
2588         WREG32(DxMODE_INT_MASK, mode_int);
2589         if (ASIC_IS_DCE3(rdev)) {
2590                 WREG32(DC_HPD1_INT_CONTROL, hpd1);
2591                 WREG32(DC_HPD2_INT_CONTROL, hpd2);
2592                 WREG32(DC_HPD3_INT_CONTROL, hpd3);
2593                 WREG32(DC_HPD4_INT_CONTROL, hpd4);
2594                 if (ASIC_IS_DCE32(rdev)) {
2595                         WREG32(DC_HPD5_INT_CONTROL, hpd5);
2596                         WREG32(DC_HPD6_INT_CONTROL, hpd6);
2597                 }
2598         } else {
2599                 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, hpd1);
2600                 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, hpd2);
2601                 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, hpd3);
2602         }
2603
2604         return 0;
2605 }
2606
2607 static inline void r600_irq_ack(struct radeon_device *rdev,
2608                                 u32 *disp_int,
2609                                 u32 *disp_int_cont,
2610                                 u32 *disp_int_cont2)
2611 {
2612         u32 tmp;
2613
2614         if (ASIC_IS_DCE3(rdev)) {
2615                 *disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
2616                 *disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
2617                 *disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
2618         } else {
2619                 *disp_int = RREG32(DISP_INTERRUPT_STATUS);
2620                 *disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
2621                 *disp_int_cont2 = 0;
2622         }
2623
2624         if (*disp_int & LB_D1_VBLANK_INTERRUPT)
2625                 WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
2626         if (*disp_int & LB_D1_VLINE_INTERRUPT)
2627                 WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
2628         if (*disp_int & LB_D2_VBLANK_INTERRUPT)
2629                 WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
2630         if (*disp_int & LB_D2_VLINE_INTERRUPT)
2631                 WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
2632         if (*disp_int & DC_HPD1_INTERRUPT) {
2633                 if (ASIC_IS_DCE3(rdev)) {
2634                         tmp = RREG32(DC_HPD1_INT_CONTROL);
2635                         tmp |= DC_HPDx_INT_ACK;
2636                         WREG32(DC_HPD1_INT_CONTROL, tmp);
2637                 } else {
2638                         tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
2639                         tmp |= DC_HPDx_INT_ACK;
2640                         WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
2641                 }
2642         }
2643         if (*disp_int & DC_HPD2_INTERRUPT) {
2644                 if (ASIC_IS_DCE3(rdev)) {
2645                         tmp = RREG32(DC_HPD2_INT_CONTROL);
2646                         tmp |= DC_HPDx_INT_ACK;
2647                         WREG32(DC_HPD2_INT_CONTROL, tmp);
2648                 } else {
2649                         tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
2650                         tmp |= DC_HPDx_INT_ACK;
2651                         WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
2652                 }
2653         }
2654         if (*disp_int_cont & DC_HPD3_INTERRUPT) {
2655                 if (ASIC_IS_DCE3(rdev)) {
2656                         tmp = RREG32(DC_HPD3_INT_CONTROL);
2657                         tmp |= DC_HPDx_INT_ACK;
2658                         WREG32(DC_HPD3_INT_CONTROL, tmp);
2659                 } else {
2660                         tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
2661                         tmp |= DC_HPDx_INT_ACK;
2662                         WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
2663                 }
2664         }
2665         if (*disp_int_cont & DC_HPD4_INTERRUPT) {
2666                 tmp = RREG32(DC_HPD4_INT_CONTROL);
2667                 tmp |= DC_HPDx_INT_ACK;
2668                 WREG32(DC_HPD4_INT_CONTROL, tmp);
2669         }
2670         if (ASIC_IS_DCE32(rdev)) {
2671                 if (*disp_int_cont2 & DC_HPD5_INTERRUPT) {
2672                         tmp = RREG32(DC_HPD5_INT_CONTROL);
2673                         tmp |= DC_HPDx_INT_ACK;
2674                         WREG32(DC_HPD5_INT_CONTROL, tmp);
2675                 }
2676                 if (*disp_int_cont2 & DC_HPD6_INTERRUPT) {
2677                         tmp = RREG32(DC_HPD5_INT_CONTROL);
2678                         tmp |= DC_HPDx_INT_ACK;
2679                         WREG32(DC_HPD6_INT_CONTROL, tmp);
2680                 }
2681         }
2682 }
2683
2684 void r600_irq_disable(struct radeon_device *rdev)
2685 {
2686         u32 disp_int, disp_int_cont, disp_int_cont2;
2687
2688         r600_disable_interrupts(rdev);
2689         /* Wait and acknowledge irq */
2690         mdelay(1);
2691         r600_irq_ack(rdev, &disp_int, &disp_int_cont, &disp_int_cont2);
2692         r600_disable_interrupt_state(rdev);
2693 }
2694
2695 static inline u32 r600_get_ih_wptr(struct radeon_device *rdev)
2696 {
2697         u32 wptr, tmp;
2698
2699         /* XXX use writeback */
2700         wptr = RREG32(IH_RB_WPTR);
2701
2702         if (wptr & RB_OVERFLOW) {
2703                 /* When a ring buffer overflow happen start parsing interrupt
2704                  * from the last not overwritten vector (wptr + 16). Hopefully
2705                  * this should allow us to catchup.
2706                  */
2707                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
2708                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
2709                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
2710                 tmp = RREG32(IH_RB_CNTL);
2711                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
2712                 WREG32(IH_RB_CNTL, tmp);
2713         }
2714         return (wptr & rdev->ih.ptr_mask);
2715 }
2716
2717 /*        r600 IV Ring
2718  * Each IV ring entry is 128 bits:
2719  * [7:0]    - interrupt source id
2720  * [31:8]   - reserved
2721  * [59:32]  - interrupt source data
2722  * [127:60]  - reserved
2723  *
2724  * The basic interrupt vector entries
2725  * are decoded as follows:
2726  * src_id  src_data  description
2727  *      1         0  D1 Vblank
2728  *      1         1  D1 Vline
2729  *      5         0  D2 Vblank
2730  *      5         1  D2 Vline
2731  *     19         0  FP Hot plug detection A
2732  *     19         1  FP Hot plug detection B
2733  *     19         2  DAC A auto-detection
2734  *     19         3  DAC B auto-detection
2735  *    176         -  CP_INT RB
2736  *    177         -  CP_INT IB1
2737  *    178         -  CP_INT IB2
2738  *    181         -  EOP Interrupt
2739  *    233         -  GUI Idle
2740  *
2741  * Note, these are based on r600 and may need to be
2742  * adjusted or added to on newer asics
2743  */
2744
2745 int r600_irq_process(struct radeon_device *rdev)
2746 {
2747         u32 wptr = r600_get_ih_wptr(rdev);
2748         u32 rptr = rdev->ih.rptr;
2749         u32 src_id, src_data;
2750         u32 ring_index, disp_int, disp_int_cont, disp_int_cont2;
2751         unsigned long flags;
2752         bool queue_hotplug = false;
2753
2754         DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
2755         if (!rdev->ih.enabled)
2756                 return IRQ_NONE;
2757
2758         spin_lock_irqsave(&rdev->ih.lock, flags);
2759
2760         if (rptr == wptr) {
2761                 spin_unlock_irqrestore(&rdev->ih.lock, flags);
2762                 return IRQ_NONE;
2763         }
2764         if (rdev->shutdown) {
2765                 spin_unlock_irqrestore(&rdev->ih.lock, flags);
2766                 return IRQ_NONE;
2767         }
2768
2769 restart_ih:
2770         /* display interrupts */
2771         r600_irq_ack(rdev, &disp_int, &disp_int_cont, &disp_int_cont2);
2772
2773         rdev->ih.wptr = wptr;
2774         while (rptr != wptr) {
2775                 /* wptr/rptr are in bytes! */
2776                 ring_index = rptr / 4;
2777                 src_id =  rdev->ih.ring[ring_index] & 0xff;
2778                 src_data = rdev->ih.ring[ring_index + 1] & 0xfffffff;
2779
2780                 switch (src_id) {
2781                 case 1: /* D1 vblank/vline */
2782                         switch (src_data) {
2783                         case 0: /* D1 vblank */
2784                                 if (disp_int & LB_D1_VBLANK_INTERRUPT) {
2785                                         drm_handle_vblank(rdev->ddev, 0);
2786                                         rdev->pm.vblank_sync = true;
2787                                         wake_up(&rdev->irq.vblank_queue);
2788                                         disp_int &= ~LB_D1_VBLANK_INTERRUPT;
2789                                         DRM_DEBUG("IH: D1 vblank\n");
2790                                 }
2791                                 break;
2792                         case 1: /* D1 vline */
2793                                 if (disp_int & LB_D1_VLINE_INTERRUPT) {
2794                                         disp_int &= ~LB_D1_VLINE_INTERRUPT;
2795                                         DRM_DEBUG("IH: D1 vline\n");
2796                                 }
2797                                 break;
2798                         default:
2799                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2800                                 break;
2801                         }
2802                         break;
2803                 case 5: /* D2 vblank/vline */
2804                         switch (src_data) {
2805                         case 0: /* D2 vblank */
2806                                 if (disp_int & LB_D2_VBLANK_INTERRUPT) {
2807                                         drm_handle_vblank(rdev->ddev, 1);
2808                                         rdev->pm.vblank_sync = true;
2809                                         wake_up(&rdev->irq.vblank_queue);
2810                                         disp_int &= ~LB_D2_VBLANK_INTERRUPT;
2811                                         DRM_DEBUG("IH: D2 vblank\n");
2812                                 }
2813                                 break;
2814                         case 1: /* D1 vline */
2815                                 if (disp_int & LB_D2_VLINE_INTERRUPT) {
2816                                         disp_int &= ~LB_D2_VLINE_INTERRUPT;
2817                                         DRM_DEBUG("IH: D2 vline\n");
2818                                 }
2819                                 break;
2820                         default:
2821                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2822                                 break;
2823                         }
2824                         break;
2825                 case 19: /* HPD/DAC hotplug */
2826                         switch (src_data) {
2827                         case 0:
2828                                 if (disp_int & DC_HPD1_INTERRUPT) {
2829                                         disp_int &= ~DC_HPD1_INTERRUPT;
2830                                         queue_hotplug = true;
2831                                         DRM_DEBUG("IH: HPD1\n");
2832                                 }
2833                                 break;
2834                         case 1:
2835                                 if (disp_int & DC_HPD2_INTERRUPT) {
2836                                         disp_int &= ~DC_HPD2_INTERRUPT;
2837                                         queue_hotplug = true;
2838                                         DRM_DEBUG("IH: HPD2\n");
2839                                 }
2840                                 break;
2841                         case 4:
2842                                 if (disp_int_cont & DC_HPD3_INTERRUPT) {
2843                                         disp_int_cont &= ~DC_HPD3_INTERRUPT;
2844                                         queue_hotplug = true;
2845                                         DRM_DEBUG("IH: HPD3\n");
2846                                 }
2847                                 break;
2848                         case 5:
2849                                 if (disp_int_cont & DC_HPD4_INTERRUPT) {
2850                                         disp_int_cont &= ~DC_HPD4_INTERRUPT;
2851                                         queue_hotplug = true;
2852                                         DRM_DEBUG("IH: HPD4\n");
2853                                 }
2854                                 break;
2855                         case 10:
2856                                 if (disp_int_cont2 & DC_HPD5_INTERRUPT) {
2857                                         disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
2858                                         queue_hotplug = true;
2859                                         DRM_DEBUG("IH: HPD5\n");
2860                                 }
2861                                 break;
2862                         case 12:
2863                                 if (disp_int_cont2 & DC_HPD6_INTERRUPT) {
2864                                         disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
2865                                         queue_hotplug = true;
2866                                         DRM_DEBUG("IH: HPD6\n");
2867                                 }
2868                                 break;
2869                         default:
2870                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2871                                 break;
2872                         }
2873                         break;
2874                 case 176: /* CP_INT in ring buffer */
2875                 case 177: /* CP_INT in IB1 */
2876                 case 178: /* CP_INT in IB2 */
2877                         DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
2878                         radeon_fence_process(rdev);
2879                         break;
2880                 case 181: /* CP EOP event */
2881                         DRM_DEBUG("IH: CP EOP\n");
2882                         break;
2883                 default:
2884                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2885                         break;
2886                 }
2887
2888                 /* wptr/rptr are in bytes! */
2889                 rptr += 16;
2890                 rptr &= rdev->ih.ptr_mask;
2891         }
2892         /* make sure wptr hasn't changed while processing */
2893         wptr = r600_get_ih_wptr(rdev);
2894         if (wptr != rdev->ih.wptr)
2895                 goto restart_ih;
2896         if (queue_hotplug)
2897                 queue_work(rdev->wq, &rdev->hotplug_work);
2898         rdev->ih.rptr = rptr;
2899         WREG32(IH_RB_RPTR, rdev->ih.rptr);
2900         spin_unlock_irqrestore(&rdev->ih.lock, flags);
2901         return IRQ_HANDLED;
2902 }
2903
2904 /*
2905  * Debugfs info
2906  */
2907 #if defined(CONFIG_DEBUG_FS)
2908
2909 static int r600_debugfs_cp_ring_info(struct seq_file *m, void *data)
2910 {
2911         struct drm_info_node *node = (struct drm_info_node *) m->private;
2912         struct drm_device *dev = node->minor->dev;
2913         struct radeon_device *rdev = dev->dev_private;
2914         unsigned count, i, j;
2915
2916         radeon_ring_free_size(rdev);
2917         count = (rdev->cp.ring_size / 4) - rdev->cp.ring_free_dw;
2918         seq_printf(m, "CP_STAT 0x%08x\n", RREG32(CP_STAT));
2919         seq_printf(m, "CP_RB_WPTR 0x%08x\n", RREG32(CP_RB_WPTR));
2920         seq_printf(m, "CP_RB_RPTR 0x%08x\n", RREG32(CP_RB_RPTR));
2921         seq_printf(m, "driver's copy of the CP_RB_WPTR 0x%08x\n", rdev->cp.wptr);
2922         seq_printf(m, "driver's copy of the CP_RB_RPTR 0x%08x\n", rdev->cp.rptr);
2923         seq_printf(m, "%u free dwords in ring\n", rdev->cp.ring_free_dw);
2924         seq_printf(m, "%u dwords in ring\n", count);
2925         i = rdev->cp.rptr;
2926         for (j = 0; j <= count; j++) {
2927                 seq_printf(m, "r[%04d]=0x%08x\n", i, rdev->cp.ring[i]);
2928                 i = (i + 1) & rdev->cp.ptr_mask;
2929         }
2930         return 0;
2931 }
2932
2933 static int r600_debugfs_mc_info(struct seq_file *m, void *data)
2934 {
2935         struct drm_info_node *node = (struct drm_info_node *) m->private;
2936         struct drm_device *dev = node->minor->dev;
2937         struct radeon_device *rdev = dev->dev_private;
2938
2939         DREG32_SYS(m, rdev, R_000E50_SRBM_STATUS);
2940         DREG32_SYS(m, rdev, VM_L2_STATUS);
2941         return 0;
2942 }
2943
2944 static struct drm_info_list r600_mc_info_list[] = {
2945         {"r600_mc_info", r600_debugfs_mc_info, 0, NULL},
2946         {"r600_ring_info", r600_debugfs_cp_ring_info, 0, NULL},
2947 };
2948 #endif
2949
2950 int r600_debugfs_mc_info_init(struct radeon_device *rdev)
2951 {
2952 #if defined(CONFIG_DEBUG_FS)
2953         return radeon_debugfs_add_files(rdev, r600_mc_info_list, ARRAY_SIZE(r600_mc_info_list));
2954 #else
2955         return 0;
2956 #endif
2957 }
2958
2959 /**
2960  * r600_ioctl_wait_idle - flush host path cache on wait idle ioctl
2961  * rdev: radeon device structure
2962  * bo: buffer object struct which userspace is waiting for idle
2963  *
2964  * Some R6XX/R7XX doesn't seems to take into account HDP flush performed
2965  * through ring buffer, this leads to corruption in rendering, see
2966  * http://bugzilla.kernel.org/show_bug.cgi?id=15186 to avoid this we
2967  * directly perform HDP flush by writing register through MMIO.
2968  */
2969 void r600_ioctl_wait_idle(struct radeon_device *rdev, struct radeon_bo *bo)
2970 {
2971         WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2972 }