drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23 #include <linux/firmware.h>
  24 #include "drmP.h"
  25 #include "amdgpu.h"
  26 #include "amdgpu_gfx.h"
  27 #include "soc15.h"
  28 #include "soc15d.h"
  29
  30 #include "vega10/soc15ip.h"
  31 #include "vega10/GC/gc_9_0_offset.h"
  32 #include "vega10/GC/gc_9_0_sh_mask.h"
  33 #include "vega10/vega10_enum.h"
  34 #include "vega10/HDP/hdp_4_0_offset.h"
  35
  36 #include "soc15_common.h"
  37 #include "clearstate_gfx9.h"
  38 #include "v9_structs.h"
  39
  40 #define GFX9_NUM_GFX_RINGS     1
  41 #define GFX9_NUM_COMPUTE_RINGS 8
  42 #define GFX9_NUM_SE             4
  43 #define RLCG_UCODE_LOADING_START_ADDRESS 0x2000
  44
  45 MODULE_FIRMWARE("amdgpu/vega10_ce.bin");
  46 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin");
  47 MODULE_FIRMWARE("amdgpu/vega10_me.bin");
  48 MODULE_FIRMWARE("amdgpu/vega10_mec.bin");
  49 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin");
  50 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin");
  51
  52 static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] =
  53 {
  54         {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE),
  55                 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0)},
  56         {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID1_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID1_SIZE),
  57                 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID1), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID1)},
  58         {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID2_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID2_SIZE),
  59                 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID2), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID2)},
  60         {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID3_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID3_SIZE),
  61                 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID3), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID3)},
  62         {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID4_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID4_SIZE),
  63                 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID4), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID4)},
  64         {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID5_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID5_SIZE),
  65                 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID5), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID5)},
  66         {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID6_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID6_SIZE),
  67                 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID6), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID6)},
  68         {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID7_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID7_SIZE),
  69                 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID7), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID7)},
  70         {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID8_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID8_SIZE),
  71                 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID8), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID8)},
  72         {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID9_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID9_SIZE),
  73                 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID9), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID9)},
  74         {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID10_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID10_SIZE),
  75                 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID10), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID10)},
  76         {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID11_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID11_SIZE),
  77                 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID11), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID11)},
  78         {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID12_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID12_SIZE),
  79                 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID12), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID12)},
  80         {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID13_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID13_SIZE),
  81                 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID13), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID13)},
  82         {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID14_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID14_SIZE),
  83                 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID14), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID14)},
  84         {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID15_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID15_SIZE),
  85                 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID15), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID15)}
  86 };
  87
  88 static const u32 golden_settings_gc_9_0[] =
  89 {
  90         SOC15_REG_OFFSET(GC, 0, mmDB_DEBUG2), 0xf00ffeff, 0x00000400,
  91         SOC15_REG_OFFSET(GC, 0, mmGB_GPU_ID), 0x0000000f, 0x00000000,
  92         SOC15_REG_OFFSET(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3), 0x00000003, 0x82400024,
  93         SOC15_REG_OFFSET(GC, 0, mmPA_SC_ENHANCE), 0x3fffffff, 0x00000001,
  94         SOC15_REG_OFFSET(GC, 0, mmPA_SC_LINE_STIPPLE_STATE), 0x0000ff0f, 0x00000000,
  95         SOC15_REG_OFFSET(GC, 0, mmTA_CNTL_AUX), 0xfffffeef, 0x010b0000,
  96         SOC15_REG_OFFSET(GC, 0, mmTCP_CHAN_STEER_HI), 0xffffffff, 0x4a2c0e68,
  97         SOC15_REG_OFFSET(GC, 0, mmTCP_CHAN_STEER_LO), 0xffffffff, 0xb5d3f197,
  98         SOC15_REG_OFFSET(GC, 0, mmVGT_GS_MAX_WAVE_ID), 0x00000fff, 0x000003ff
  99 };
 100
 101 static const u32 golden_settings_gc_9_0_vg10[] =
 102 {
 103         SOC15_REG_OFFSET(GC, 0, mmCB_HW_CONTROL), 0x0000f000, 0x00012107,
 104         SOC15_REG_OFFSET(GC, 0, mmCB_HW_CONTROL_3), 0x30000000, 0x10000000,
 105         SOC15_REG_OFFSET(GC, 0, mmGB_ADDR_CONFIG), 0xffff77ff, 0x2a114042,
 106         SOC15_REG_OFFSET(GC, 0, mmGB_ADDR_CONFIG_READ), 0xffff77ff, 0x2a114042,
 107         SOC15_REG_OFFSET(GC, 0, mmPA_SC_ENHANCE_1), 0x00008000, 0x00048000,
 108         SOC15_REG_OFFSET(GC, 0, mmRMI_UTCL1_CNTL2), 0x00030000, 0x00020000,
 109         SOC15_REG_OFFSET(GC, 0, mmTD_CNTL), 0x00001800, 0x00000800,
 110         SOC15_REG_OFFSET(GC, 0, mmSPI_CONFIG_CNTL_1),0x0000000f, 0x00000007
 111 };
 112
 113 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
 114
 115 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev);
 116 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);
 117 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
 118 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
 119 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
 120                                  struct amdgpu_cu_info *cu_info);
 121 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
 122 static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance);
 123
 124 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
 125 {
 126         switch (adev->asic_type) {
 127         case CHIP_VEGA10:
 128                 amdgpu_program_register_sequence(adev,
 129                                                  golden_settings_gc_9_0,
 130                                                  (const u32)ARRAY_SIZE(golden_settings_gc_9_0));
 131                 amdgpu_program_register_sequence(adev,
 132                                                  golden_settings_gc_9_0_vg10,
 133                                                  (const u32)ARRAY_SIZE(golden_settings_gc_9_0_vg10));
 134                 break;
 135         default:
 136                 break;
 137         }
 138 }
 139
 140 static void gfx_v9_0_scratch_init(struct amdgpu_device *adev)
 141 {
 142         adev->gfx.scratch.num_reg = 7;
 143         adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
 144         adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
 145 }
 146
 147 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
 148                                        bool wc, uint32_t reg, uint32_t val)
 149 {
 150         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
 151         amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
 152                                 WRITE_DATA_DST_SEL(0) |
 153                                 (wc ? WR_CONFIRM : 0));
 154         amdgpu_ring_write(ring, reg);
 155         amdgpu_ring_write(ring, 0);
 156         amdgpu_ring_write(ring, val);
 157 }
 158
 159 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
 160                                   int mem_space, int opt, uint32_t addr0,
 161                                   uint32_t addr1, uint32_t ref, uint32_t mask,
 162                                   uint32_t inv)
 163 {
 164         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
 165         amdgpu_ring_write(ring,
 166                                  /* memory (1) or register (0) */
 167                                  (WAIT_REG_MEM_MEM_SPACE(mem_space) |
 168                                  WAIT_REG_MEM_OPERATION(opt) | /* wait */
 169                                  WAIT_REG_MEM_FUNCTION(3) |  /* equal */
 170                                  WAIT_REG_MEM_ENGINE(eng_sel)));
 171
 172         if (mem_space)
 173                 BUG_ON(addr0 & 0x3); /* Dword align */
 174         amdgpu_ring_write(ring, addr0);
 175         amdgpu_ring_write(ring, addr1);
 176         amdgpu_ring_write(ring, ref);
 177         amdgpu_ring_write(ring, mask);
 178         amdgpu_ring_write(ring, inv); /* poll interval */
 179 }
 180
 181 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
 182 {
 183         struct amdgpu_device *adev = ring->adev;
 184         uint32_t scratch;
 185         uint32_t tmp = 0;
 186         unsigned i;
 187         int r;
 188
 189         r = amdgpu_gfx_scratch_get(adev, &scratch);
 190         if (r) {
 191                 DRM_ERROR("amdgpu: cp failed to get scratch reg (%d).\n", r);
 192                 return r;
 193         }
 194         WREG32(scratch, 0xCAFEDEAD);
 195         r = amdgpu_ring_alloc(ring, 3);
 196         if (r) {
 197                 DRM_ERROR("amdgpu: cp failed to lock ring %d (%d).\n",
 198                           ring->idx, r);
 199                 amdgpu_gfx_scratch_free(adev, scratch);
 200                 return r;
 201         }
 202         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
 203         amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
 204         amdgpu_ring_write(ring, 0xDEADBEEF);
 205         amdgpu_ring_commit(ring);
 206
 207         for (i = 0; i < adev->usec_timeout; i++) {
 208                 tmp = RREG32(scratch);
 209                 if (tmp == 0xDEADBEEF)
 210                         break;
 211                 DRM_UDELAY(1);
 212         }
 213         if (i < adev->usec_timeout) {
 214                 DRM_INFO("ring test on %d succeeded in %d usecs\n",
 215                          ring->idx, i);
 216         } else {
 217                 DRM_ERROR("amdgpu: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
 218                           ring->idx, scratch, tmp);
 219                 r = -EINVAL;
 220         }
 221         amdgpu_gfx_scratch_free(adev, scratch);
 222         return r;
 223 }
 224
 225 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 226 {
 227         struct amdgpu_device *adev = ring->adev;
 228         struct amdgpu_ib ib;
 229         struct dma_fence *f = NULL;
 230         uint32_t scratch;
 231         uint32_t tmp = 0;
 232         long r;
 233
 234         r = amdgpu_gfx_scratch_get(adev, &scratch);
 235         if (r) {
 236                 DRM_ERROR("amdgpu: failed to get scratch reg (%ld).\n", r);
 237                 return r;
 238         }
 239         WREG32(scratch, 0xCAFEDEAD);
 240         memset(&ib, 0, sizeof(ib));
 241         r = amdgpu_ib_get(adev, NULL, 256, &ib);
 242         if (r) {
 243                 DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
 244                 goto err1;
 245         }
 246         ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
 247         ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START));
 248         ib.ptr[2] = 0xDEADBEEF;
 249         ib.length_dw = 3;
 250
 251         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
 252         if (r)
 253                 goto err2;
 254
 255         r = dma_fence_wait_timeout(f, false, timeout);
 256         if (r == 0) {
 257                 DRM_ERROR("amdgpu: IB test timed out.\n");
 258                 r = -ETIMEDOUT;
 259                 goto err2;
 260         } else if (r < 0) {
 261                 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
 262                 goto err2;
 263         }
 264         tmp = RREG32(scratch);
 265         if (tmp == 0xDEADBEEF) {
 266                 DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
 267                 r = 0;
 268         } else {
 269                 DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n",
 270                           scratch, tmp);
 271                 r = -EINVAL;
 272         }
 273 err2:
 274         amdgpu_ib_free(adev, &ib, NULL);
 275         dma_fence_put(f);
 276 err1:
 277         amdgpu_gfx_scratch_free(adev, scratch);
 278         return r;
 279 }
 280
 281 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
 282 {
 283         const char *chip_name;
 284         char fw_name[30];
 285         int err;
 286         struct amdgpu_firmware_info *info = NULL;
 287         const struct common_firmware_header *header = NULL;
 288         const struct gfx_firmware_header_v1_0 *cp_hdr;
 289
 290         DRM_DEBUG("\n");
 291
 292         switch (adev->asic_type) {
 293         case CHIP_VEGA10:
 294                 chip_name = "vega10";
 295                 break;
 296         default:
 297                 BUG();
 298         }
 299
 300         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
 301         err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
 302         if (err)
 303                 goto out;
 304         err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
 305         if (err)
 306                 goto out;
 307         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
 308         adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
 309         adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
 310
 311         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
 312         err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
 313         if (err)
 314                 goto out;
 315         err = amdgpu_ucode_validate(adev->gfx.me_fw);
 316         if (err)
 317                 goto out;
 318         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
 319         adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
 320         adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
 321
 322         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
 323         err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
 324         if (err)
 325                 goto out;
 326         err = amdgpu_ucode_validate(adev->gfx.ce_fw);
 327         if (err)
 328                 goto out;
 329         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
 330         adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
 331         adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
 332
 333         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
 334         err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
 335         if (err)
 336                 goto out;
 337         err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
 338         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.rlc_fw->data;
 339         adev->gfx.rlc_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
 340         adev->gfx.rlc_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
 341
 342         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
 343         err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
 344         if (err)
 345                 goto out;
 346         err = amdgpu_ucode_validate(adev->gfx.mec_fw);
 347         if (err)
 348                 goto out;
 349         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
 350         adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
 351         adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
 352
 353
 354         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
 355         err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
 356         if (!err) {
 357                 err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
 358                 if (err)
 359                         goto out;
 360                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
 361                 adev->gfx.mec2_fw->data;
 362                 adev->gfx.mec2_fw_version =
 363                 le32_to_cpu(cp_hdr->header.ucode_version);
 364                 adev->gfx.mec2_feature_version =
 365                 le32_to_cpu(cp_hdr->ucode_feature_version);
 366         } else {
 367                 err = 0;
 368                 adev->gfx.mec2_fw = NULL;
 369         }
 370
 371         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 372                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];
 373                 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;
 374                 info->fw = adev->gfx.pfp_fw;
 375                 header = (const struct common_firmware_header *)info->fw->data;
 376                 adev->firmware.fw_size +=
 377                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
 378
 379                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];
 380                 info->ucode_id = AMDGPU_UCODE_ID_CP_ME;
 381                 info->fw = adev->gfx.me_fw;
 382                 header = (const struct common_firmware_header *)info->fw->data;
 383                 adev->firmware.fw_size +=
 384                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
 385
 386                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE];
 387                 info->ucode_id = AMDGPU_UCODE_ID_CP_CE;
 388                 info->fw = adev->gfx.ce_fw;
 389                 header = (const struct common_firmware_header *)info->fw->data;
 390                 adev->firmware.fw_size +=
 391                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
 392
 393                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
 394                 info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
 395                 info->fw = adev->gfx.rlc_fw;
 396                 header = (const struct common_firmware_header *)info->fw->data;
 397                 adev->firmware.fw_size +=
 398                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
 399
 400                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
 401                 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
 402                 info->fw = adev->gfx.mec_fw;
 403                 header = (const struct common_firmware_header *)info->fw->data;
 404                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
 405                 adev->firmware.fw_size +=
 406                         ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
 407
 408                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT];
 409                 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT;
 410                 info->fw = adev->gfx.mec_fw;
 411                 adev->firmware.fw_size +=
 412                         ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
 413
 414                 if (adev->gfx.mec2_fw) {
 415                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2];
 416                         info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
 417                         info->fw = adev->gfx.mec2_fw;
 418                         header = (const struct common_firmware_header *)info->fw->data;
 419                         cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
 420                         adev->firmware.fw_size +=
 421                                 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
 422                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT];
 423                         info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT;
 424                         info->fw = adev->gfx.mec2_fw;
 425                         adev->firmware.fw_size +=
 426                                 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
 427                 }
 428
 429         }
 430
 431 out:
 432         if (err) {
 433                 dev_err(adev->dev,
 434                         "gfx9: Failed to load firmware \"%s\"\n",
 435                         fw_name);
 436                 release_firmware(adev->gfx.pfp_fw);
 437                 adev->gfx.pfp_fw = NULL;
 438                 release_firmware(adev->gfx.me_fw);
 439                 adev->gfx.me_fw = NULL;
 440                 release_firmware(adev->gfx.ce_fw);
 441                 adev->gfx.ce_fw = NULL;
 442                 release_firmware(adev->gfx.rlc_fw);
 443                 adev->gfx.rlc_fw = NULL;
 444                 release_firmware(adev->gfx.mec_fw);
 445                 adev->gfx.mec_fw = NULL;
 446                 release_firmware(adev->gfx.mec2_fw);
 447                 adev->gfx.mec2_fw = NULL;
 448         }
 449         return err;
 450 }
 451
 452 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
 453 {
 454         int r;
 455
 456         if (adev->gfx.mec.hpd_eop_obj) {
 457                 r = amdgpu_bo_reserve(adev->gfx.mec.hpd_eop_obj, false);
 458                 if (unlikely(r != 0))
 459                         dev_warn(adev->dev, "(%d) reserve HPD EOP bo failed\n", r);
 460                 amdgpu_bo_unpin(adev->gfx.mec.hpd_eop_obj);
 461                 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
 462
 463                 amdgpu_bo_unref(&adev->gfx.mec.hpd_eop_obj);
 464                 adev->gfx.mec.hpd_eop_obj = NULL;
 465         }
 466         if (adev->gfx.mec.mec_fw_obj) {
 467                 r = amdgpu_bo_reserve(adev->gfx.mec.mec_fw_obj, false);
 468                 if (unlikely(r != 0))
 469                         dev_warn(adev->dev, "(%d) reserve mec firmware bo failed\n", r);
 470                 amdgpu_bo_unpin(adev->gfx.mec.mec_fw_obj);
 471                 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
 472
 473                 amdgpu_bo_unref(&adev->gfx.mec.mec_fw_obj);
 474                 adev->gfx.mec.mec_fw_obj = NULL;
 475         }
 476 }
 477
 478 #define MEC_HPD_SIZE 2048
 479
 480 static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
 481 {
 482         int r;
 483         u32 *hpd;
 484         const __le32 *fw_data;
 485         unsigned fw_size;
 486         u32 *fw;
 487
 488         const struct gfx_firmware_header_v1_0 *mec_hdr;
 489
 490         /*
 491          * we assign only 1 pipe because all other pipes will
 492          * be handled by KFD
 493          */
 494         adev->gfx.mec.num_mec = 1;
 495         adev->gfx.mec.num_pipe = 1;
 496         adev->gfx.mec.num_queue = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe * 8;
 497
 498         if (adev->gfx.mec.hpd_eop_obj == NULL) {
 499                 r = amdgpu_bo_create(adev,
 500                                      adev->gfx.mec.num_queue * MEC_HPD_SIZE,
 501                                      PAGE_SIZE, true,
 502                                      AMDGPU_GEM_DOMAIN_GTT, 0, NULL, NULL,
 503                                      &adev->gfx.mec.hpd_eop_obj);
 504                 if (r) {
 505                         dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
 506                         return r;
 507                 }
 508         }
 509
 510         r = amdgpu_bo_reserve(adev->gfx.mec.hpd_eop_obj, false);
 511         if (unlikely(r != 0)) {
 512                 gfx_v9_0_mec_fini(adev);
 513                 return r;
 514         }
 515         r = amdgpu_bo_pin(adev->gfx.mec.hpd_eop_obj, AMDGPU_GEM_DOMAIN_GTT,
 516                           &adev->gfx.mec.hpd_eop_gpu_addr);
 517         if (r) {
 518                 dev_warn(adev->dev, "(%d) pin HDP EOP bo failed\n", r);
 519                 gfx_v9_0_mec_fini(adev);
 520                 return r;
 521         }
 522         r = amdgpu_bo_kmap(adev->gfx.mec.hpd_eop_obj, (void **)&hpd);
 523         if (r) {
 524                 dev_warn(adev->dev, "(%d) map HDP EOP bo failed\n", r);
 525                 gfx_v9_0_mec_fini(adev);
 526                 return r;
 527         }
 528
 529         memset(hpd, 0, adev->gfx.mec.hpd_eop_obj->tbo.mem.size);
 530
 531         amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
 532         amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
 533
 534         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
 535
 536         fw_data = (const __le32 *)
 537                 (adev->gfx.mec_fw->data +
 538                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
 539         fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
 540
 541         if (adev->gfx.mec.mec_fw_obj == NULL) {
 542                 r = amdgpu_bo_create(adev,
 543                         mec_hdr->header.ucode_size_bytes,
 544                         PAGE_SIZE, true,
 545                         AMDGPU_GEM_DOMAIN_GTT, 0, NULL, NULL,
 546                         &adev->gfx.mec.mec_fw_obj);
 547                 if (r) {
 548                         dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
 549                         return r;
 550                 }
 551         }
 552
 553         r = amdgpu_bo_reserve(adev->gfx.mec.mec_fw_obj, false);
 554         if (unlikely(r != 0)) {
 555                 gfx_v9_0_mec_fini(adev);
 556                 return r;
 557         }
 558         r = amdgpu_bo_pin(adev->gfx.mec.mec_fw_obj, AMDGPU_GEM_DOMAIN_GTT,
 559                         &adev->gfx.mec.mec_fw_gpu_addr);
 560         if (r) {
 561                 dev_warn(adev->dev, "(%d) pin mec firmware bo failed\n", r);
 562                 gfx_v9_0_mec_fini(adev);
 563                 return r;
 564         }
 565         r = amdgpu_bo_kmap(adev->gfx.mec.mec_fw_obj, (void **)&fw);
 566         if (r) {
 567                 dev_warn(adev->dev, "(%d) map firmware bo failed\n", r);
 568                 gfx_v9_0_mec_fini(adev);
 569                 return r;
 570         }
 571         memcpy(fw, fw_data, fw_size);
 572
 573         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
 574         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
 575
 576
 577         return 0;
 578 }
 579
 580 static void gfx_v9_0_kiq_fini(struct amdgpu_device *adev)
 581 {
 582         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
 583
 584         amdgpu_bo_free_kernel(&kiq->eop_obj, &kiq->eop_gpu_addr, NULL);
 585 }
 586
 587 static int gfx_v9_0_kiq_init(struct amdgpu_device *adev)
 588 {
 589         int r;
 590         u32 *hpd;
 591         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
 592
 593         r = amdgpu_bo_create_kernel(adev, MEC_HPD_SIZE, PAGE_SIZE,
 594                                     AMDGPU_GEM_DOMAIN_GTT, &kiq->eop_obj,
 595                                     &kiq->eop_gpu_addr, (void **)&hpd);
 596         if (r) {
 597                 dev_warn(adev->dev, "failed to create KIQ bo (%d).\n", r);
 598                 return r;
 599         }
 600
 601         memset(hpd, 0, MEC_HPD_SIZE);
 602
 603         amdgpu_bo_kunmap(kiq->eop_obj);
 604
 605         return 0;
 606 }
 607
 608 static int gfx_v9_0_kiq_init_ring(struct amdgpu_device *adev,
 609                                   struct amdgpu_ring *ring,
 610                                   struct amdgpu_irq_src *irq)
 611 {
 612         int r = 0;
 613
 614         r = amdgpu_wb_get(adev, &adev->virt.reg_val_offs);
 615         if (r)
 616                 return r;
 617
 618         ring->adev = NULL;
 619         ring->ring_obj = NULL;
 620         ring->use_doorbell = true;
 621         ring->doorbell_index = AMDGPU_DOORBELL_KIQ;
 622         if (adev->gfx.mec2_fw) {
 623                 ring->me = 2;
 624                 ring->pipe = 0;
 625         } else {
 626                 ring->me = 1;
 627                 ring->pipe = 1;
 628         }
 629
 630         irq->data = ring;
 631         ring->queue = 0;
 632         sprintf(ring->name, "kiq %d.%d.%d", ring->me, ring->pipe, ring->queue);
 633         r = amdgpu_ring_init(adev, ring, 1024,
 634                              irq, AMDGPU_CP_KIQ_IRQ_DRIVER0);
 635         if (r)
 636                 dev_warn(adev->dev, "(%d) failed to init kiq ring\n", r);
 637
 638         return r;
 639 }
 640 static void gfx_v9_0_kiq_free_ring(struct amdgpu_ring *ring,
 641                                    struct amdgpu_irq_src *irq)
 642 {
 643         amdgpu_wb_free(ring->adev, ring->adev->virt.reg_val_offs);
 644         amdgpu_ring_fini(ring);
 645         irq->data = NULL;
 646 }
 647
 648 /* create MQD for each compute queue */
 649 static int gfx_v9_0_compute_mqd_soft_init(struct amdgpu_device *adev)
 650 {
 651         struct amdgpu_ring *ring = NULL;
 652         int r, i;
 653
 654         /* create MQD for KIQ */
 655         ring = &adev->gfx.kiq.ring;
 656         if (!ring->mqd_obj) {
 657                 r = amdgpu_bo_create_kernel(adev, sizeof(struct v9_mqd), PAGE_SIZE,
 658                                             AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
 659                                             &ring->mqd_gpu_addr, (void **)&ring->mqd_ptr);
 660                 if (r) {
 661                         dev_warn(adev->dev, "failed to create ring mqd ob (%d)", r);
 662                         return r;
 663                 }
 664
 665                 /*TODO: prepare MQD backup */
 666         }
 667
 668         /* create MQD for each KCQ */
 669         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 670                 ring = &adev->gfx.compute_ring[i];
 671                 if (!ring->mqd_obj) {
 672                         r = amdgpu_bo_create_kernel(adev, sizeof(struct v9_mqd), PAGE_SIZE,
 673                                                     AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
 674                                                     &ring->mqd_gpu_addr, (void **)&ring->mqd_ptr);
 675                         if (r) {
 676                                 dev_warn(adev->dev, "failed to create ring mqd ob (%d)", r);
 677                                 return r;
 678                         }
 679
 680                         /* TODO: prepare MQD backup */
 681                 }
 682         }
 683
 684         return 0;
 685 }
 686
 687 static void gfx_v9_0_compute_mqd_soft_fini(struct amdgpu_device *adev)
 688 {
 689         struct amdgpu_ring *ring = NULL;
 690         int i;
 691
 692         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 693                 ring = &adev->gfx.compute_ring[i];
 694                 amdgpu_bo_free_kernel(&ring->mqd_obj, &ring->mqd_gpu_addr, (void **)&ring->mqd_ptr);
 695         }
 696
 697         ring = &adev->gfx.kiq.ring;
 698         amdgpu_bo_free_kernel(&ring->mqd_obj, &ring->mqd_gpu_addr, (void **)&ring->mqd_ptr);
 699 }
 700
 701 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
 702 {
 703         WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_IND_INDEX),
 704                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
 705                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
 706                 (address << SQ_IND_INDEX__INDEX__SHIFT) |
 707                 (SQ_IND_INDEX__FORCE_READ_MASK));
 708         return RREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_IND_DATA));
 709 }
 710
 711 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
 712                            uint32_t wave, uint32_t thread,
 713                            uint32_t regno, uint32_t num, uint32_t *out)
 714 {
 715         WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_IND_INDEX),
 716                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
 717                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
 718                 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
 719                 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
 720                 (SQ_IND_INDEX__FORCE_READ_MASK) |
 721                 (SQ_IND_INDEX__AUTO_INCR_MASK));
 722         while (num--)
 723                 *(out++) = RREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_IND_DATA));
 724 }
 725
 726 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
 727 {
 728         /* type 1 wave data */
 729         dst[(*no_fields)++] = 1;
 730         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
 731         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
 732         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
 733         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
 734         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
 735         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
 736         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
 737         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
 738         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
 739         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
 740         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
 741         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
 742         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
 743         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
 744 }
 745
 746 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
 747                                      uint32_t wave, uint32_t start,
 748                                      uint32_t size, uint32_t *dst)
 749 {
 750         wave_read_regs(
 751                 adev, simd, wave, 0,
 752                 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
 753 }
 754
 755
 756 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
 757         .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
 758         .select_se_sh = &gfx_v9_0_select_se_sh,
 759         .read_wave_data = &gfx_v9_0_read_wave_data,
 760         .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
 761 };
 762
 763 static void gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
 764 {
 765         u32 gb_addr_config;
 766
 767         adev->gfx.funcs = &gfx_v9_0_gfx_funcs;
 768
 769         switch (adev->asic_type) {
 770         case CHIP_VEGA10:
 771                 adev->gfx.config.max_shader_engines = 4;
 772                 adev->gfx.config.max_tile_pipes = 8; //??
 773                 adev->gfx.config.max_cu_per_sh = 16;
 774                 adev->gfx.config.max_sh_per_se = 1;
 775                 adev->gfx.config.max_backends_per_se = 4;
 776                 adev->gfx.config.max_texture_channel_caches = 16;
 777                 adev->gfx.config.max_gprs = 256;
 778                 adev->gfx.config.max_gs_threads = 32;
 779                 adev->gfx.config.max_hw_contexts = 8;
 780
 781                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
 782                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
 783                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
 784                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
 785                 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN;
 786                 break;
 787         default:
 788                 BUG();
 789                 break;
 790         }
 791
 792         adev->gfx.config.gb_addr_config = gb_addr_config;
 793
 794         adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
 795                         REG_GET_FIELD(
 796                                         adev->gfx.config.gb_addr_config,
 797                                         GB_ADDR_CONFIG,
 798                                         NUM_PIPES);
 799         adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<
 800                         REG_GET_FIELD(
 801                                         adev->gfx.config.gb_addr_config,
 802                                         GB_ADDR_CONFIG,
 803                                         NUM_BANKS);
 804         adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
 805                         REG_GET_FIELD(
 806                                         adev->gfx.config.gb_addr_config,
 807                                         GB_ADDR_CONFIG,
 808                                         MAX_COMPRESSED_FRAGS);
 809         adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
 810                         REG_GET_FIELD(
 811                                         adev->gfx.config.gb_addr_config,
 812                                         GB_ADDR_CONFIG,
 813                                         NUM_RB_PER_SE);
 814         adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
 815                         REG_GET_FIELD(
 816                                         adev->gfx.config.gb_addr_config,
 817                                         GB_ADDR_CONFIG,
 818                                         NUM_SHADER_ENGINES);
 819         adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
 820                         REG_GET_FIELD(
 821                                         adev->gfx.config.gb_addr_config,
 822                                         GB_ADDR_CONFIG,
 823                                         PIPE_INTERLEAVE_SIZE));
 824 }
 825
 826 static int gfx_v9_0_ngg_create_buf(struct amdgpu_device *adev,
 827                                    struct amdgpu_ngg_buf *ngg_buf,
 828                                    int size_se,
 829                                    int default_size_se)
 830 {
 831         int r;
 832
 833         if (size_se < 0) {
 834                 dev_err(adev->dev, "Buffer size is invalid: %d\n", size_se);
 835                 return -EINVAL;
 836         }
 837         size_se = size_se ? size_se : default_size_se;
 838
 839         ngg_buf->size = size_se * GFX9_NUM_SE;
 840         r = amdgpu_bo_create_kernel(adev, ngg_buf->size,
 841                                     PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 842                                     &ngg_buf->bo,
 843                                     &ngg_buf->gpu_addr,
 844                                     NULL);
 845         if (r) {
 846                 dev_err(adev->dev, "(%d) failed to create NGG buffer\n", r);
 847                 return r;
 848         }
 849         ngg_buf->bo_size = amdgpu_bo_size(ngg_buf->bo);
 850
 851         return r;
 852 }
 853
 854 static int gfx_v9_0_ngg_fini(struct amdgpu_device *adev)
 855 {
 856         int i;
 857
 858         for (i = 0; i < NGG_BUF_MAX; i++)
 859                 amdgpu_bo_free_kernel(&adev->gfx.ngg.buf[i].bo,
 860                                       &adev->gfx.ngg.buf[i].gpu_addr,
 861                                       NULL);
 862
 863         memset(&adev->gfx.ngg.buf[0], 0,
 864                         sizeof(struct amdgpu_ngg_buf) * NGG_BUF_MAX);
 865
 866         adev->gfx.ngg.init = false;
 867
 868         return 0;
 869 }
 870
 871 static int gfx_v9_0_ngg_init(struct amdgpu_device *adev)
 872 {
 873         int r;
 874
 875         if (!amdgpu_ngg || adev->gfx.ngg.init == true)
 876                 return 0;
 877
 878         /* GDS reserve memory: 64 bytes alignment */
 879         adev->gfx.ngg.gds_reserve_size = ALIGN(5 * 4, 0x40);
 880         adev->gds.mem.total_size -= adev->gfx.ngg.gds_reserve_size;
 881         adev->gds.mem.gfx_partition_size -= adev->gfx.ngg.gds_reserve_size;
 882         adev->gfx.ngg.gds_reserve_addr = amdgpu_gds_reg_offset[0].mem_base;
 883         adev->gfx.ngg.gds_reserve_addr += adev->gds.mem.gfx_partition_size;
 884
 885         /* Primitive Buffer */
 886         r = gfx_v9_0_ngg_create_buf(adev, &adev->gfx.ngg.buf[PRIM],
 887                                     amdgpu_prim_buf_per_se,
 888                                     64 * 1024);
 889         if (r) {
 890                 dev_err(adev->dev, "Failed to create Primitive Buffer\n");
 891                 goto err;
 892         }
 893
 894         /* Position Buffer */
 895         r = gfx_v9_0_ngg_create_buf(adev, &adev->gfx.ngg.buf[POS],
 896                                     amdgpu_pos_buf_per_se,
 897                                     256 * 1024);
 898         if (r) {
 899                 dev_err(adev->dev, "Failed to create Position Buffer\n");
 900                 goto err;
 901         }
 902
 903         /* Control Sideband */
 904         r = gfx_v9_0_ngg_create_buf(adev, &adev->gfx.ngg.buf[CNTL],
 905                                     amdgpu_cntl_sb_buf_per_se,
 906                                     256);
 907         if (r) {
 908                 dev_err(adev->dev, "Failed to create Control Sideband Buffer\n");
 909                 goto err;
 910         }
 911
 912         /* Parameter Cache, not created by default */
 913         if (amdgpu_param_buf_per_se <= 0)
 914                 goto out;
 915
 916         r = gfx_v9_0_ngg_create_buf(adev, &adev->gfx.ngg.buf[PARAM],
 917                                     amdgpu_param_buf_per_se,
 918                                     512 * 1024);
 919         if (r) {
 920                 dev_err(adev->dev, "Failed to create Parameter Cache\n");
 921                 goto err;
 922         }
 923
 924 out:
 925         adev->gfx.ngg.init = true;
 926         return 0;
 927 err:
 928         gfx_v9_0_ngg_fini(adev);
 929         return r;
 930 }
 931
 932 static int gfx_v9_0_ngg_en(struct amdgpu_device *adev)
 933 {
 934         struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
 935         int r;
 936         u32 data;
 937         u32 size;
 938         u32 base;
 939
 940         if (!amdgpu_ngg)
 941                 return 0;
 942
 943         /* Program buffer size */
 944         data = 0;
 945         size = adev->gfx.ngg.buf[PRIM].size / 256;
 946         data = REG_SET_FIELD(data, WD_BUF_RESOURCE_1, INDEX_BUF_SIZE, size);
 947
 948         size = adev->gfx.ngg.buf[POS].size / 256;
 949         data = REG_SET_FIELD(data, WD_BUF_RESOURCE_1, POS_BUF_SIZE, size);
 950
 951         WREG32(SOC15_REG_OFFSET(GC, 0, mmWD_BUF_RESOURCE_1), data);
 952
 953         data = 0;
 954         size = adev->gfx.ngg.buf[CNTL].size / 256;
 955         data = REG_SET_FIELD(data, WD_BUF_RESOURCE_2, CNTL_SB_BUF_SIZE, size);
 956
 957         size = adev->gfx.ngg.buf[PARAM].size / 1024;
 958         data = REG_SET_FIELD(data, WD_BUF_RESOURCE_2, PARAM_BUF_SIZE, size);
 959
 960         WREG32(SOC15_REG_OFFSET(GC, 0, mmWD_BUF_RESOURCE_2), data);
 961
 962         /* Program buffer base address */
 963         base = lower_32_bits(adev->gfx.ngg.buf[PRIM].gpu_addr);
 964         data = REG_SET_FIELD(0, WD_INDEX_BUF_BASE, BASE, base);
 965         WREG32(SOC15_REG_OFFSET(GC, 0, mmWD_INDEX_BUF_BASE), data);
 966
 967         base = upper_32_bits(adev->gfx.ngg.buf[PRIM].gpu_addr);
 968         data = REG_SET_FIELD(0, WD_INDEX_BUF_BASE_HI, BASE_HI, base);
 969         WREG32(SOC15_REG_OFFSET(GC, 0, mmWD_INDEX_BUF_BASE_HI), data);
 970
 971         base = lower_32_bits(adev->gfx.ngg.buf[POS].gpu_addr);
 972         data = REG_SET_FIELD(0, WD_POS_BUF_BASE, BASE, base);
 973         WREG32(SOC15_REG_OFFSET(GC, 0, mmWD_POS_BUF_BASE), data);
 974
 975         base = upper_32_bits(adev->gfx.ngg.buf[POS].gpu_addr);
 976         data = REG_SET_FIELD(0, WD_POS_BUF_BASE_HI, BASE_HI, base);
 977         WREG32(SOC15_REG_OFFSET(GC, 0, mmWD_POS_BUF_BASE_HI), data);
 978
 979         base = lower_32_bits(adev->gfx.ngg.buf[CNTL].gpu_addr);
 980         data = REG_SET_FIELD(0, WD_CNTL_SB_BUF_BASE, BASE, base);
 981         WREG32(SOC15_REG_OFFSET(GC, 0, mmWD_CNTL_SB_BUF_BASE), data);
 982
 983         base = upper_32_bits(adev->gfx.ngg.buf[CNTL].gpu_addr);
 984         data = REG_SET_FIELD(0, WD_CNTL_SB_BUF_BASE_HI, BASE_HI, base);
 985         WREG32(SOC15_REG_OFFSET(GC, 0, mmWD_CNTL_SB_BUF_BASE_HI), data);
 986
 987         /* Clear GDS reserved memory */
 988         r = amdgpu_ring_alloc(ring, 17);
 989         if (r) {
 990                 DRM_ERROR("amdgpu: NGG failed to lock ring %d (%d).\n",
 991                           ring->idx, r);
 992                 return r;
 993         }
 994
 995         gfx_v9_0_write_data_to_reg(ring, 0, false,
 996                                    amdgpu_gds_reg_offset[0].mem_size,
 997                                    (adev->gds.mem.total_size +
 998                                     adev->gfx.ngg.gds_reserve_size) >>
 999                                    AMDGPU_GDS_SHIFT);
1000
1001         amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
1002         amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC |
1003                                 PACKET3_DMA_DATA_SRC_SEL(2)));
1004         amdgpu_ring_write(ring, 0);
1005         amdgpu_ring_write(ring, 0);
1006         amdgpu_ring_write(ring, adev->gfx.ngg.gds_reserve_addr);
1007         amdgpu_ring_write(ring, 0);
1008         amdgpu_ring_write(ring, adev->gfx.ngg.gds_reserve_size);
1009
1010
1011         gfx_v9_0_write_data_to_reg(ring, 0, false,
1012                                    amdgpu_gds_reg_offset[0].mem_size, 0);
1013
1014         amdgpu_ring_commit(ring);
1015
1016         return 0;
1017 }
1018
1019 static int gfx_v9_0_sw_init(void *handle)
1020 {
1021         int i, r;
1022         struct amdgpu_ring *ring;
1023         struct amdgpu_kiq *kiq;
1024         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1025
1026         /* KIQ event */
1027         r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_GRBM_CP, 178, &adev->gfx.kiq.irq);
1028         if (r)
1029                 return r;
1030
1031         /* EOP Event */
1032         r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_GRBM_CP, 181, &adev->gfx.eop_irq);
1033         if (r)
1034                 return r;
1035
1036         /* Privileged reg */
1037         r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_GRBM_CP, 184,
1038                               &adev->gfx.priv_reg_irq);
1039         if (r)
1040                 return r;
1041
1042         /* Privileged inst */
1043         r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_GRBM_CP, 185,
1044                               &adev->gfx.priv_inst_irq);
1045         if (r)
1046                 return r;
1047
1048         adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
1049
1050         gfx_v9_0_scratch_init(adev);
1051
1052         r = gfx_v9_0_init_microcode(adev);
1053         if (r) {
1054                 DRM_ERROR("Failed to load gfx firmware!\n");
1055                 return r;
1056         }
1057
1058         r = gfx_v9_0_mec_init(adev);
1059         if (r) {
1060                 DRM_ERROR("Failed to init MEC BOs!\n");
1061                 return r;
1062         }
1063
1064         /* set up the gfx ring */
1065         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
1066                 ring = &adev->gfx.gfx_ring[i];
1067                 ring->ring_obj = NULL;
1068                 sprintf(ring->name, "gfx");
1069                 ring->use_doorbell = true;
1070                 ring->doorbell_index = AMDGPU_DOORBELL64_GFX_RING0 << 1;
1071                 r = amdgpu_ring_init(adev, ring, 1024,
1072                                      &adev->gfx.eop_irq, AMDGPU_CP_IRQ_GFX_EOP);
1073                 if (r)
1074                         return r;
1075         }
1076
1077         /* set up the compute queues */
1078         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
1079                 unsigned irq_type;
1080
1081                 /* max 32 queues per MEC */
1082                 if ((i >= 32) || (i >= AMDGPU_MAX_COMPUTE_RINGS)) {
1083                         DRM_ERROR("Too many (%d) compute rings!\n", i);
1084                         break;
1085                 }
1086                 ring = &adev->gfx.compute_ring[i];
1087                 ring->ring_obj = NULL;
1088                 ring->use_doorbell = true;
1089                 ring->doorbell_index = (AMDGPU_DOORBELL64_MEC_RING0 + i) << 1;
1090                 ring->me = 1; /* first MEC */
1091                 ring->pipe = i / 8;
1092                 ring->queue = i % 8;
1093                 sprintf(ring->name, "comp %d.%d.%d", ring->me, ring->pipe, ring->queue);
1094                 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + ring->pipe;
1095                 /* type-2 packets are deprecated on MEC, use type-3 instead */
1096                 r = amdgpu_ring_init(adev, ring, 1024,
1097                                      &adev->gfx.eop_irq, irq_type);
1098                 if (r)
1099                         return r;
1100         }
1101
1102         if (amdgpu_sriov_vf(adev)) {
1103                 r = gfx_v9_0_kiq_init(adev);
1104                 if (r) {
1105                         DRM_ERROR("Failed to init KIQ BOs!\n");
1106                         return r;
1107                 }
1108
1109                 kiq = &adev->gfx.kiq;
1110                 r = gfx_v9_0_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
1111                 if (r)
1112                         return r;
1113
1114                 /* create MQD for all compute queues as wel as KIQ for SRIOV case */
1115                 r = gfx_v9_0_compute_mqd_soft_init(adev);
1116                 if (r)
1117                         return r;
1118         }
1119
1120         /* reserve GDS, GWS and OA resource for gfx */
1121         r = amdgpu_bo_create_kernel(adev, adev->gds.mem.gfx_partition_size,
1122                                     PAGE_SIZE, AMDGPU_GEM_DOMAIN_GDS,
1123                                     &adev->gds.gds_gfx_bo, NULL, NULL);
1124         if (r)
1125                 return r;
1126
1127         r = amdgpu_bo_create_kernel(adev, adev->gds.gws.gfx_partition_size,
1128                                     PAGE_SIZE, AMDGPU_GEM_DOMAIN_GWS,
1129                                     &adev->gds.gws_gfx_bo, NULL, NULL);
1130         if (r)
1131                 return r;
1132
1133         r = amdgpu_bo_create_kernel(adev, adev->gds.oa.gfx_partition_size,
1134                                     PAGE_SIZE, AMDGPU_GEM_DOMAIN_OA,
1135                                     &adev->gds.oa_gfx_bo, NULL, NULL);
1136         if (r)
1137                 return r;
1138
1139         adev->gfx.ce_ram_size = 0x8000;
1140
1141         gfx_v9_0_gpu_early_init(adev);
1142
1143         r = gfx_v9_0_ngg_init(adev);
1144         if (r)
1145                 return r;
1146
1147         return 0;
1148 }
1149
1150
1151 static int gfx_v9_0_sw_fini(void *handle)
1152 {
1153         int i;
1154         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1155
1156         amdgpu_bo_free_kernel(&adev->gds.oa_gfx_bo, NULL, NULL);
1157         amdgpu_bo_free_kernel(&adev->gds.gws_gfx_bo, NULL, NULL);
1158         amdgpu_bo_free_kernel(&adev->gds.gds_gfx_bo, NULL, NULL);
1159
1160         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
1161                 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
1162         for (i = 0; i < adev->gfx.num_compute_rings; i++)
1163                 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
1164
1165         if (amdgpu_sriov_vf(adev)) {
1166                 gfx_v9_0_compute_mqd_soft_fini(adev);
1167                 gfx_v9_0_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);
1168                 gfx_v9_0_kiq_fini(adev);
1169         }
1170
1171         gfx_v9_0_mec_fini(adev);
1172         gfx_v9_0_ngg_fini(adev);
1173
1174         return 0;
1175 }
1176
1177
1178 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev)
1179 {
1180         /* TODO */
1181 }
1182
1183 static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance)
1184 {
1185         u32 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
1186
1187         if ((se_num == 0xffffffff) && (sh_num == 0xffffffff)) {
1188                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
1189                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
1190         } else if (se_num == 0xffffffff) {
1191                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
1192                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
1193         } else if (sh_num == 0xffffffff) {
1194                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
1195                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
1196         } else {
1197                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
1198                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
1199         }
1200         WREG32( SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX), data);
1201 }
1202
1203 static u32 gfx_v9_0_create_bitmask(u32 bit_width)
1204 {
1205         return (u32)((1ULL << bit_width) - 1);
1206 }
1207
1208 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)
1209 {
1210         u32 data, mask;
1211
1212         data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCC_RB_BACKEND_DISABLE));
1213         data |= RREG32(SOC15_REG_OFFSET(GC, 0, mmGC_USER_RB_BACKEND_DISABLE));
1214
1215         data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
1216         data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
1217
1218         mask = gfx_v9_0_create_bitmask(adev->gfx.config.max_backends_per_se /
1219                                        adev->gfx.config.max_sh_per_se);
1220
1221         return (~data) & mask;
1222 }
1223
1224 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
1225 {
1226         int i, j;
1227         u32 data, tmp, num_rbs = 0;
1228         u32 active_rbs = 0;
1229         u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
1230                                         adev->gfx.config.max_sh_per_se;
1231
1232         mutex_lock(&adev->grbm_idx_mutex);
1233         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1234                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1235                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
1236                         data = gfx_v9_0_get_rb_active_bitmap(adev);
1237                         active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
1238                                                rb_bitmap_width_per_sh);
1239                 }
1240         }
1241         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1242         mutex_unlock(&adev->grbm_idx_mutex);
1243
1244         adev->gfx.config.backend_enable_mask = active_rbs;
1245         tmp = active_rbs;
1246         while (tmp >>= 1)
1247                 num_rbs++;
1248         adev->gfx.config.num_rbs = num_rbs;
1249 }
1250
1251 #define DEFAULT_SH_MEM_BASES    (0x6000)
1252 #define FIRST_COMPUTE_VMID      (8)
1253 #define LAST_COMPUTE_VMID       (16)
1254 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
1255 {
1256         int i;
1257         uint32_t sh_mem_config;
1258         uint32_t sh_mem_bases;
1259
1260         /*
1261          * Configure apertures:
1262          * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
1263          * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
1264          * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
1265          */
1266         sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
1267
1268         sh_mem_config = SH_MEM_ADDRESS_MODE_64 |
1269                         SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
1270                         SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
1271
1272         mutex_lock(&adev->srbm_mutex);
1273         for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) {
1274                 soc15_grbm_select(adev, 0, 0, 0, i);
1275                 /* CP and shaders */
1276                 WREG32(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_CONFIG), sh_mem_config);
1277                 WREG32(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_BASES), sh_mem_bases);
1278         }
1279         soc15_grbm_select(adev, 0, 0, 0, 0);
1280         mutex_unlock(&adev->srbm_mutex);
1281 }
1282
1283 static void gfx_v9_0_gpu_init(struct amdgpu_device *adev)
1284 {
1285         u32 tmp;
1286         int i;
1287
1288         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_CNTL));
1289         tmp = REG_SET_FIELD(tmp, GRBM_CNTL, READ_TIMEOUT, 0xff);
1290         WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_CNTL), tmp);
1291
1292         gfx_v9_0_tiling_mode_table_init(adev);
1293
1294         gfx_v9_0_setup_rb(adev);
1295         gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info);
1296
1297         /* XXX SH_MEM regs */
1298         /* where to put LDS, scratch, GPUVM in FSA64 space */
1299         mutex_lock(&adev->srbm_mutex);
1300         for (i = 0; i < 16; i++) {
1301                 soc15_grbm_select(adev, 0, 0, 0, i);
1302                 /* CP and shaders */
1303                 tmp = 0;
1304                 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE,
1305                                     SH_MEM_ALIGNMENT_MODE_UNALIGNED);
1306                 WREG32(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_CONFIG), tmp);
1307                 WREG32(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_BASES), 0);
1308         }
1309         soc15_grbm_select(adev, 0, 0, 0, 0);
1310
1311         mutex_unlock(&adev->srbm_mutex);
1312
1313         gfx_v9_0_init_compute_vmid(adev);
1314
1315         mutex_lock(&adev->grbm_idx_mutex);
1316         /*
1317          * making sure that the following register writes will be broadcasted
1318          * to all the shaders
1319          */
1320         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1321
1322         WREG32(SOC15_REG_OFFSET(GC, 0, mmPA_SC_FIFO_SIZE),
1323                    (adev->gfx.config.sc_prim_fifo_size_frontend <<
1324                         PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT) |
1325                    (adev->gfx.config.sc_prim_fifo_size_backend <<
1326                         PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT) |
1327                    (adev->gfx.config.sc_hiz_tile_fifo_size <<
1328                         PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT) |
1329                    (adev->gfx.config.sc_earlyz_tile_fifo_size <<
1330                         PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT));
1331         mutex_unlock(&adev->grbm_idx_mutex);
1332
1333 }
1334
1335 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
1336 {
1337         u32 i, j, k;
1338         u32 mask;
1339
1340         mutex_lock(&adev->grbm_idx_mutex);
1341         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1342                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1343                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
1344                         for (k = 0; k < adev->usec_timeout; k++) {
1345                                 if (RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY)) == 0)
1346                                         break;
1347                                 udelay(1);
1348                         }
1349                 }
1350         }
1351         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1352         mutex_unlock(&adev->grbm_idx_mutex);
1353
1354         mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
1355                 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
1356                 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
1357                 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
1358         for (k = 0; k < adev->usec_timeout; k++) {
1359                 if ((RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY)) & mask) == 0)
1360                         break;
1361                 udelay(1);
1362         }
1363 }
1364
1365 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
1366                                                bool enable)
1367 {
1368         u32 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0));
1369
1370         if (enable)
1371                 return;
1372
1373         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
1374         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
1375         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
1376         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
1377
1378         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0), tmp);
1379 }
1380
1381 void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
1382 {
1383         u32 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CNTL));
1384
1385         tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0);
1386         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CNTL), tmp);
1387
1388         gfx_v9_0_enable_gui_idle_interrupt(adev, false);
1389
1390         gfx_v9_0_wait_for_rlc_serdes(adev);
1391 }
1392
1393 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev)
1394 {
1395         u32 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_SOFT_RESET));
1396
1397         tmp = REG_SET_FIELD(tmp, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
1398         WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_SOFT_RESET), tmp);
1399         udelay(50);
1400         tmp = REG_SET_FIELD(tmp, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
1401         WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_SOFT_RESET), tmp);
1402         udelay(50);
1403 }
1404
1405 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)
1406 {
1407 #ifdef AMDGPU_RLC_DEBUG_RETRY
1408         u32 rlc_ucode_ver;
1409 #endif
1410         u32 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CNTL));
1411
1412         tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 1);
1413         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CNTL), tmp);
1414
1415         /* carrizo do enable cp interrupt after cp inited */
1416         if (!(adev->flags & AMD_IS_APU))
1417                 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
1418
1419         udelay(50);
1420
1421 #ifdef AMDGPU_RLC_DEBUG_RETRY
1422         /* RLC_GPM_GENERAL_6 : RLC Ucode version */
1423         rlc_ucode_ver = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_GENERAL_6));
1424         if(rlc_ucode_ver == 0x108) {
1425                 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",
1426                                 rlc_ucode_ver, adev->gfx.rlc_fw_version);
1427                 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,
1428                  * default is 0x9C4 to create a 100us interval */
1429                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_TIMER_INT_3), 0x9C4);
1430                 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr
1431                  * to disable the page fault retry interrupts, default is
1432                  * 0x100 (256) */
1433                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_GENERAL_12), 0x100);
1434         }
1435 #endif
1436 }
1437
1438 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev)
1439 {
1440         const struct rlc_firmware_header_v2_0 *hdr;
1441         const __le32 *fw_data;
1442         unsigned i, fw_size;
1443
1444         if (!adev->gfx.rlc_fw)
1445                 return -EINVAL;
1446
1447         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1448         amdgpu_ucode_print_rlc_hdr(&hdr->header);
1449
1450         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1451                            le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1452         fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1453
1454         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_UCODE_ADDR),
1455                         RLCG_UCODE_LOADING_START_ADDRESS);
1456         for (i = 0; i < fw_size; i++)
1457                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_UCODE_DATA), le32_to_cpup(fw_data++));
1458         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_UCODE_ADDR), adev->gfx.rlc_fw_version);
1459
1460         return 0;
1461 }
1462
1463 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
1464 {
1465         int r;
1466
1467         if (amdgpu_sriov_vf(adev))
1468                 return 0;
1469
1470         gfx_v9_0_rlc_stop(adev);
1471
1472         /* disable CG */
1473         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL), 0);
1474
1475         /* disable PG */
1476         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), 0);
1477
1478         gfx_v9_0_rlc_reset(adev);
1479
1480         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
1481                 /* legacy rlc firmware loading */
1482                 r = gfx_v9_0_rlc_load_microcode(adev);
1483                 if (r)
1484                         return r;
1485         }
1486
1487         gfx_v9_0_rlc_start(adev);
1488
1489         return 0;
1490 }
1491
1492 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
1493 {
1494         int i;
1495         u32 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_ME_CNTL));
1496
1497         if (enable) {
1498                 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 0);
1499                 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 0);
1500                 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 0);
1501         } else {
1502                 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 1);
1503                 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 1);
1504                 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 1);
1505                 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
1506                         adev->gfx.gfx_ring[i].ready = false;
1507         }
1508         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_ME_CNTL), tmp);
1509         udelay(50);
1510 }
1511
1512 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
1513 {
1514         const struct gfx_firmware_header_v1_0 *pfp_hdr;
1515         const struct gfx_firmware_header_v1_0 *ce_hdr;
1516         const struct gfx_firmware_header_v1_0 *me_hdr;
1517         const __le32 *fw_data;
1518         unsigned i, fw_size;
1519
1520         if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
1521                 return -EINVAL;
1522
1523         pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
1524                 adev->gfx.pfp_fw->data;
1525         ce_hdr = (const struct gfx_firmware_header_v1_0 *)
1526                 adev->gfx.ce_fw->data;
1527         me_hdr = (const struct gfx_firmware_header_v1_0 *)
1528                 adev->gfx.me_fw->data;
1529
1530         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
1531         amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
1532         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
1533
1534         gfx_v9_0_cp_gfx_enable(adev, false);
1535
1536         /* PFP */
1537         fw_data = (const __le32 *)
1538                 (adev->gfx.pfp_fw->data +
1539                  le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
1540         fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
1541         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PFP_UCODE_ADDR), 0);
1542         for (i = 0; i < fw_size; i++)
1543                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PFP_UCODE_DATA), le32_to_cpup(fw_data++));
1544         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PFP_UCODE_ADDR), adev->gfx.pfp_fw_version);
1545
1546         /* CE */
1547         fw_data = (const __le32 *)
1548                 (adev->gfx.ce_fw->data +
1549                  le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
1550         fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
1551         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_CE_UCODE_ADDR), 0);
1552         for (i = 0; i < fw_size; i++)
1553                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_CE_UCODE_DATA), le32_to_cpup(fw_data++));
1554         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_CE_UCODE_ADDR), adev->gfx.ce_fw_version);
1555
1556         /* ME */
1557         fw_data = (const __le32 *)
1558                 (adev->gfx.me_fw->data +
1559                  le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
1560         fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
1561         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_ME_RAM_WADDR), 0);
1562         for (i = 0; i < fw_size; i++)
1563                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_ME_RAM_DATA), le32_to_cpup(fw_data++));
1564         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_ME_RAM_WADDR), adev->gfx.me_fw_version);
1565
1566         return 0;
1567 }
1568
1569 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev)
1570 {
1571         u32 count = 0;
1572         const struct cs_section_def *sect = NULL;
1573         const struct cs_extent_def *ext = NULL;
1574
1575         /* begin clear state */
1576         count += 2;
1577         /* context control state */
1578         count += 3;
1579
1580         for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
1581                 for (ext = sect->section; ext->extent != NULL; ++ext) {
1582                         if (sect->id == SECT_CONTEXT)
1583                                 count += 2 + ext->reg_count;
1584                         else
1585                                 return 0;
1586                 }
1587         }
1588         /* pa_sc_raster_config/pa_sc_raster_config1 */
1589         count += 4;
1590         /* end clear state */
1591         count += 2;
1592         /* clear state */
1593         count += 2;
1594
1595         return count;
1596 }
1597
1598 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
1599 {
1600         struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
1601         const struct cs_section_def *sect = NULL;
1602         const struct cs_extent_def *ext = NULL;
1603         int r, i;
1604
1605         /* init the CP */
1606         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MAX_CONTEXT), adev->gfx.config.max_hw_contexts - 1);
1607         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_DEVICE_ID), 1);
1608
1609         gfx_v9_0_cp_gfx_enable(adev, true);
1610
1611         r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4);
1612         if (r) {
1613                 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
1614                 return r;
1615         }
1616
1617         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1618         amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1619
1620         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
1621         amdgpu_ring_write(ring, 0x80000000);
1622         amdgpu_ring_write(ring, 0x80000000);
1623
1624         for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
1625                 for (ext = sect->section; ext->extent != NULL; ++ext) {
1626                         if (sect->id == SECT_CONTEXT) {
1627                                 amdgpu_ring_write(ring,
1628                                        PACKET3(PACKET3_SET_CONTEXT_REG,
1629                                                ext->reg_count));
1630                                 amdgpu_ring_write(ring,
1631                                        ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
1632                                 for (i = 0; i < ext->reg_count; i++)
1633                                         amdgpu_ring_write(ring, ext->extent[i]);
1634                         }
1635                 }
1636         }
1637
1638         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1639         amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1640
1641         amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1642         amdgpu_ring_write(ring, 0);
1643
1644         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
1645         amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
1646         amdgpu_ring_write(ring, 0x8000);
1647         amdgpu_ring_write(ring, 0x8000);
1648
1649         amdgpu_ring_commit(ring);
1650
1651         return 0;
1652 }
1653
1654 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
1655 {
1656         struct amdgpu_ring *ring;
1657         u32 tmp;
1658         u32 rb_bufsz;
1659         u64 rb_addr, rptr_addr, wptr_gpu_addr;
1660
1661         /* Set the write pointer delay */
1662         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_DELAY), 0);
1663
1664         /* set the RB to use vmid 0 */
1665         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_VMID), 0);
1666
1667         /* Set ring buffer size */
1668         ring = &adev->gfx.gfx_ring[0];
1669         rb_bufsz = order_base_2(ring->ring_size / 8);
1670         tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
1671         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
1672 #ifdef __BIG_ENDIAN
1673         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
1674 #endif
1675         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_CNTL), tmp);
1676
1677         /* Initialize the ring buffer's write pointers */
1678         ring->wptr = 0;
1679         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_WPTR), lower_32_bits(ring->wptr));
1680         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_WPTR_HI), upper_32_bits(ring->wptr));
1681
1682         /* set the wb address wether it's enabled or not */
1683         rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
1684         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_RPTR_ADDR), lower_32_bits(rptr_addr));
1685         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_RPTR_ADDR_HI), upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
1686
1687         wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
1688         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO), lower_32_bits(wptr_gpu_addr));
1689         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI), upper_32_bits(wptr_gpu_addr));
1690
1691         mdelay(1);
1692         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_CNTL), tmp);
1693
1694         rb_addr = ring->gpu_addr >> 8;
1695         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_BASE), rb_addr);
1696         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_BASE_HI), upper_32_bits(rb_addr));
1697
1698         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_DOORBELL_CONTROL));
1699         if (ring->use_doorbell) {
1700                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
1701                                     DOORBELL_OFFSET, ring->doorbell_index);
1702                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
1703                                     DOORBELL_EN, 1);
1704         } else {
1705                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
1706         }
1707         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_DOORBELL_CONTROL), tmp);
1708
1709         tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
1710                         DOORBELL_RANGE_LOWER, ring->doorbell_index);
1711         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER), tmp);
1712
1713         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER),
1714                        CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
1715
1716
1717         /* start the ring */
1718         gfx_v9_0_cp_gfx_start(adev);
1719         ring->ready = true;
1720
1721         return 0;
1722 }
1723
1724 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
1725 {
1726         int i;
1727
1728         if (enable) {
1729                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEC_CNTL), 0);
1730         } else {
1731                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEC_CNTL),
1732                         (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
1733                 for (i = 0; i < adev->gfx.num_compute_rings; i++)
1734                         adev->gfx.compute_ring[i].ready = false;
1735                 adev->gfx.kiq.ring.ready = false;
1736         }
1737         udelay(50);
1738 }
1739
1740 static int gfx_v9_0_cp_compute_start(struct amdgpu_device *adev)
1741 {
1742         gfx_v9_0_cp_compute_enable(adev, true);
1743
1744         return 0;
1745 }
1746
1747 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev)
1748 {
1749         const struct gfx_firmware_header_v1_0 *mec_hdr;
1750         const __le32 *fw_data;
1751         unsigned i;
1752         u32 tmp;
1753
1754         if (!adev->gfx.mec_fw)
1755                 return -EINVAL;
1756
1757         gfx_v9_0_cp_compute_enable(adev, false);
1758
1759         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1760         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
1761
1762         fw_data = (const __le32 *)
1763                 (adev->gfx.mec_fw->data +
1764                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
1765         tmp = 0;
1766         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
1767         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
1768         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_CPC_IC_BASE_CNTL), tmp);
1769
1770         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_CPC_IC_BASE_LO),
1771                 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);
1772         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_CPC_IC_BASE_HI),
1773                 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
1774
1775         /* MEC1 */
1776         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEC_ME1_UCODE_ADDR),
1777                          mec_hdr->jt_offset);
1778         for (i = 0; i < mec_hdr->jt_size; i++)
1779                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEC_ME1_UCODE_DATA),
1780                         le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
1781
1782         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEC_ME1_UCODE_ADDR),
1783                         adev->gfx.mec_fw_version);
1784         /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */
1785
1786         return 0;
1787 }
1788
1789 static void gfx_v9_0_cp_compute_fini(struct amdgpu_device *adev)
1790 {
1791         int i, r;
1792
1793         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
1794                 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
1795
1796                 if (ring->mqd_obj) {
1797                         r = amdgpu_bo_reserve(ring->mqd_obj, false);
1798                         if (unlikely(r != 0))
1799                                 dev_warn(adev->dev, "(%d) reserve MQD bo failed\n", r);
1800
1801                         amdgpu_bo_unpin(ring->mqd_obj);
1802                         amdgpu_bo_unreserve(ring->mqd_obj);
1803
1804                         amdgpu_bo_unref(&ring->mqd_obj);
1805                         ring->mqd_obj = NULL;
1806                 }
1807         }
1808 }
1809
1810 static int gfx_v9_0_init_queue(struct amdgpu_ring *ring);
1811
1812 static int gfx_v9_0_cp_compute_resume(struct amdgpu_device *adev)
1813 {
1814         int i, r;
1815         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
1816                 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
1817                 if (gfx_v9_0_init_queue(ring))
1818                         dev_warn(adev->dev, "compute queue %d init failed!\n", i);
1819         }
1820
1821         r = gfx_v9_0_cp_compute_start(adev);
1822         if (r)
1823                 return r;
1824
1825         return 0;
1826 }
1827
1828 /* KIQ functions */
1829 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)
1830 {
1831         uint32_t tmp;
1832         struct amdgpu_device *adev = ring->adev;
1833
1834         /* tell RLC which is KIQ queue */
1835         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS));
1836         tmp &= 0xffffff00;
1837         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
1838         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS), tmp);
1839         tmp |= 0x80;
1840         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS), tmp);
1841 }
1842
1843 static void gfx_v9_0_kiq_enable(struct amdgpu_ring *ring)
1844 {
1845         amdgpu_ring_alloc(ring, 8);
1846         /* set resources */
1847         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_RESOURCES, 6));
1848         amdgpu_ring_write(ring, 0);     /* vmid_mask:0 queue_type:0 (KIQ) */
1849         amdgpu_ring_write(ring, 0x000000FF);    /* queue mask lo */
1850         amdgpu_ring_write(ring, 0);     /* queue mask hi */
1851         amdgpu_ring_write(ring, 0);     /* gws mask lo */
1852         amdgpu_ring_write(ring, 0);     /* gws mask hi */
1853         amdgpu_ring_write(ring, 0);     /* oac mask */
1854         amdgpu_ring_write(ring, 0);     /* gds heap base:0, gds heap size:0 */
1855         amdgpu_ring_commit(ring);
1856         udelay(50);
1857 }
1858
1859 static void gfx_v9_0_map_queue_enable(struct amdgpu_ring *kiq_ring,
1860                                    struct amdgpu_ring *ring)
1861 {
1862         struct amdgpu_device *adev = kiq_ring->adev;
1863         uint64_t mqd_addr, wptr_addr;
1864
1865         mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
1866         wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
1867         amdgpu_ring_alloc(kiq_ring, 8);
1868
1869         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
1870         /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
1871         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
1872                           (0 << 4) | /* Queue_Sel */
1873                           (0 << 8) | /* VMID */
1874                           (ring->queue << 13 ) |
1875                           (ring->pipe << 16) |
1876                           ((ring->me == 1 ? 0 : 1) << 18) |
1877                           (0 << 21) | /*queue_type: normal compute queue */
1878                           (1 << 24) | /* alloc format: all_on_one_pipe */
1879                           (0 << 26) | /* engine_sel: compute */
1880                           (1 << 29)); /* num_queues: must be 1 */
1881         amdgpu_ring_write(kiq_ring, (ring->doorbell_index << 2));
1882         amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
1883         amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
1884         amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
1885         amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
1886         amdgpu_ring_commit(kiq_ring);
1887         udelay(50);
1888 }
1889
1890 static int gfx_v9_0_mqd_init(struct amdgpu_device *adev,
1891                              struct v9_mqd *mqd,
1892                              uint64_t mqd_gpu_addr,
1893                              uint64_t eop_gpu_addr,
1894                              struct amdgpu_ring *ring)
1895 {
1896         uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
1897         uint32_t tmp;
1898
1899         mqd->header = 0xC0310800;
1900         mqd->compute_pipelinestat_enable = 0x00000001;
1901         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
1902         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
1903         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
1904         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
1905         mqd->compute_misc_reserved = 0x00000003;
1906
1907         eop_base_addr = eop_gpu_addr >> 8;
1908         mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
1909         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
1910
1911         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
1912         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_CONTROL));
1913         tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
1914                         (order_base_2(MEC_HPD_SIZE / 4) - 1));
1915
1916         mqd->cp_hqd_eop_control = tmp;
1917
1918         /* enable doorbell? */
1919         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL));
1920
1921         if (ring->use_doorbell) {
1922                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1923                                     DOORBELL_OFFSET, ring->doorbell_index);
1924                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1925                                     DOORBELL_EN, 1);
1926                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1927                                     DOORBELL_SOURCE, 0);
1928                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1929                                     DOORBELL_HIT, 0);
1930         }
1931         else
1932                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1933                                          DOORBELL_EN, 0);
1934
1935         mqd->cp_hqd_pq_doorbell_control = tmp;
1936
1937         /* disable the queue if it's active */
1938         ring->wptr = 0;
1939         mqd->cp_hqd_dequeue_request = 0;
1940         mqd->cp_hqd_pq_rptr = 0;
1941         mqd->cp_hqd_pq_wptr_lo = 0;
1942         mqd->cp_hqd_pq_wptr_hi = 0;
1943
1944         /* set the pointer to the MQD */
1945         mqd->cp_mqd_base_addr_lo = mqd_gpu_addr & 0xfffffffc;
1946         mqd->cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
1947
1948         /* set MQD vmid to 0 */
1949         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MQD_CONTROL));
1950         tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
1951         mqd->cp_mqd_control = tmp;
1952
1953         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
1954         hqd_gpu_addr = ring->gpu_addr >> 8;
1955         mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
1956         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
1957
1958         /* set up the HQD, this is similar to CP_RB0_CNTL */
1959         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_CONTROL));
1960         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
1961                             (order_base_2(ring->ring_size / 4) - 1));
1962         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
1963                         ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
1964 #ifdef __BIG_ENDIAN
1965         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
1966 #endif
1967         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
1968         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
1969         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
1970         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
1971         mqd->cp_hqd_pq_control = tmp;
1972
1973         /* set the wb address whether it's enabled or not */
1974         wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
1975         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
1976         mqd->cp_hqd_pq_rptr_report_addr_hi =
1977                 upper_32_bits(wb_gpu_addr) & 0xffff;
1978
1979         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
1980         wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
1981         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
1982         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
1983
1984         tmp = 0;
1985         /* enable the doorbell if requested */
1986         if (ring->use_doorbell) {
1987                 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL));
1988                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1989                                 DOORBELL_OFFSET, ring->doorbell_index);
1990
1991                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1992                                          DOORBELL_EN, 1);
1993                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1994                                          DOORBELL_SOURCE, 0);
1995                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1996                                          DOORBELL_HIT, 0);
1997         }
1998
1999         mqd->cp_hqd_pq_doorbell_control = tmp;
2000
2001         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
2002         ring->wptr = 0;
2003         mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR);
2004
2005         /* set the vmid for the queue */
2006         mqd->cp_hqd_vmid = 0;
2007
2008         tmp = RREG32(mmCP_HQD_PERSISTENT_STATE);
2009         tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
2010         mqd->cp_hqd_persistent_state = tmp;
2011
2012         /* activate the queue */
2013         mqd->cp_hqd_active = 1;
2014
2015         return 0;
2016 }
2017
2018 static int gfx_v9_0_kiq_init_register(struct amdgpu_device *adev,
2019                                       struct v9_mqd *mqd,
2020                                       struct amdgpu_ring *ring)
2021 {
2022         uint32_t tmp;
2023         int j;
2024
2025         /* disable wptr polling */
2026         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PQ_WPTR_POLL_CNTL));
2027         tmp = REG_SET_FIELD(tmp, CP_PQ_WPTR_POLL_CNTL, EN, 0);
2028         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PQ_WPTR_POLL_CNTL), tmp);
2029
2030         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_BASE_ADDR),
2031                mqd->cp_hqd_eop_base_addr_lo);
2032         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI),
2033                mqd->cp_hqd_eop_base_addr_hi);
2034
2035         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
2036         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_CONTROL),
2037                mqd->cp_hqd_eop_control);
2038
2039         /* enable doorbell? */
2040         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL),
2041                mqd->cp_hqd_pq_doorbell_control);
2042
2043         /* disable the queue if it's active */
2044         if (RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE)) & 1) {
2045                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_DEQUEUE_REQUEST), 1);
2046                 for (j = 0; j < adev->usec_timeout; j++) {
2047                         if (!(RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE)) & 1))
2048                                 break;
2049                         udelay(1);
2050                 }
2051                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_DEQUEUE_REQUEST),
2052                        mqd->cp_hqd_dequeue_request);
2053                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_RPTR),
2054                        mqd->cp_hqd_pq_rptr);
2055                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_LO),
2056                        mqd->cp_hqd_pq_wptr_lo);
2057                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI),
2058                        mqd->cp_hqd_pq_wptr_hi);
2059         }
2060
2061         /* set the pointer to the MQD */
2062         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR),
2063                mqd->cp_mqd_base_addr_lo);
2064         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR_HI),
2065                mqd->cp_mqd_base_addr_hi);
2066
2067         /* set MQD vmid to 0 */
2068         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MQD_CONTROL),
2069                mqd->cp_mqd_control);
2070
2071         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
2072         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_BASE),
2073                mqd->cp_hqd_pq_base_lo);
2074         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_BASE_HI),
2075                mqd->cp_hqd_pq_base_hi);
2076
2077         /* set up the HQD, this is similar to CP_RB0_CNTL */
2078         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_CONTROL),
2079                mqd->cp_hqd_pq_control);
2080
2081         /* set the wb address whether it's enabled or not */
2082         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR),
2083                                 mqd->cp_hqd_pq_rptr_report_addr_lo);
2084         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI),
2085                                 mqd->cp_hqd_pq_rptr_report_addr_hi);
2086
2087         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
2088         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR),
2089                mqd->cp_hqd_pq_wptr_poll_addr_lo);
2090         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI),
2091                mqd->cp_hqd_pq_wptr_poll_addr_hi);
2092
2093         /* enable the doorbell if requested */
2094         if (ring->use_doorbell) {
2095                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER),
2096                                         (AMDGPU_DOORBELL64_KIQ *2) << 2);
2097                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER),
2098                                         (AMDGPU_DOORBELL64_USERQUEUE_END * 2) << 2);
2099         }
2100
2101         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL),
2102                mqd->cp_hqd_pq_doorbell_control);
2103
2104         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
2105         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_LO),
2106                mqd->cp_hqd_pq_wptr_lo);
2107         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI),
2108                mqd->cp_hqd_pq_wptr_hi);
2109
2110         /* set the vmid for the queue */
2111         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_VMID), mqd->cp_hqd_vmid);
2112
2113         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PERSISTENT_STATE),
2114                mqd->cp_hqd_persistent_state);
2115
2116         /* activate the queue */
2117         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE),
2118                mqd->cp_hqd_active);
2119
2120         if (ring->use_doorbell) {
2121                 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PQ_STATUS));
2122                 tmp = REG_SET_FIELD(tmp, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
2123                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PQ_STATUS), tmp);
2124         }
2125
2126         return 0;
2127 }
2128
2129 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring,
2130                                    struct v9_mqd *mqd,
2131                                    u64 mqd_gpu_addr)
2132 {
2133         struct amdgpu_device *adev = ring->adev;
2134         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
2135         uint64_t eop_gpu_addr;
2136         bool is_kiq = (ring->funcs->type == AMDGPU_RING_TYPE_KIQ);
2137         int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS;
2138
2139         if (is_kiq) {
2140                 eop_gpu_addr = kiq->eop_gpu_addr;
2141                 gfx_v9_0_kiq_setting(&kiq->ring);
2142         } else {
2143                 eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr +
2144                                         ring->queue * MEC_HPD_SIZE;
2145                 mqd_idx = ring - &adev->gfx.compute_ring[0];
2146         }
2147
2148         if (!adev->gfx.in_reset) {
2149                 memset((void *)mqd, 0, sizeof(*mqd));
2150                 mutex_lock(&adev->srbm_mutex);
2151                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
2152                 gfx_v9_0_mqd_init(adev, mqd, mqd_gpu_addr, eop_gpu_addr, ring);
2153                 if (is_kiq)
2154                         gfx_v9_0_kiq_init_register(adev, mqd, ring);
2155                 soc15_grbm_select(adev, 0, 0, 0, 0);
2156                 mutex_unlock(&adev->srbm_mutex);
2157
2158         } else { /* for GPU_RESET case */
2159                 /* reset MQD to a clean status */
2160
2161                 /* reset ring buffer */
2162                 ring->wptr = 0;
2163
2164                 if (is_kiq) {
2165                     mutex_lock(&adev->srbm_mutex);
2166                     soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
2167                     gfx_v9_0_kiq_init_register(adev, mqd, ring);
2168                     soc15_grbm_select(adev, 0, 0, 0, 0);
2169                     mutex_unlock(&adev->srbm_mutex);
2170                 }
2171         }
2172
2173         if (is_kiq)
2174                 gfx_v9_0_kiq_enable(ring);
2175         else
2176                 gfx_v9_0_map_queue_enable(&kiq->ring, ring);
2177
2178         return 0;
2179 }
2180
2181 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev)
2182 {
2183         struct amdgpu_ring *ring = NULL;
2184         int r = 0, i;
2185
2186         gfx_v9_0_cp_compute_enable(adev, true);
2187
2188         ring = &adev->gfx.kiq.ring;
2189         if (!amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr)) {
2190                 r = gfx_v9_0_kiq_init_queue(ring, ring->mqd_ptr, ring->mqd_gpu_addr);
2191                 amdgpu_bo_kunmap(ring->mqd_obj);
2192                 ring->mqd_ptr = NULL;
2193                 if (r)
2194                         return r;
2195         } else {
2196                 return r;
2197         }
2198
2199         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
2200                 ring = &adev->gfx.compute_ring[i];
2201                 if (!amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr)) {
2202                         r = gfx_v9_0_kiq_init_queue(ring, ring->mqd_ptr, ring->mqd_gpu_addr);
2203                         amdgpu_bo_kunmap(ring->mqd_obj);
2204                         ring->mqd_ptr = NULL;
2205                         if (r)
2206                         return r;
2207                 } else {
2208                         return r;
2209                 }
2210         }
2211
2212         return 0;
2213 }
2214
2215 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev)
2216 {
2217         int r,i;
2218         struct amdgpu_ring *ring;
2219
2220         if (!(adev->flags & AMD_IS_APU))
2221                 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
2222
2223         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
2224                 /* legacy firmware loading */
2225                 r = gfx_v9_0_cp_gfx_load_microcode(adev);
2226                 if (r)
2227                         return r;
2228
2229                 r = gfx_v9_0_cp_compute_load_microcode(adev);
2230                 if (r)
2231                         return r;
2232         }
2233
2234         r = gfx_v9_0_cp_gfx_resume(adev);
2235         if (r)
2236                 return r;
2237
2238         if (amdgpu_sriov_vf(adev))
2239                 r = gfx_v9_0_kiq_resume(adev);
2240         else
2241                 r = gfx_v9_0_cp_compute_resume(adev);
2242         if (r)
2243                 return r;
2244
2245         ring = &adev->gfx.gfx_ring[0];
2246         r = amdgpu_ring_test_ring(ring);
2247         if (r) {
2248                 ring->ready = false;
2249                 return r;
2250         }
2251         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
2252                 ring = &adev->gfx.compute_ring[i];
2253
2254                 ring->ready = true;
2255                 r = amdgpu_ring_test_ring(ring);
2256                 if (r)
2257                         ring->ready = false;
2258         }
2259
2260         if (amdgpu_sriov_vf(adev)) {
2261                 ring = &adev->gfx.kiq.ring;
2262                 ring->ready = true;
2263                 r = amdgpu_ring_test_ring(ring);
2264                 if (r)
2265                         ring->ready = false;
2266         }
2267
2268         gfx_v9_0_enable_gui_idle_interrupt(adev, true);
2269
2270         return 0;
2271 }
2272
2273 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable)
2274 {
2275         gfx_v9_0_cp_gfx_enable(adev, enable);
2276         gfx_v9_0_cp_compute_enable(adev, enable);
2277 }
2278
2279 static int gfx_v9_0_hw_init(void *handle)
2280 {
2281         int r;
2282         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2283
2284         gfx_v9_0_init_golden_registers(adev);
2285
2286         gfx_v9_0_gpu_init(adev);
2287
2288         r = gfx_v9_0_rlc_resume(adev);
2289         if (r)
2290                 return r;
2291
2292         r = gfx_v9_0_cp_resume(adev);
2293         if (r)
2294                 return r;
2295
2296         r = gfx_v9_0_ngg_en(adev);
2297         if (r)
2298                 return r;
2299
2300         return r;
2301 }
2302
2303 static int gfx_v9_0_hw_fini(void *handle)
2304 {
2305         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2306
2307         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
2308         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
2309         if (amdgpu_sriov_vf(adev)) {
2310                 pr_debug("For SRIOV client, shouldn't do anything.\n");
2311                 return 0;
2312         }
2313         gfx_v9_0_cp_enable(adev, false);
2314         gfx_v9_0_rlc_stop(adev);
2315         gfx_v9_0_cp_compute_fini(adev);
2316
2317         return 0;
2318 }
2319
2320 static int gfx_v9_0_suspend(void *handle)
2321 {
2322         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2323
2324         return gfx_v9_0_hw_fini(adev);
2325 }
2326
2327 static int gfx_v9_0_resume(void *handle)
2328 {
2329         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2330
2331         return gfx_v9_0_hw_init(adev);
2332 }
2333
2334 static bool gfx_v9_0_is_idle(void *handle)
2335 {
2336         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2337
2338         if (REG_GET_FIELD(RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS)),
2339                                 GRBM_STATUS, GUI_ACTIVE))
2340                 return false;
2341         else
2342                 return true;
2343 }
2344
2345 static int gfx_v9_0_wait_for_idle(void *handle)
2346 {
2347         unsigned i;
2348         u32 tmp;
2349         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2350
2351         for (i = 0; i < adev->usec_timeout; i++) {
2352                 /* read MC_STATUS */
2353                 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS)) &
2354                         GRBM_STATUS__GUI_ACTIVE_MASK;
2355
2356                 if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE))
2357                         return 0;
2358                 udelay(1);
2359         }
2360         return -ETIMEDOUT;
2361 }
2362
2363 static void gfx_v9_0_print_status(void *handle)
2364 {
2365         int i;
2366         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2367
2368         dev_info(adev->dev, "GFX 9.x registers\n");
2369         dev_info(adev->dev, "  GRBM_STATUS=0x%08X\n",
2370                  RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS)));
2371         dev_info(adev->dev, "  GRBM_STATUS2=0x%08X\n",
2372                  RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS2)));
2373         dev_info(adev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
2374                  RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS_SE0)));
2375         dev_info(adev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
2376                  RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS_SE1)));
2377         dev_info(adev->dev, "  GRBM_STATUS_SE2=0x%08X\n",
2378                  RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS_SE2)));
2379         dev_info(adev->dev, "  GRBM_STATUS_SE3=0x%08X\n",
2380                  RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS_SE3)));
2381         dev_info(adev->dev, "  CP_STAT = 0x%08x\n", RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_STAT)));
2382         dev_info(adev->dev, "  CP_STALLED_STAT1 = 0x%08x\n",
2383                  RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_STALLED_STAT1)));
2384         dev_info(adev->dev, "  CP_STALLED_STAT2 = 0x%08x\n",
2385                  RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_STALLED_STAT2)));
2386         dev_info(adev->dev, "  CP_STALLED_STAT3 = 0x%08x\n",
2387                  RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_STALLED_STAT3)));
2388         dev_info(adev->dev, "  CP_CPF_BUSY_STAT = 0x%08x\n",
2389                  RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_CPF_BUSY_STAT)));
2390         dev_info(adev->dev, "  CP_CPF_STALLED_STAT1 = 0x%08x\n",
2391                  RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_CPF_STALLED_STAT1)));
2392         dev_info(adev->dev, "  CP_CPF_STATUS = 0x%08x\n", RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_CPF_STATUS)));
2393         dev_info(adev->dev, "  CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_CPC_BUSY_STAT)));
2394         dev_info(adev->dev, "  CP_CPC_STALLED_STAT1 = 0x%08x\n",
2395                  RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_CPC_STALLED_STAT1)));
2396         dev_info(adev->dev, "  CP_CPC_STATUS = 0x%08x\n", RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_CPC_STATUS)));
2397
2398         for (i = 0; i < 32; i++) {
2399                 dev_info(adev->dev, "  GB_TILE_MODE%d=0x%08X\n",
2400                          i, RREG32(SOC15_REG_OFFSET(GC, 0, mmGB_TILE_MODE0 ) + i*4));
2401         }
2402         for (i = 0; i < 16; i++) {
2403                 dev_info(adev->dev, "  GB_MACROTILE_MODE%d=0x%08X\n",
2404                          i, RREG32(SOC15_REG_OFFSET(GC, 0, mmGB_MACROTILE_MODE0) + i*4));
2405         }
2406         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2407                 dev_info(adev->dev, "  se: %d\n", i);
2408                 gfx_v9_0_select_se_sh(adev, i, 0xffffffff, 0xffffffff);
2409                 dev_info(adev->dev, "  PA_SC_RASTER_CONFIG=0x%08X\n",
2410                          RREG32(SOC15_REG_OFFSET(GC, 0, mmPA_SC_RASTER_CONFIG)));
2411                 dev_info(adev->dev, "  PA_SC_RASTER_CONFIG_1=0x%08X\n",
2412                          RREG32(SOC15_REG_OFFSET(GC, 0, mmPA_SC_RASTER_CONFIG_1)));
2413         }
2414         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2415
2416         dev_info(adev->dev, "  GB_ADDR_CONFIG=0x%08X\n",
2417                  RREG32(SOC15_REG_OFFSET(GC, 0, mmGB_ADDR_CONFIG)));
2418
2419         dev_info(adev->dev, "  CP_MEQ_THRESHOLDS=0x%08X\n",
2420                  RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEQ_THRESHOLDS)));
2421         dev_info(adev->dev, "  SX_DEBUG_1=0x%08X\n",
2422                  RREG32(SOC15_REG_OFFSET(GC, 0, mmSX_DEBUG_1)));
2423         dev_info(adev->dev, "  TA_CNTL_AUX=0x%08X\n",
2424                  RREG32(SOC15_REG_OFFSET(GC, 0, mmTA_CNTL_AUX)));
2425         dev_info(adev->dev, "  SPI_CONFIG_CNTL=0x%08X\n",
2426                  RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_CONFIG_CNTL)));
2427         dev_info(adev->dev, "  SQ_CONFIG=0x%08X\n",
2428                  RREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_CONFIG)));
2429         dev_info(adev->dev, "  DB_DEBUG=0x%08X\n",
2430                  RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_DEBUG)));
2431         dev_info(adev->dev, "  DB_DEBUG2=0x%08X\n",
2432                  RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_DEBUG2)));
2433         dev_info(adev->dev, "  DB_DEBUG3=0x%08X\n",
2434                  RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_DEBUG3)));
2435         dev_info(adev->dev, "  CB_HW_CONTROL=0x%08X\n",
2436                  RREG32(SOC15_REG_OFFSET(GC, 0, mmCB_HW_CONTROL)));
2437         dev_info(adev->dev, "  SPI_CONFIG_CNTL_1=0x%08X\n",
2438                  RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_CONFIG_CNTL_1)));
2439         dev_info(adev->dev, "  PA_SC_FIFO_SIZE=0x%08X\n",
2440                  RREG32(SOC15_REG_OFFSET(GC, 0, mmPA_SC_FIFO_SIZE)));
2441         dev_info(adev->dev, "  VGT_NUM_INSTANCES=0x%08X\n",
2442                  RREG32(SOC15_REG_OFFSET(GC, 0, mmVGT_NUM_INSTANCES)));
2443         dev_info(adev->dev, "  CP_PERFMON_CNTL=0x%08X\n",
2444                  RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PERFMON_CNTL)));
2445         dev_info(adev->dev, "  PA_SC_FORCE_EOV_MAX_CNTS=0x%08X\n",
2446                  RREG32(SOC15_REG_OFFSET(GC, 0, mmPA_SC_FORCE_EOV_MAX_CNTS)));
2447         dev_info(adev->dev, "  VGT_CACHE_INVALIDATION=0x%08X\n",
2448                  RREG32(SOC15_REG_OFFSET(GC, 0, mmVGT_CACHE_INVALIDATION)));
2449         dev_info(adev->dev, "  VGT_GS_VERTEX_REUSE=0x%08X\n",
2450                  RREG32(SOC15_REG_OFFSET(GC, 0, mmVGT_GS_VERTEX_REUSE)));
2451         dev_info(adev->dev, "  PA_SC_LINE_STIPPLE_STATE=0x%08X\n",
2452                  RREG32(SOC15_REG_OFFSET(GC, 0, mmPA_SC_LINE_STIPPLE_STATE)));
2453         dev_info(adev->dev, "  PA_CL_ENHANCE=0x%08X\n",
2454                  RREG32(SOC15_REG_OFFSET(GC, 0, mmPA_CL_ENHANCE)));
2455         dev_info(adev->dev, "  PA_SC_ENHANCE=0x%08X\n",
2456                  RREG32(SOC15_REG_OFFSET(GC, 0, mmPA_SC_ENHANCE)));
2457
2458         dev_info(adev->dev, "  CP_ME_CNTL=0x%08X\n",
2459                  RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_ME_CNTL)));
2460         dev_info(adev->dev, "  CP_MAX_CONTEXT=0x%08X\n",
2461                  RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MAX_CONTEXT)));
2462         dev_info(adev->dev, "  CP_DEVICE_ID=0x%08X\n",
2463                  RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_DEVICE_ID)));
2464
2465         dev_info(adev->dev, "  CP_SEM_WAIT_TIMER=0x%08X\n",
2466                  RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_SEM_WAIT_TIMER)));
2467
2468         dev_info(adev->dev, "  CP_RB_WPTR_DELAY=0x%08X\n",
2469                  RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_DELAY)));
2470         dev_info(adev->dev, "  CP_RB_VMID=0x%08X\n",
2471                  RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_VMID)));
2472         dev_info(adev->dev, "  CP_RB0_CNTL=0x%08X\n",
2473                  RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_CNTL)));
2474         dev_info(adev->dev, "  CP_RB0_WPTR=0x%08X\n",
2475                  RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_WPTR)));
2476         dev_info(adev->dev, "  CP_RB0_RPTR_ADDR=0x%08X\n",
2477                  RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_RPTR_ADDR)));
2478         dev_info(adev->dev, "  CP_RB0_RPTR_ADDR_HI=0x%08X\n",
2479                  RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_RPTR_ADDR_HI)));
2480         dev_info(adev->dev, "  CP_RB0_CNTL=0x%08X\n",
2481                  RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_CNTL)));
2482         dev_info(adev->dev, "  CP_RB0_BASE=0x%08X\n",
2483                  RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_BASE)));
2484         dev_info(adev->dev, "  CP_RB0_BASE_HI=0x%08X\n",
2485                  RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_BASE_HI)));
2486         dev_info(adev->dev, "  CP_MEC_CNTL=0x%08X\n",
2487                  RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEC_CNTL)));
2488
2489         dev_info(adev->dev, "  SCRATCH_ADDR=0x%08X\n",
2490                  RREG32(SOC15_REG_OFFSET(GC, 0, mmSCRATCH_ADDR)));
2491         dev_info(adev->dev, "  SCRATCH_UMSK=0x%08X\n",
2492                  RREG32(SOC15_REG_OFFSET(GC, 0, mmSCRATCH_UMSK)));
2493
2494         dev_info(adev->dev, "  CP_INT_CNTL_RING0=0x%08X\n",
2495                  RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0)));
2496         dev_info(adev->dev, "  RLC_LB_CNTL=0x%08X\n",
2497                  RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_LB_CNTL)));
2498         dev_info(adev->dev, "  RLC_CNTL=0x%08X\n",
2499                  RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CNTL)));
2500         dev_info(adev->dev, "  RLC_CGCG_CGLS_CTRL=0x%08X\n",
2501                  RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL)));
2502         dev_info(adev->dev, "  RLC_LB_CNTR_INIT=0x%08X\n",
2503                  RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_LB_CNTR_INIT)));
2504         dev_info(adev->dev, "  RLC_LB_CNTR_MAX=0x%08X\n",
2505                  RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_LB_CNTR_MAX)));
2506         dev_info(adev->dev, "  RLC_LB_INIT_CU_MASK=0x%08X\n",
2507                  RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_LB_INIT_CU_MASK)));
2508         dev_info(adev->dev, "  RLC_LB_PARAMS=0x%08X\n",
2509                  RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_LB_PARAMS)));
2510         dev_info(adev->dev, "  RLC_LB_CNTL=0x%08X\n",
2511                  RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_LB_CNTL)));
2512         dev_info(adev->dev, "  RLC_UCODE_CNTL=0x%08X\n",
2513                  RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_UCODE_CNTL)));
2514
2515         dev_info(adev->dev, "  RLC_GPM_GENERAL_6=0x%08X\n",
2516                  RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_GENERAL_6)));
2517         dev_info(adev->dev, "  RLC_GPM_GENERAL_12=0x%08X\n",
2518                  RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_GENERAL_12)));
2519         dev_info(adev->dev, "  RLC_GPM_TIMER_INT_3=0x%08X\n",
2520                  RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_TIMER_INT_3)));
2521         mutex_lock(&adev->srbm_mutex);
2522         for (i = 0; i < 16; i++) {
2523                 soc15_grbm_select(adev, 0, 0, 0, i);
2524                 dev_info(adev->dev, "  VM %d:\n", i);
2525                 dev_info(adev->dev, "  SH_MEM_CONFIG=0x%08X\n",
2526                          RREG32(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_CONFIG)));
2527                 dev_info(adev->dev, "  SH_MEM_BASES=0x%08X\n",
2528                          RREG32(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_BASES)));
2529         }
2530         soc15_grbm_select(adev, 0, 0, 0, 0);
2531         mutex_unlock(&adev->srbm_mutex);
2532 }
2533
2534 static int gfx_v9_0_soft_reset(void *handle)
2535 {
2536         u32 grbm_soft_reset = 0;
2537         u32 tmp;
2538         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2539
2540         /* GRBM_STATUS */
2541         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS));
2542         if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
2543                    GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
2544                    GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
2545                    GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
2546                    GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
2547                    GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {
2548                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
2549                                                 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
2550                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
2551                                                 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
2552         }
2553
2554         if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
2555                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
2556                                                 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
2557         }
2558
2559         /* GRBM_STATUS2 */
2560         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS2));
2561         if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
2562                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
2563                                                 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
2564
2565
2566         if (grbm_soft_reset ) {
2567                 gfx_v9_0_print_status((void *)adev);
2568                 /* stop the rlc */
2569                 gfx_v9_0_rlc_stop(adev);
2570
2571                 /* Disable GFX parsing/prefetching */
2572                 gfx_v9_0_cp_gfx_enable(adev, false);
2573
2574                 /* Disable MEC parsing/prefetching */
2575                 gfx_v9_0_cp_compute_enable(adev, false);
2576
2577                 if (grbm_soft_reset) {
2578                         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_SOFT_RESET));
2579                         tmp |= grbm_soft_reset;
2580                         dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
2581                         WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_SOFT_RESET), tmp);
2582                         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_SOFT_RESET));
2583
2584                         udelay(50);
2585
2586                         tmp &= ~grbm_soft_reset;
2587                         WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_SOFT_RESET), tmp);
2588                         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_SOFT_RESET));
2589                 }
2590
2591                 /* Wait a little for things to settle down */
2592                 udelay(50);
2593                 gfx_v9_0_print_status((void *)adev);
2594         }
2595         return 0;
2596 }
2597
2598 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
2599 {
2600         uint64_t clock;
2601
2602         mutex_lock(&adev->gfx.gpu_clock_mutex);
2603         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT), 1);
2604         clock = (uint64_t)RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB)) |
2605                 ((uint64_t)RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB)) << 32ULL);
2606         mutex_unlock(&adev->gfx.gpu_clock_mutex);
2607         return clock;
2608 }
2609
2610 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
2611                                           uint32_t vmid,
2612                                           uint32_t gds_base, uint32_t gds_size,
2613                                           uint32_t gws_base, uint32_t gws_size,
2614                                           uint32_t oa_base, uint32_t oa_size)
2615 {
2616         gds_base = gds_base >> AMDGPU_GDS_SHIFT;
2617         gds_size = gds_size >> AMDGPU_GDS_SHIFT;
2618
2619         gws_base = gws_base >> AMDGPU_GWS_SHIFT;
2620         gws_size = gws_size >> AMDGPU_GWS_SHIFT;
2621
2622         oa_base = oa_base >> AMDGPU_OA_SHIFT;
2623         oa_size = oa_size >> AMDGPU_OA_SHIFT;
2624
2625         /* GDS Base */
2626         gfx_v9_0_write_data_to_reg(ring, 0, false,
2627                                    amdgpu_gds_reg_offset[vmid].mem_base,
2628                                    gds_base);
2629
2630         /* GDS Size */
2631         gfx_v9_0_write_data_to_reg(ring, 0, false,
2632                                    amdgpu_gds_reg_offset[vmid].mem_size,
2633                                    gds_size);
2634
2635         /* GWS */
2636         gfx_v9_0_write_data_to_reg(ring, 0, false,
2637                                    amdgpu_gds_reg_offset[vmid].gws,
2638                                    gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
2639
2640         /* OA */
2641         gfx_v9_0_write_data_to_reg(ring, 0, false,
2642                                    amdgpu_gds_reg_offset[vmid].oa,
2643                                    (1 << (oa_size + oa_base)) - (1 << oa_base));
2644 }
2645
2646 static int gfx_v9_0_early_init(void *handle)
2647 {
2648         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2649
2650         adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS;
2651         adev->gfx.num_compute_rings = GFX9_NUM_COMPUTE_RINGS;
2652         gfx_v9_0_set_ring_funcs(adev);
2653         gfx_v9_0_set_irq_funcs(adev);
2654         gfx_v9_0_set_gds_init(adev);
2655         gfx_v9_0_set_rlc_funcs(adev);
2656
2657         return 0;
2658 }
2659
2660 static int gfx_v9_0_late_init(void *handle)
2661 {
2662         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2663         int r;
2664
2665         r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
2666         if (r)
2667                 return r;
2668
2669         r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
2670         if (r)
2671                 return r;
2672
2673         return 0;
2674 }
2675
2676 static void gfx_v9_0_enter_rlc_safe_mode(struct amdgpu_device *adev)
2677 {
2678         uint32_t rlc_setting, data;
2679         unsigned i;
2680
2681         if (adev->gfx.rlc.in_safe_mode)
2682                 return;
2683
2684         /* if RLC is not enabled, do nothing */
2685         rlc_setting = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CNTL));
2686         if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
2687                 return;
2688
2689         if (adev->cg_flags &
2690             (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_MGCG |
2691              AMD_CG_SUPPORT_GFX_3D_CGCG)) {
2692                 data = RLC_SAFE_MODE__CMD_MASK;
2693                 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
2694                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SAFE_MODE), data);
2695
2696                 /* wait for RLC_SAFE_MODE */
2697                 for (i = 0; i < adev->usec_timeout; i++) {
2698                         if (!REG_GET_FIELD(SOC15_REG_OFFSET(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
2699                                 break;
2700                         udelay(1);
2701                 }
2702                 adev->gfx.rlc.in_safe_mode = true;
2703         }
2704 }
2705
2706 static void gfx_v9_0_exit_rlc_safe_mode(struct amdgpu_device *adev)
2707 {
2708         uint32_t rlc_setting, data;
2709
2710         if (!adev->gfx.rlc.in_safe_mode)
2711                 return;
2712
2713         /* if RLC is not enabled, do nothing */
2714         rlc_setting = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CNTL));
2715         if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
2716                 return;
2717
2718         if (adev->cg_flags &
2719             (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_MGCG)) {
2720                 /*
2721                  * Try to exit safe mode only if it is already in safe
2722                  * mode.
2723                  */
2724                 data = RLC_SAFE_MODE__CMD_MASK;
2725                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SAFE_MODE), data);
2726                 adev->gfx.rlc.in_safe_mode = false;
2727         }
2728 }
2729
2730 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
2731                                                       bool enable)
2732 {
2733         uint32_t data, def;
2734
2735         /* It is disabled by HW by default */
2736         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
2737                 /* 1 - RLC_CGTT_MGCG_OVERRIDE */
2738                 def = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE));
2739                 data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK |
2740                           RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
2741                           RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
2742                           RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
2743
2744                 /* only for Vega10 & Raven1 */
2745                 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;
2746
2747                 if (def != data)
2748                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE), data);
2749
2750                 /* MGLS is a global flag to control all MGLS in GFX */
2751                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
2752                         /* 2 - RLC memory Light sleep */
2753                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
2754                                 def = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL));
2755                                 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
2756                                 if (def != data)
2757                                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL), data);
2758                         }
2759                         /* 3 - CP memory Light sleep */
2760                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
2761                                 def = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL));
2762                                 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
2763                                 if (def != data)
2764                                         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL), data);
2765                         }
2766                 }
2767         } else {
2768                 /* 1 - MGCG_OVERRIDE */
2769                 def = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE));
2770                 data |= (RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK |
2771                          RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
2772                          RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
2773                          RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
2774                          RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
2775                 if (def != data)
2776                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE), data);
2777
2778                 /* 2 - disable MGLS in RLC */
2779                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL));
2780                 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
2781                         data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
2782                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL), data);
2783                 }
2784
2785                 /* 3 - disable MGLS in CP */
2786                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL));
2787                 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
2788                         data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
2789                         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL), data);
2790                 }
2791         }
2792 }
2793
2794 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,
2795                                            bool enable)
2796 {
2797         uint32_t data, def;
2798
2799         adev->gfx.rlc.funcs->enter_safe_mode(adev);
2800
2801         /* Enable 3D CGCG/CGLS */
2802         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)) {
2803                 /* write cmd to clear cgcg/cgls ov */
2804                 def = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE));
2805                 /* unset CGCG override */
2806                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
2807                 /* update CGCG and CGLS override bits */
2808                 if (def != data)
2809                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE), data);
2810                 /* enable 3Dcgcg FSM(0x0020003f) */
2811                 def = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D));
2812                 data = (0x2000 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
2813                         RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
2814                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
2815                         data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
2816                                 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
2817                 if (def != data)
2818                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D), data);
2819
2820                 /* set IDLE_POLL_COUNT(0x00900100) */
2821                 def = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL));
2822                 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
2823                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2824                 if (def != data)
2825                         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data);
2826         } else {
2827                 /* Disable CGCG/CGLS */
2828                 def = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D));
2829                 /* disable cgcg, cgls should be disabled */
2830                 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |
2831                           RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);
2832                 /* disable cgcg and cgls in FSM */
2833                 if (def != data)
2834                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D), data);
2835         }
2836
2837         adev->gfx.rlc.funcs->exit_safe_mode(adev);
2838 }
2839
2840 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
2841                                                       bool enable)
2842 {
2843         uint32_t def, data;
2844
2845         adev->gfx.rlc.funcs->enter_safe_mode(adev);
2846
2847         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
2848                 def = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE));
2849                 /* unset CGCG override */
2850                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
2851                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
2852                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
2853                 else
2854                         data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
2855                 /* update CGCG and CGLS override bits */
2856                 if (def != data)
2857                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE), data);
2858
2859                 /* enable cgcg FSM(0x0020003F) */
2860                 def = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL));
2861                 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
2862                         RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
2863                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
2864                         data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
2865                                 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
2866                 if (def != data)
2867                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL), data);
2868
2869                 /* set IDLE_POLL_COUNT(0x00900100) */
2870                 def = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL));
2871                 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
2872                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2873                 if (def != data)
2874                         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data);
2875         } else {
2876                 def = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL));
2877                 /* reset CGCG/CGLS bits */
2878                 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
2879                 /* disable cgcg and cgls in FSM */
2880                 if (def != data)
2881                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL), data);
2882         }
2883
2884         adev->gfx.rlc.funcs->exit_safe_mode(adev);
2885 }
2886
2887 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev,
2888                                             bool enable)
2889 {
2890         if (enable) {
2891                 /* CGCG/CGLS should be enabled after MGCG/MGLS
2892                  * ===  MGCG + MGLS ===
2893                  */
2894                 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
2895                 /* ===  CGCG /CGLS for GFX 3D Only === */
2896                 gfx_v9_0_update_3d_clock_gating(adev, enable);
2897                 /* ===  CGCG + CGLS === */
2898                 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
2899         } else {
2900                 /* CGCG/CGLS should be disabled before MGCG/MGLS
2901                  * ===  CGCG + CGLS ===
2902                  */
2903                 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
2904                 /* ===  CGCG /CGLS for GFX 3D Only === */
2905                 gfx_v9_0_update_3d_clock_gating(adev, enable);
2906                 /* ===  MGCG + MGLS === */
2907                 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
2908         }
2909         return 0;
2910 }
2911
2912 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = {
2913         .enter_safe_mode = gfx_v9_0_enter_rlc_safe_mode,
2914         .exit_safe_mode = gfx_v9_0_exit_rlc_safe_mode
2915 };
2916
2917 static int gfx_v9_0_set_powergating_state(void *handle,
2918                                           enum amd_powergating_state state)
2919 {
2920         return 0;
2921 }
2922
2923 static int gfx_v9_0_set_clockgating_state(void *handle,
2924                                           enum amd_clockgating_state state)
2925 {
2926         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2927
2928         switch (adev->asic_type) {
2929         case CHIP_VEGA10:
2930                 gfx_v9_0_update_gfx_clock_gating(adev,
2931                                                  state == AMD_CG_STATE_GATE ? true : false);
2932                 break;
2933         default:
2934                 break;
2935         }
2936         return 0;
2937 }
2938
2939 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
2940 {
2941         return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 is 32bit rptr*/
2942 }
2943
2944 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
2945 {
2946         struct amdgpu_device *adev = ring->adev;
2947         u64 wptr;
2948
2949         /* XXX check if swapping is necessary on BE */
2950         if (ring->use_doorbell) {
2951                 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]);
2952         } else {
2953                 wptr = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_WPTR));
2954                 wptr += (u64)RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_WPTR_HI)) << 32;
2955         }
2956
2957         return wptr;
2958 }
2959
2960 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
2961 {
2962         struct amdgpu_device *adev = ring->adev;
2963
2964         if (ring->use_doorbell) {
2965                 /* XXX check if swapping is necessary on BE */
2966                 atomic64_set((atomic64_t*)&adev->wb.wb[ring->wptr_offs], ring->wptr);
2967                 WDOORBELL64(ring->doorbell_index, ring->wptr);
2968         } else {
2969                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_WPTR), lower_32_bits(ring->wptr));
2970                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_WPTR_HI), upper_32_bits(ring->wptr));
2971         }
2972 }
2973
2974 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
2975 {
2976         u32 ref_and_mask, reg_mem_engine;
2977         struct nbio_hdp_flush_reg *nbio_hf_reg;
2978
2979         if (ring->adev->asic_type == CHIP_VEGA10)
2980                 nbio_hf_reg = &nbio_v6_1_hdp_flush_reg;
2981
2982         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
2983                 switch (ring->me) {
2984                 case 1:
2985                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
2986                         break;
2987                 case 2:
2988                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
2989                         break;
2990                 default:
2991                         return;
2992                 }
2993                 reg_mem_engine = 0;
2994         } else {
2995                 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
2996                 reg_mem_engine = 1; /* pfp */
2997         }
2998
2999         gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
3000                               nbio_hf_reg->hdp_flush_req_offset,
3001                               nbio_hf_reg->hdp_flush_done_offset,
3002                               ref_and_mask, ref_and_mask, 0x20);
3003 }
3004
3005 static void gfx_v9_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
3006 {
3007         gfx_v9_0_write_data_to_reg(ring, 0, true,
3008                                    SOC15_REG_OFFSET(HDP, 0, mmHDP_DEBUG0), 1);
3009 }
3010
3011 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
3012                                       struct amdgpu_ib *ib,
3013                                       unsigned vm_id, bool ctx_switch)
3014 {
3015         u32 header, control = 0;
3016
3017         if (ib->flags & AMDGPU_IB_FLAG_CE)
3018                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3019         else
3020                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3021
3022         control |= ib->length_dw | (vm_id << 24);
3023
3024         amdgpu_ring_write(ring, header);
3025         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
3026         amdgpu_ring_write(ring,
3027 #ifdef __BIG_ENDIAN
3028                           (2 << 0) |
3029 #endif
3030                           lower_32_bits(ib->gpu_addr));
3031         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
3032         amdgpu_ring_write(ring, control);
3033 }
3034
3035 #define INDIRECT_BUFFER_VALID                   (1 << 23)
3036
3037 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
3038                                           struct amdgpu_ib *ib,
3039                                           unsigned vm_id, bool ctx_switch)
3040 {
3041         u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vm_id << 24);
3042
3043         amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
3044         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
3045         amdgpu_ring_write(ring,
3046 #ifdef __BIG_ENDIAN
3047                                 (2 << 0) |
3048 #endif
3049                                 lower_32_bits(ib->gpu_addr));
3050         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
3051         amdgpu_ring_write(ring, control);
3052 }
3053
3054 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
3055                                      u64 seq, unsigned flags)
3056 {
3057         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
3058         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
3059
3060         /* RELEASE_MEM - flush caches, send int */
3061         amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
3062         amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
3063                                  EOP_TC_ACTION_EN |
3064                                  EOP_TC_WB_ACTION_EN |
3065                                  EOP_TC_MD_ACTION_EN |
3066                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3067                                  EVENT_INDEX(5)));
3068         amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
3069
3070         /*
3071          * the address should be Qword aligned if 64bit write, Dword
3072          * aligned if only send 32bit data low (discard data high)
3073          */
3074         if (write64bit)
3075                 BUG_ON(addr & 0x7);
3076         else
3077                 BUG_ON(addr & 0x3);
3078         amdgpu_ring_write(ring, lower_32_bits(addr));
3079         amdgpu_ring_write(ring, upper_32_bits(addr));
3080         amdgpu_ring_write(ring, lower_32_bits(seq));
3081         amdgpu_ring_write(ring, upper_32_bits(seq));
3082         amdgpu_ring_write(ring, 0);
3083 }
3084
3085 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
3086 {
3087         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
3088         uint32_t seq = ring->fence_drv.sync_seq;
3089         uint64_t addr = ring->fence_drv.gpu_addr;
3090
3091         gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0,
3092                               lower_32_bits(addr), upper_32_bits(addr),
3093                               seq, 0xffffffff, 4);
3094 }
3095
3096 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
3097                                         unsigned vm_id, uint64_t pd_addr)
3098 {
3099         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
3100         unsigned eng = ring->idx;
3101         unsigned i;
3102
3103         pd_addr = pd_addr | 0x1; /* valid bit */
3104         /* now only use physical base address of PDE and valid */
3105         BUG_ON(pd_addr & 0xFFFF00000000003EULL);
3106
3107         for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
3108                 struct amdgpu_vmhub *hub = &ring->adev->vmhub[i];
3109                 uint32_t req = hub->get_invalidate_req(vm_id);
3110
3111                 gfx_v9_0_write_data_to_reg(ring, usepfp, true,
3112                                            hub->ctx0_ptb_addr_lo32
3113                                            + (2 * vm_id),
3114                                            lower_32_bits(pd_addr));
3115
3116                 gfx_v9_0_write_data_to_reg(ring, usepfp, true,
3117                                            hub->ctx0_ptb_addr_hi32
3118                                            + (2 * vm_id),
3119                                            upper_32_bits(pd_addr));
3120
3121                 gfx_v9_0_write_data_to_reg(ring, usepfp, true,
3122                                            hub->vm_inv_eng0_req + eng, req);
3123
3124                 /* wait for the invalidate to complete */
3125                 gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, hub->vm_inv_eng0_ack +
3126                                       eng, 0, 1 << vm_id, 1 << vm_id, 0x20);
3127         }
3128
3129         /* compute doesn't have PFP */
3130         if (usepfp) {
3131                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
3132                 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3133                 amdgpu_ring_write(ring, 0x0);
3134                 /* Emits 128 dw nop to prevent CE access VM before vm_flush finish */
3135                 amdgpu_ring_insert_nop(ring, 128);
3136         }
3137 }
3138
3139 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
3140 {
3141         return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */
3142 }
3143
3144 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
3145 {
3146         u64 wptr;
3147
3148         /* XXX check if swapping is necessary on BE */
3149         if (ring->use_doorbell)
3150                 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]);
3151         else
3152                 BUG();
3153         return wptr;
3154 }
3155
3156 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
3157 {
3158         struct amdgpu_device *adev = ring->adev;
3159
3160         /* XXX check if swapping is necessary on BE */
3161         if (ring->use_doorbell) {
3162                 atomic64_set((atomic64_t*)&adev->wb.wb[ring->wptr_offs], ring->wptr);
3163                 WDOORBELL64(ring->doorbell_index, ring->wptr);
3164         } else{
3165                 BUG(); /* only DOORBELL method supported on gfx9 now */
3166         }
3167 }
3168
3169 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
3170                                          u64 seq, unsigned int flags)
3171 {
3172         /* we only allocate 32bit for each seq wb address */
3173         BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
3174
3175         /* write fence seq to the "addr" */
3176         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3177         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3178                                  WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
3179         amdgpu_ring_write(ring, lower_32_bits(addr));
3180         amdgpu_ring_write(ring, upper_32_bits(addr));
3181         amdgpu_ring_write(ring, lower_32_bits(seq));
3182
3183         if (flags & AMDGPU_FENCE_FLAG_INT) {
3184                 /* set register to trigger INT */
3185                 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3186                 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3187                                          WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
3188                 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));
3189                 amdgpu_ring_write(ring, 0);
3190                 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
3191         }
3192 }
3193
3194 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
3195 {
3196         amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3197         amdgpu_ring_write(ring, 0);
3198 }
3199
3200 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
3201 {
3202         static struct v9_ce_ib_state ce_payload = {0};
3203         uint64_t csa_addr;
3204         int cnt;
3205
3206         cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
3207         csa_addr = AMDGPU_VA_RESERVED_SIZE - 2 * 4096;
3208
3209         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
3210         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
3211                                  WRITE_DATA_DST_SEL(8) |
3212                                  WR_CONFIRM) |
3213                                  WRITE_DATA_CACHE_POLICY(0));
3214         amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload)));
3215         amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload)));
3216         amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2);
3217 }
3218
3219 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring)
3220 {
3221         static struct v9_de_ib_state de_payload = {0};
3222         uint64_t csa_addr, gds_addr;
3223         int cnt;
3224
3225         csa_addr = AMDGPU_VA_RESERVED_SIZE - 2 * 4096;
3226         gds_addr = csa_addr + 4096;
3227         de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
3228         de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
3229
3230         cnt = (sizeof(de_payload) >> 2) + 4 - 2;
3231         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
3232         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
3233                                  WRITE_DATA_DST_SEL(8) |
3234                                  WR_CONFIRM) |
3235                                  WRITE_DATA_CACHE_POLICY(0));
3236         amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload)));
3237         amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload)));
3238         amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2);
3239 }
3240
3241 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
3242 {
3243         uint32_t dw2 = 0;
3244
3245         if (amdgpu_sriov_vf(ring->adev))
3246                 gfx_v9_0_ring_emit_ce_meta(ring);
3247
3248         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
3249         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
3250                 /* set load_global_config & load_global_uconfig */
3251                 dw2 |= 0x8001;
3252                 /* set load_cs_sh_regs */
3253                 dw2 |= 0x01000000;
3254                 /* set load_per_context_state & load_gfx_sh_regs for GFX */
3255                 dw2 |= 0x10002;
3256
3257                 /* set load_ce_ram if preamble presented */
3258                 if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
3259                         dw2 |= 0x10000000;
3260         } else {
3261                 /* still load_ce_ram if this is the first time preamble presented
3262                  * although there is no context switch happens.
3263                  */
3264                 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
3265                         dw2 |= 0x10000000;
3266         }
3267
3268         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3269         amdgpu_ring_write(ring, dw2);
3270         amdgpu_ring_write(ring, 0);
3271
3272         if (amdgpu_sriov_vf(ring->adev))
3273                 gfx_v9_0_ring_emit_de_meta(ring);
3274 }
3275
3276 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring)
3277 {
3278         unsigned ret;
3279         amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
3280         amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
3281         amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
3282         amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */
3283         ret = ring->wptr & ring->buf_mask;
3284         amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */
3285         return ret;
3286 }
3287
3288 static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset)
3289 {
3290         unsigned cur;
3291         BUG_ON(offset > ring->buf_mask);
3292         BUG_ON(ring->ring[offset] != 0x55aa55aa);
3293
3294         cur = (ring->wptr & ring->buf_mask) - 1;
3295         if (likely(cur > offset))
3296                 ring->ring[offset] = cur - offset;
3297         else
3298                 ring->ring[offset] = (ring->ring_size>>2) - offset + cur;
3299 }
3300
3301 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg)
3302 {
3303         struct amdgpu_device *adev = ring->adev;
3304
3305         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
3306         amdgpu_ring_write(ring, 0 |     /* src: register*/
3307                                 (5 << 8) |      /* dst: memory */
3308                                 (1 << 20));     /* write confirm */
3309         amdgpu_ring_write(ring, reg);
3310         amdgpu_ring_write(ring, 0);
3311         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
3312                                 adev->virt.reg_val_offs * 4));
3313         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
3314                                 adev->virt.reg_val_offs * 4));
3315 }
3316
3317 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
3318                                   uint32_t val)
3319 {
3320         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3321         amdgpu_ring_write(ring, (1 << 16)); /* no inc addr */
3322         amdgpu_ring_write(ring, reg);
3323         amdgpu_ring_write(ring, 0);
3324         amdgpu_ring_write(ring, val);
3325 }
3326
3327 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
3328                                                  enum amdgpu_interrupt_state state)
3329 {
3330         u32 cp_int_cntl;
3331
3332         switch (state) {
3333         case AMDGPU_IRQ_STATE_DISABLE:
3334                 cp_int_cntl = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0));
3335                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
3336                                             TIME_STAMP_INT_ENABLE, 0);
3337                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0), cp_int_cntl);
3338                 break;
3339         case AMDGPU_IRQ_STATE_ENABLE:
3340                 cp_int_cntl = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0));
3341                 cp_int_cntl =
3342                         REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
3343                                       TIME_STAMP_INT_ENABLE, 1);
3344                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0), cp_int_cntl);
3345                 break;
3346         default:
3347                 break;
3348         }
3349 }
3350
3351 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
3352                                                      int me, int pipe,
3353                                                      enum amdgpu_interrupt_state state)
3354 {
3355         u32 mec_int_cntl, mec_int_cntl_reg;
3356
3357         /*
3358          * amdgpu controls only pipe 0 of MEC1. That's why this function only
3359          * handles the setting of interrupts for this specific pipe. All other
3360          * pipes' interrupts are set by amdkfd.
3361          */
3362
3363         if (me == 1) {
3364                 switch (pipe) {
3365                 case 0:
3366                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
3367                         break;
3368                 default:
3369                         DRM_DEBUG("invalid pipe %d\n", pipe);
3370                         return;
3371                 }
3372         } else {
3373                 DRM_DEBUG("invalid me %d\n", me);
3374                 return;
3375         }
3376
3377         switch (state) {
3378         case AMDGPU_IRQ_STATE_DISABLE:
3379                 mec_int_cntl = RREG32(mec_int_cntl_reg);
3380                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
3381                                              TIME_STAMP_INT_ENABLE, 0);
3382                 WREG32(mec_int_cntl_reg, mec_int_cntl);
3383                 break;
3384         case AMDGPU_IRQ_STATE_ENABLE:
3385                 mec_int_cntl = RREG32(mec_int_cntl_reg);
3386                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
3387                                              TIME_STAMP_INT_ENABLE, 1);
3388                 WREG32(mec_int_cntl_reg, mec_int_cntl);
3389                 break;
3390         default:
3391                 break;
3392         }
3393 }
3394
3395 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
3396                                              struct amdgpu_irq_src *source,
3397                                              unsigned type,
3398                                              enum amdgpu_interrupt_state state)
3399 {
3400         u32 cp_int_cntl;
3401
3402         switch (state) {
3403         case AMDGPU_IRQ_STATE_DISABLE:
3404                 cp_int_cntl = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0));
3405                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
3406                                             PRIV_REG_INT_ENABLE, 0);
3407                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0), cp_int_cntl);
3408                 break;
3409         case AMDGPU_IRQ_STATE_ENABLE:
3410                 cp_int_cntl = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0));
3411                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
3412                                             PRIV_REG_INT_ENABLE, 1);
3413                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0), cp_int_cntl);
3414                 break;
3415         default:
3416                 break;
3417         }
3418
3419         return 0;
3420 }
3421
3422 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
3423                                               struct amdgpu_irq_src *source,
3424                                               unsigned type,
3425                                               enum amdgpu_interrupt_state state)
3426 {
3427         u32 cp_int_cntl;
3428
3429         switch (state) {
3430         case AMDGPU_IRQ_STATE_DISABLE:
3431                 cp_int_cntl = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0));
3432                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
3433                                             PRIV_INSTR_INT_ENABLE, 0);
3434                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0), cp_int_cntl);
3435                 break;
3436         case AMDGPU_IRQ_STATE_ENABLE:
3437                 cp_int_cntl = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0));
3438                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
3439                                             PRIV_INSTR_INT_ENABLE, 1);
3440                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0), cp_int_cntl);
3441                 break;
3442         default:
3443                 break;
3444         }
3445
3446         return 0;
3447 }
3448
3449 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,
3450                                             struct amdgpu_irq_src *src,
3451                                             unsigned type,
3452                                             enum amdgpu_interrupt_state state)
3453 {
3454         switch (type) {
3455         case AMDGPU_CP_IRQ_GFX_EOP:
3456                 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state);
3457                 break;
3458         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
3459                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
3460                 break;
3461         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
3462                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
3463                 break;
3464         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
3465                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
3466                 break;
3467         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
3468                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
3469                 break;
3470         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
3471                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
3472                 break;
3473         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
3474                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
3475                 break;
3476         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
3477                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
3478                 break;
3479         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
3480                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
3481                 break;
3482         default:
3483                 break;
3484         }
3485         return 0;
3486 }
3487
3488 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev,
3489                             struct amdgpu_irq_src *source,
3490                             struct amdgpu_iv_entry *entry)
3491 {
3492         int i;
3493         u8 me_id, pipe_id, queue_id;
3494         struct amdgpu_ring *ring;
3495
3496         DRM_DEBUG("IH: CP EOP\n");
3497         me_id = (entry->ring_id & 0x0c) >> 2;
3498         pipe_id = (entry->ring_id & 0x03) >> 0;
3499         queue_id = (entry->ring_id & 0x70) >> 4;
3500
3501         switch (me_id) {
3502         case 0:
3503                 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
3504                 break;
3505         case 1:
3506         case 2:
3507                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3508                         ring = &adev->gfx.compute_ring[i];
3509                         /* Per-queue interrupt is supported for MEC starting from VI.
3510                           * The interrupt can only be enabled/disabled per pipe instead of per queue.
3511                           */
3512                         if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
3513                                 amdgpu_fence_process(ring);
3514                 }
3515                 break;
3516         }
3517         return 0;
3518 }
3519
3520 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev,
3521                                  struct amdgpu_irq_src *source,
3522                                  struct amdgpu_iv_entry *entry)
3523 {
3524         DRM_ERROR("Illegal register access in command stream\n");
3525         schedule_work(&adev->reset_work);
3526         return 0;
3527 }
3528
3529 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev,
3530                                   struct amdgpu_irq_src *source,
3531                                   struct amdgpu_iv_entry *entry)
3532 {
3533         DRM_ERROR("Illegal instruction in command stream\n");
3534         schedule_work(&adev->reset_work);
3535         return 0;
3536 }
3537
3538 static int gfx_v9_0_kiq_set_interrupt_state(struct amdgpu_device *adev,
3539                                             struct amdgpu_irq_src *src,
3540                                             unsigned int type,
3541                                             enum amdgpu_interrupt_state state)
3542 {
3543         uint32_t tmp, target;
3544         struct amdgpu_ring *ring = (struct amdgpu_ring *)src->data;
3545
3546         BUG_ON(!ring || (ring->funcs->type != AMDGPU_RING_TYPE_KIQ));
3547
3548         if (ring->me == 1)
3549                 target = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
3550         else
3551                 target = SOC15_REG_OFFSET(GC, 0, mmCP_ME2_PIPE0_INT_CNTL);
3552         target += ring->pipe;
3553
3554         switch (type) {
3555         case AMDGPU_CP_KIQ_IRQ_DRIVER0:
3556                 if (state == AMDGPU_IRQ_STATE_DISABLE) {
3557                         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCPC_INT_CNTL));
3558                         tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
3559                                                  GENERIC2_INT_ENABLE, 0);
3560                         WREG32(SOC15_REG_OFFSET(GC, 0, mmCPC_INT_CNTL), tmp);
3561
3562                         tmp = RREG32(target);
3563                         tmp = REG_SET_FIELD(tmp, CP_ME2_PIPE0_INT_CNTL,
3564                                                  GENERIC2_INT_ENABLE, 0);
3565                         WREG32(target, tmp);
3566                 } else {
3567                         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCPC_INT_CNTL));
3568                         tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
3569                                                  GENERIC2_INT_ENABLE, 1);
3570                         WREG32(SOC15_REG_OFFSET(GC, 0, mmCPC_INT_CNTL), tmp);
3571
3572                         tmp = RREG32(target);
3573                         tmp = REG_SET_FIELD(tmp, CP_ME2_PIPE0_INT_CNTL,
3574                                                  GENERIC2_INT_ENABLE, 1);
3575                         WREG32(target, tmp);
3576                 }
3577                 break;
3578         default:
3579                 BUG(); /* kiq only support GENERIC2_INT now */
3580                 break;
3581         }
3582         return 0;
3583 }
3584
3585 static int gfx_v9_0_kiq_irq(struct amdgpu_device *adev,
3586                             struct amdgpu_irq_src *source,
3587                             struct amdgpu_iv_entry *entry)
3588 {
3589         u8 me_id, pipe_id, queue_id;
3590         struct amdgpu_ring *ring = (struct amdgpu_ring *)source->data;
3591
3592         BUG_ON(!ring || (ring->funcs->type != AMDGPU_RING_TYPE_KIQ));
3593
3594         me_id = (entry->ring_id & 0x0c) >> 2;
3595         pipe_id = (entry->ring_id & 0x03) >> 0;
3596         queue_id = (entry->ring_id & 0x70) >> 4;
3597         DRM_DEBUG("IH: CPC GENERIC2_INT, me:%d, pipe:%d, queue:%d\n",
3598                    me_id, pipe_id, queue_id);
3599
3600         amdgpu_fence_process(ring);
3601         return 0;
3602 }
3603
3604 const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
3605         .name = "gfx_v9_0",
3606         .early_init = gfx_v9_0_early_init,
3607         .late_init = gfx_v9_0_late_init,
3608         .sw_init = gfx_v9_0_sw_init,
3609         .sw_fini = gfx_v9_0_sw_fini,
3610         .hw_init = gfx_v9_0_hw_init,
3611         .hw_fini = gfx_v9_0_hw_fini,
3612         .suspend = gfx_v9_0_suspend,
3613         .resume = gfx_v9_0_resume,
3614         .is_idle = gfx_v9_0_is_idle,
3615         .wait_for_idle = gfx_v9_0_wait_for_idle,
3616         .soft_reset = gfx_v9_0_soft_reset,
3617         .set_clockgating_state = gfx_v9_0_set_clockgating_state,
3618         .set_powergating_state = gfx_v9_0_set_powergating_state,
3619 };
3620
3621 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
3622         .type = AMDGPU_RING_TYPE_GFX,
3623         .align_mask = 0xff,
3624         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
3625         .support_64bit_ptrs = true,
3626         .get_rptr = gfx_v9_0_ring_get_rptr_gfx,
3627         .get_wptr = gfx_v9_0_ring_get_wptr_gfx,
3628         .set_wptr = gfx_v9_0_ring_set_wptr_gfx,
3629         .emit_frame_size =
3630                 20 + /* gfx_v9_0_ring_emit_gds_switch */
3631                 7 + /* gfx_v9_0_ring_emit_hdp_flush */
3632                 5 + /* gfx_v9_0_ring_emit_hdp_invalidate */
3633                 8 + 8 + 8 +/* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */
3634                 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
3635                 128 + 66 + /* gfx_v9_0_ring_emit_vm_flush */
3636                 2 + /* gfx_v9_ring_emit_sb */
3637                 3, /* gfx_v9_ring_emit_cntxcntl */
3638         .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
3639         .emit_ib = gfx_v9_0_ring_emit_ib_gfx,
3640         .emit_fence = gfx_v9_0_ring_emit_fence,
3641         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
3642         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
3643         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
3644         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
3645         .emit_hdp_invalidate = gfx_v9_0_ring_emit_hdp_invalidate,
3646         .test_ring = gfx_v9_0_ring_test_ring,
3647         .test_ib = gfx_v9_0_ring_test_ib,
3648         .insert_nop = amdgpu_ring_insert_nop,
3649         .pad_ib = amdgpu_ring_generic_pad_ib,
3650         .emit_switch_buffer = gfx_v9_ring_emit_sb,
3651         .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
3652         .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
3653         .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec,
3654 };
3655
3656 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
3657         .type = AMDGPU_RING_TYPE_COMPUTE,
3658         .align_mask = 0xff,
3659         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
3660         .support_64bit_ptrs = true,
3661         .get_rptr = gfx_v9_0_ring_get_rptr_compute,
3662         .get_wptr = gfx_v9_0_ring_get_wptr_compute,
3663         .set_wptr = gfx_v9_0_ring_set_wptr_compute,
3664         .emit_frame_size =
3665                 20 + /* gfx_v9_0_ring_emit_gds_switch */
3666                 7 + /* gfx_v9_0_ring_emit_hdp_flush */
3667                 5 + /* gfx_v9_0_ring_emit_hdp_invalidate */
3668                 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
3669                 64 + /* gfx_v9_0_ring_emit_vm_flush */
3670                 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */
3671         .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_compute */
3672         .emit_ib = gfx_v9_0_ring_emit_ib_compute,
3673         .emit_fence = gfx_v9_0_ring_emit_fence,
3674         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
3675         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
3676         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
3677         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
3678         .emit_hdp_invalidate = gfx_v9_0_ring_emit_hdp_invalidate,
3679         .test_ring = gfx_v9_0_ring_test_ring,
3680         .test_ib = gfx_v9_0_ring_test_ib,
3681         .insert_nop = amdgpu_ring_insert_nop,
3682         .pad_ib = amdgpu_ring_generic_pad_ib,
3683 };
3684
3685 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
3686         .type = AMDGPU_RING_TYPE_KIQ,
3687         .align_mask = 0xff,
3688         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
3689         .support_64bit_ptrs = true,
3690         .get_rptr = gfx_v9_0_ring_get_rptr_compute,
3691         .get_wptr = gfx_v9_0_ring_get_wptr_compute,
3692         .set_wptr = gfx_v9_0_ring_set_wptr_compute,
3693         .emit_frame_size =
3694                 20 + /* gfx_v9_0_ring_emit_gds_switch */
3695                 7 + /* gfx_v9_0_ring_emit_hdp_flush */
3696                 5 + /* gfx_v9_0_ring_emit_hdp_invalidate */
3697                 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
3698                 64 + /* gfx_v9_0_ring_emit_vm_flush */
3699                 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */
3700         .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_compute */
3701         .emit_ib = gfx_v9_0_ring_emit_ib_compute,
3702         .emit_fence = gfx_v9_0_ring_emit_fence_kiq,
3703         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
3704         .emit_hdp_invalidate = gfx_v9_0_ring_emit_hdp_invalidate,
3705         .test_ring = gfx_v9_0_ring_test_ring,
3706         .test_ib = gfx_v9_0_ring_test_ib,
3707         .insert_nop = amdgpu_ring_insert_nop,
3708         .pad_ib = amdgpu_ring_generic_pad_ib,
3709         .emit_rreg = gfx_v9_0_ring_emit_rreg,
3710         .emit_wreg = gfx_v9_0_ring_emit_wreg,
3711 };
3712
3713 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev)
3714 {
3715         int i;
3716
3717         adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq;
3718
3719         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
3720                 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx;
3721
3722         for (i = 0; i < adev->gfx.num_compute_rings; i++)
3723                 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute;
3724 }
3725
3726 static const struct amdgpu_irq_src_funcs gfx_v9_0_kiq_irq_funcs = {
3727         .set = gfx_v9_0_kiq_set_interrupt_state,
3728         .process = gfx_v9_0_kiq_irq,
3729 };
3730
3731 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = {
3732         .set = gfx_v9_0_set_eop_interrupt_state,
3733         .process = gfx_v9_0_eop_irq,
3734 };
3735
3736 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = {
3737         .set = gfx_v9_0_set_priv_reg_fault_state,
3738         .process = gfx_v9_0_priv_reg_irq,
3739 };
3740
3741 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = {
3742         .set = gfx_v9_0_set_priv_inst_fault_state,
3743         .process = gfx_v9_0_priv_inst_irq,
3744 };
3745
3746 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)
3747 {
3748         adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
3749         adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs;
3750
3751         adev->gfx.priv_reg_irq.num_types = 1;
3752         adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs;
3753
3754         adev->gfx.priv_inst_irq.num_types = 1;
3755         adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;
3756
3757         adev->gfx.kiq.irq.num_types = AMDGPU_CP_KIQ_IRQ_LAST;
3758         adev->gfx.kiq.irq.funcs = &gfx_v9_0_kiq_irq_funcs;
3759 }
3760
3761 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
3762 {
3763         switch (adev->asic_type) {
3764         case CHIP_VEGA10:
3765                 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;
3766                 break;
3767         default:
3768                 break;
3769         }
3770 }
3771
3772 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
3773 {
3774         /* init asci gds info */
3775         adev->gds.mem.total_size = RREG32(SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE));
3776         adev->gds.gws.total_size = 64;
3777         adev->gds.oa.total_size = 16;
3778
3779         if (adev->gds.mem.total_size == 64 * 1024) {
3780                 adev->gds.mem.gfx_partition_size = 4096;
3781                 adev->gds.mem.cs_partition_size = 4096;
3782
3783                 adev->gds.gws.gfx_partition_size = 4;
3784                 adev->gds.gws.cs_partition_size = 4;
3785
3786                 adev->gds.oa.gfx_partition_size = 4;
3787                 adev->gds.oa.cs_partition_size = 1;
3788         } else {
3789                 adev->gds.mem.gfx_partition_size = 1024;
3790                 adev->gds.mem.cs_partition_size = 1024;
3791
3792                 adev->gds.gws.gfx_partition_size = 16;
3793                 adev->gds.gws.cs_partition_size = 16;
3794
3795                 adev->gds.oa.gfx_partition_size = 4;
3796                 adev->gds.oa.cs_partition_size = 4;
3797         }
3798 }
3799
3800 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev)
3801 {
3802         u32 data, mask;
3803
3804         data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG));
3805         data |= RREG32(SOC15_REG_OFFSET(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG));
3806
3807         data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
3808         data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
3809
3810         mask = gfx_v9_0_create_bitmask(adev->gfx.config.max_cu_per_sh);
3811
3812         return (~data) & mask;
3813 }
3814
3815 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
3816                                  struct amdgpu_cu_info *cu_info)
3817 {
3818         int i, j, k, counter, active_cu_number = 0;
3819         u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
3820
3821         if (!adev || !cu_info)
3822                 return -EINVAL;
3823
3824         memset(cu_info, 0, sizeof(*cu_info));
3825
3826         mutex_lock(&adev->grbm_idx_mutex);
3827         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
3828                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
3829                         mask = 1;
3830                         ao_bitmap = 0;
3831                         counter = 0;
3832                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
3833                         bitmap = gfx_v9_0_get_cu_active_bitmap(adev);
3834                         cu_info->bitmap[i][j] = bitmap;
3835
3836                         for (k = 0; k < 16; k ++) {
3837                                 if (bitmap & mask) {
3838                                         if (counter < 2)
3839                                                 ao_bitmap |= mask;
3840                                         counter ++;
3841                                 }
3842                                 mask <<= 1;
3843                         }
3844                         active_cu_number += counter;
3845                         ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
3846                 }
3847         }
3848         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3849         mutex_unlock(&adev->grbm_idx_mutex);
3850
3851         cu_info->number = active_cu_number;
3852         cu_info->ao_cu_mask = ao_cu_mask;
3853
3854         return 0;
3855 }
3856
3857 static int gfx_v9_0_init_queue(struct amdgpu_ring *ring)
3858 {
3859         int r, j;
3860         u32 tmp;
3861         bool use_doorbell = true;
3862         u64 hqd_gpu_addr;
3863         u64 mqd_gpu_addr;
3864         u64 eop_gpu_addr;
3865         u64 wb_gpu_addr;
3866         u32 *buf;
3867         struct v9_mqd *mqd;
3868         struct amdgpu_device *adev;
3869
3870         adev = ring->adev;
3871         if (ring->mqd_obj == NULL) {
3872                 r = amdgpu_bo_create(adev,
3873                                 sizeof(struct v9_mqd),
3874                                 PAGE_SIZE,true,
3875                                 AMDGPU_GEM_DOMAIN_GTT, 0, NULL,
3876                                 NULL, &ring->mqd_obj);
3877                 if (r) {
3878                         dev_warn(adev->dev, "(%d) create MQD bo failed\n", r);
3879                         return r;
3880                 }
3881         }
3882
3883         r = amdgpu_bo_reserve(ring->mqd_obj, false);
3884         if (unlikely(r != 0)) {
3885                 gfx_v9_0_cp_compute_fini(adev);
3886                 return r;
3887         }
3888
3889         r = amdgpu_bo_pin(ring->mqd_obj, AMDGPU_GEM_DOMAIN_GTT,
3890                                   &mqd_gpu_addr);
3891         if (r) {
3892                 dev_warn(adev->dev, "(%d) pin MQD bo failed\n", r);
3893                 gfx_v9_0_cp_compute_fini(adev);
3894                 return r;
3895         }
3896         r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&buf);
3897         if (r) {
3898                 dev_warn(adev->dev, "(%d) map MQD bo failed\n", r);
3899                 gfx_v9_0_cp_compute_fini(adev);
3900                 return r;
3901         }
3902
3903         /* init the mqd struct */
3904         memset(buf, 0, sizeof(struct v9_mqd));
3905
3906         mqd = (struct v9_mqd *)buf;
3907         mqd->header = 0xC0310800;
3908         mqd->compute_pipelinestat_enable = 0x00000001;
3909         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
3910         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
3911         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
3912         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
3913         mqd->compute_misc_reserved = 0x00000003;
3914         mutex_lock(&adev->srbm_mutex);
3915         soc15_grbm_select(adev, ring->me,
3916                                ring->pipe,
3917                                ring->queue, 0);
3918         /* disable wptr polling */
3919         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PQ_WPTR_POLL_CNTL));
3920         tmp = REG_SET_FIELD(tmp, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3921         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PQ_WPTR_POLL_CNTL), tmp);
3922
3923         /* write the EOP addr */
3924         BUG_ON(ring->me != 1 || ring->pipe != 0); /* can't handle other cases eop address */
3925         eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr + (ring->queue * MEC_HPD_SIZE);
3926         eop_gpu_addr >>= 8;
3927
3928         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_BASE_ADDR), lower_32_bits(eop_gpu_addr));
3929         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI), upper_32_bits(eop_gpu_addr));
3930         mqd->cp_hqd_eop_base_addr_lo = lower_32_bits(eop_gpu_addr);
3931         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_gpu_addr);
3932
3933         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3934         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_CONTROL));
3935         tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3936                                     (order_base_2(MEC_HPD_SIZE / 4) - 1));
3937         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_CONTROL), tmp);
3938
3939         /* enable doorbell? */
3940         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL));
3941         if (use_doorbell)
3942                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
3943         else
3944                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 0);
3945
3946         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL), tmp);
3947         mqd->cp_hqd_pq_doorbell_control = tmp;
3948
3949         /* disable the queue if it's active */
3950         ring->wptr = 0;
3951         mqd->cp_hqd_dequeue_request = 0;
3952         mqd->cp_hqd_pq_rptr = 0;
3953         mqd->cp_hqd_pq_wptr_lo = 0;
3954         mqd->cp_hqd_pq_wptr_hi = 0;
3955         if (RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE)) & 1) {
3956                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_DEQUEUE_REQUEST), 1);
3957                 for (j = 0; j < adev->usec_timeout; j++) {
3958                         if (!(RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE)) & 1))
3959                                 break;
3960                         udelay(1);
3961                 }
3962                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_DEQUEUE_REQUEST), mqd->cp_hqd_dequeue_request);
3963                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_RPTR), mqd->cp_hqd_pq_rptr);
3964                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_LO), mqd->cp_hqd_pq_wptr_lo);
3965                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI), mqd->cp_hqd_pq_wptr_hi);
3966         }
3967
3968         /* set the pointer to the MQD */
3969         mqd->cp_mqd_base_addr_lo = mqd_gpu_addr & 0xfffffffc;
3970         mqd->cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
3971         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR), mqd->cp_mqd_base_addr_lo);
3972         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR_HI), mqd->cp_mqd_base_addr_hi);
3973
3974         /* set MQD vmid to 0 */
3975         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MQD_CONTROL));
3976         tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
3977         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MQD_CONTROL), tmp);
3978         mqd->cp_mqd_control = tmp;
3979
3980         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3981         hqd_gpu_addr = ring->gpu_addr >> 8;
3982         mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
3983         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3984         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_BASE), mqd->cp_hqd_pq_base_lo);
3985         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_BASE_HI), mqd->cp_hqd_pq_base_hi);
3986
3987         /* set up the HQD, this is similar to CP_RB0_CNTL */
3988         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_CONTROL));
3989         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
3990                 (order_base_2(ring->ring_size / 4) - 1));
3991         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
3992                 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
3993 #ifdef __BIG_ENDIAN
3994         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
3995 #endif
3996         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
3997         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
3998         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3999         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
4000         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_CONTROL), tmp);
4001         mqd->cp_hqd_pq_control = tmp;
4002
4003         /* set the wb address wether it's enabled or not */
4004         wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
4005         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
4006         mqd->cp_hqd_pq_rptr_report_addr_hi =
4007         upper_32_bits(wb_gpu_addr) & 0xffff;
4008         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR),
4009                 mqd->cp_hqd_pq_rptr_report_addr_lo);
4010         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI),
4011                 mqd->cp_hqd_pq_rptr_report_addr_hi);
4012
4013         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
4014         wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
4015         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
4016         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4017         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR),
4018                 mqd->cp_hqd_pq_wptr_poll_addr_lo);
4019         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI),
4020                 mqd->cp_hqd_pq_wptr_poll_addr_hi);
4021
4022         /* enable the doorbell if requested */
4023         if (use_doorbell) {
4024                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER),
4025                         (AMDGPU_DOORBELL64_KIQ * 2) << 2);
4026                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER),
4027                         (AMDGPU_DOORBELL64_MEC_RING7 * 2) << 2);
4028                 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL));
4029                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4030                         DOORBELL_OFFSET, ring->doorbell_index);
4031                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
4032                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_SOURCE, 0);
4033                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_HIT, 0);
4034                 mqd->cp_hqd_pq_doorbell_control = tmp;
4035
4036         } else {
4037                 mqd->cp_hqd_pq_doorbell_control = 0;
4038         }
4039         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL),
4040                 mqd->cp_hqd_pq_doorbell_control);
4041
4042         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4043         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_LO), mqd->cp_hqd_pq_wptr_lo);
4044         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI), mqd->cp_hqd_pq_wptr_hi);
4045
4046         /* set the vmid for the queue */
4047         mqd->cp_hqd_vmid = 0;
4048         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_VMID), mqd->cp_hqd_vmid);
4049
4050         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PERSISTENT_STATE));
4051         tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
4052         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PERSISTENT_STATE), tmp);
4053         mqd->cp_hqd_persistent_state = tmp;
4054
4055         /* activate the queue */
4056         mqd->cp_hqd_active = 1;
4057         WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE), mqd->cp_hqd_active);
4058
4059         soc15_grbm_select(adev, 0, 0, 0, 0);
4060         mutex_unlock(&adev->srbm_mutex);
4061
4062         amdgpu_bo_kunmap(ring->mqd_obj);
4063         amdgpu_bo_unreserve(ring->mqd_obj);
4064
4065         if (use_doorbell) {
4066                 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PQ_STATUS));
4067                 tmp = REG_SET_FIELD(tmp, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
4068                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PQ_STATUS), tmp);
4069         }
4070
4071         return 0;
4072 }
4073
4074 const struct amdgpu_ip_block_version gfx_v9_0_ip_block =
4075 {
4076         .type = AMD_IP_BLOCK_TYPE_GFX,
4077         .major = 9,
4078         .minor = 0,
4079         .rev = 0,
4080         .funcs = &gfx_v9_0_ip_funcs,
4081 };