drivers/gpu/drm/nouveau/nve0_graph.c

   1 /*
   2  * Copyright 2010 Red Hat Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Ben Skeggs
  23  */
  24
  25 #include <linux/firmware.h>
  26 #include <linux/module.h>
  27
  28 #include "drmP.h"
  29
  30 #include "nouveau_drv.h"
  31 #include "nouveau_mm.h"
  32
  33 #include "nve0_graph.h"
  34
  35 static void
  36 nve0_graph_ctxctl_debug_unit(struct drm_device *dev, u32 base)
  37 {
  38         NV_INFO(dev, "PGRAPH: %06x - done 0x%08x\n", base,
  39                 nv_rd32(dev, base + 0x400));
  40         NV_INFO(dev, "PGRAPH: %06x - stat 0x%08x 0x%08x 0x%08x 0x%08x\n", base,
  41                 nv_rd32(dev, base + 0x800), nv_rd32(dev, base + 0x804),
  42                 nv_rd32(dev, base + 0x808), nv_rd32(dev, base + 0x80c));
  43         NV_INFO(dev, "PGRAPH: %06x - stat 0x%08x 0x%08x 0x%08x 0x%08x\n", base,
  44                 nv_rd32(dev, base + 0x810), nv_rd32(dev, base + 0x814),
  45                 nv_rd32(dev, base + 0x818), nv_rd32(dev, base + 0x81c));
  46 }
  47
  48 static void
  49 nve0_graph_ctxctl_debug(struct drm_device *dev)
  50 {
  51         u32 gpcnr = nv_rd32(dev, 0x409604) & 0xffff;
  52         u32 gpc;
  53
  54         nve0_graph_ctxctl_debug_unit(dev, 0x409000);
  55         for (gpc = 0; gpc < gpcnr; gpc++)
  56                 nve0_graph_ctxctl_debug_unit(dev, 0x502000 + (gpc * 0x8000));
  57 }
  58
  59 static int
  60 nve0_graph_load_context(struct nouveau_channel *chan)
  61 {
  62         struct drm_device *dev = chan->dev;
  63
  64         nv_wr32(dev, 0x409840, 0x00000030);
  65         nv_wr32(dev, 0x409500, 0x80000000 | chan->ramin->vinst >> 12);
  66         nv_wr32(dev, 0x409504, 0x00000003);
  67         if (!nv_wait(dev, 0x409800, 0x00000010, 0x00000010))
  68                 NV_ERROR(dev, "PGRAPH: load_ctx timeout\n");
  69
  70         return 0;
  71 }
  72
  73 static int
  74 nve0_graph_unload_context_to(struct drm_device *dev, u64 chan)
  75 {
  76         nv_wr32(dev, 0x409840, 0x00000003);
  77         nv_wr32(dev, 0x409500, 0x80000000 | chan >> 12);
  78         nv_wr32(dev, 0x409504, 0x00000009);
  79         if (!nv_wait(dev, 0x409800, 0x00000001, 0x00000000)) {
  80                 NV_ERROR(dev, "PGRAPH: unload_ctx timeout\n");
  81                 return -EBUSY;
  82         }
  83
  84         return 0;
  85 }
  86
  87 static int
  88 nve0_graph_construct_context(struct nouveau_channel *chan)
  89 {
  90         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
  91         struct nve0_graph_priv *priv = nv_engine(chan->dev, NVOBJ_ENGINE_GR);
  92         struct nve0_graph_chan *grch = chan->engctx[NVOBJ_ENGINE_GR];
  93         struct drm_device *dev = chan->dev;
  94         int ret, i;
  95         u32 *ctx;
  96
  97         ctx = kmalloc(priv->grctx_size, GFP_KERNEL);
  98         if (!ctx)
  99                 return -ENOMEM;
 100
 101         nve0_graph_load_context(chan);
 102
 103         nv_wo32(grch->grctx, 0x1c, 1);
 104         nv_wo32(grch->grctx, 0x20, 0);
 105         nv_wo32(grch->grctx, 0x28, 0);
 106         nv_wo32(grch->grctx, 0x2c, 0);
 107         dev_priv->engine.instmem.flush(dev);
 108
 109         ret = nve0_grctx_generate(chan);
 110         if (ret)
 111                 goto err;
 112
 113         ret = nve0_graph_unload_context_to(dev, chan->ramin->vinst);
 114         if (ret)
 115                 goto err;
 116
 117         for (i = 0; i < priv->grctx_size; i += 4)
 118                 ctx[i / 4] = nv_ro32(grch->grctx, i);
 119
 120         priv->grctx_vals = ctx;
 121         return 0;
 122
 123 err:
 124         kfree(ctx);
 125         return ret;
 126 }
 127
 128 static int
 129 nve0_graph_create_context_mmio_list(struct nouveau_channel *chan)
 130 {
 131         struct nve0_graph_priv *priv = nv_engine(chan->dev, NVOBJ_ENGINE_GR);
 132         struct nve0_graph_chan *grch = chan->engctx[NVOBJ_ENGINE_GR];
 133         struct drm_device *dev = chan->dev;
 134         u32 magic[GPC_MAX][2];
 135         u16 offset = 0x0000;
 136         int gpc;
 137         int ret;
 138
 139         ret = nouveau_gpuobj_new(dev, chan, 0x3000, 256, NVOBJ_FLAG_VM,
 140                                  &grch->unk408004);
 141         if (ret)
 142                 return ret;
 143
 144         ret = nouveau_gpuobj_new(dev, chan, 0x8000, 256, NVOBJ_FLAG_VM,
 145                                  &grch->unk40800c);
 146         if (ret)
 147                 return ret;
 148
 149         ret = nouveau_gpuobj_new(dev, chan, 384 * 1024, 4096,
 150                                  NVOBJ_FLAG_VM | NVOBJ_FLAG_VM_USER,
 151                                  &grch->unk418810);
 152         if (ret)
 153                 return ret;
 154
 155         ret = nouveau_gpuobj_new(dev, chan, 0x1000, 0, NVOBJ_FLAG_VM,
 156                                  &grch->mmio);
 157         if (ret)
 158                 return ret;
 159
 160 #define mmio(r,v) do {                                                         \
 161         nv_wo32(grch->mmio, (grch->mmio_nr * 8) + 0, (r));                     \
 162         nv_wo32(grch->mmio, (grch->mmio_nr * 8) + 4, (v));                     \
 163         grch->mmio_nr++;                                                       \
 164 } while (0)
 165         mmio(0x40800c, grch->unk40800c->linst >> 8);
 166         mmio(0x408010, 0x80000000);
 167         mmio(0x419004, grch->unk40800c->linst >> 8);
 168         mmio(0x419008, 0x00000000);
 169         mmio(0x4064cc, 0x80000000);
 170         mmio(0x408004, grch->unk408004->linst >> 8);
 171         mmio(0x408008, 0x80000030);
 172         mmio(0x418808, grch->unk408004->linst >> 8);
 173         mmio(0x41880c, 0x80000030);
 174         mmio(0x4064c8, 0x01800600);
 175         mmio(0x418810, 0x80000000 | grch->unk418810->linst >> 12);
 176         mmio(0x419848, 0x10000000 | grch->unk418810->linst >> 12);
 177         mmio(0x405830, 0x02180648);
 178         mmio(0x4064c4, 0x0192ffff);
 179
 180         for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
 181                 u16 magic0 = 0x0218 * priv->tpc_nr[gpc];
 182                 u16 magic1 = 0x0648 * priv->tpc_nr[gpc];
 183                 magic[gpc][0]  = 0x10000000 | (magic0 << 16) | offset;
 184                 magic[gpc][1]  = 0x00000000 | (magic1 << 16);
 185                 offset += 0x0324 * priv->tpc_nr[gpc];
 186         }
 187
 188         for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
 189                 mmio(GPC_UNIT(gpc, 0x30c0), magic[gpc][0]);
 190                 mmio(GPC_UNIT(gpc, 0x30e4), magic[gpc][1] | offset);
 191                 offset += 0x07ff * priv->tpc_nr[gpc];
 192         }
 193
 194         mmio(0x17e91c, 0x06060609);
 195         mmio(0x17e920, 0x00090a05);
 196 #undef mmio
 197         return 0;
 198 }
 199
 200 static int
 201 nve0_graph_context_new(struct nouveau_channel *chan, int engine)
 202 {
 203         struct drm_device *dev = chan->dev;
 204         struct drm_nouveau_private *dev_priv = dev->dev_private;
 205         struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
 206         struct nve0_graph_priv *priv = nv_engine(dev, engine);
 207         struct nve0_graph_chan *grch;
 208         struct nouveau_gpuobj *grctx;
 209         int ret, i;
 210
 211         grch = kzalloc(sizeof(*grch), GFP_KERNEL);
 212         if (!grch)
 213                 return -ENOMEM;
 214         chan->engctx[NVOBJ_ENGINE_GR] = grch;
 215
 216         ret = nouveau_gpuobj_new(dev, chan, priv->grctx_size, 256,
 217                                  NVOBJ_FLAG_VM | NVOBJ_FLAG_ZERO_ALLOC,
 218                                  &grch->grctx);
 219         if (ret)
 220                 goto error;
 221         grctx = grch->grctx;
 222
 223         ret = nve0_graph_create_context_mmio_list(chan);
 224         if (ret)
 225                 goto error;
 226
 227         nv_wo32(chan->ramin, 0x0210, lower_32_bits(grctx->linst) | 4);
 228         nv_wo32(chan->ramin, 0x0214, upper_32_bits(grctx->linst));
 229         pinstmem->flush(dev);
 230
 231         if (!priv->grctx_vals) {
 232                 ret = nve0_graph_construct_context(chan);
 233                 if (ret)
 234                         goto error;
 235         }
 236
 237         for (i = 0; i < priv->grctx_size; i += 4)
 238                 nv_wo32(grctx, i, priv->grctx_vals[i / 4]);
 239         nv_wo32(grctx, 0xf4, 0);
 240         nv_wo32(grctx, 0xf8, 0);
 241         nv_wo32(grctx, 0x10, grch->mmio_nr);
 242         nv_wo32(grctx, 0x14, lower_32_bits(grch->mmio->linst));
 243         nv_wo32(grctx, 0x18, upper_32_bits(grch->mmio->linst));
 244         nv_wo32(grctx, 0x1c, 1);
 245         nv_wo32(grctx, 0x20, 0);
 246         nv_wo32(grctx, 0x28, 0);
 247         nv_wo32(grctx, 0x2c, 0);
 248
 249         pinstmem->flush(dev);
 250         return 0;
 251
 252 error:
 253         priv->base.context_del(chan, engine);
 254         return ret;
 255 }
 256
 257 static void
 258 nve0_graph_context_del(struct nouveau_channel *chan, int engine)
 259 {
 260         struct nve0_graph_chan *grch = chan->engctx[engine];
 261
 262         nouveau_gpuobj_ref(NULL, &grch->mmio);
 263         nouveau_gpuobj_ref(NULL, &grch->unk418810);
 264         nouveau_gpuobj_ref(NULL, &grch->unk40800c);
 265         nouveau_gpuobj_ref(NULL, &grch->unk408004);
 266         nouveau_gpuobj_ref(NULL, &grch->grctx);
 267         chan->engctx[engine] = NULL;
 268 }
 269
 270 static int
 271 nve0_graph_object_new(struct nouveau_channel *chan, int engine,
 272                       u32 handle, u16 class)
 273 {
 274         return 0;
 275 }
 276
 277 static int
 278 nve0_graph_fini(struct drm_device *dev, int engine, bool suspend)
 279 {
 280         return 0;
 281 }
 282
 283 static void
 284 nve0_graph_init_obj418880(struct drm_device *dev)
 285 {
 286         struct nve0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
 287         int i;
 288
 289         nv_wr32(dev, GPC_BCAST(0x0880), 0x00000000);
 290         nv_wr32(dev, GPC_BCAST(0x08a4), 0x00000000);
 291         for (i = 0; i < 4; i++)
 292                 nv_wr32(dev, GPC_BCAST(0x0888) + (i * 4), 0x00000000);
 293         nv_wr32(dev, GPC_BCAST(0x08b4), priv->unk4188b4->vinst >> 8);
 294         nv_wr32(dev, GPC_BCAST(0x08b8), priv->unk4188b8->vinst >> 8);
 295 }
 296
 297 static void
 298 nve0_graph_init_regs(struct drm_device *dev)
 299 {
 300         nv_wr32(dev, 0x400080, 0x003083c2);
 301         nv_wr32(dev, 0x400088, 0x0001ffe7);
 302         nv_wr32(dev, 0x40008c, 0x00000000);
 303         nv_wr32(dev, 0x400090, 0x00000030);
 304         nv_wr32(dev, 0x40013c, 0x003901f7);
 305         nv_wr32(dev, 0x400140, 0x00000100);
 306         nv_wr32(dev, 0x400144, 0x00000000);
 307         nv_wr32(dev, 0x400148, 0x00000110);
 308         nv_wr32(dev, 0x400138, 0x00000000);
 309         nv_wr32(dev, 0x400130, 0x00000000);
 310         nv_wr32(dev, 0x400134, 0x00000000);
 311         nv_wr32(dev, 0x400124, 0x00000002);
 312 }
 313
 314 static void
 315 nve0_graph_init_units(struct drm_device *dev)
 316 {
 317         nv_wr32(dev, 0x409ffc, 0x00000000);
 318         nv_wr32(dev, 0x409c14, 0x00003e3e);
 319         nv_wr32(dev, 0x409c24, 0x000f0000);
 320
 321         nv_wr32(dev, 0x404000, 0xc0000000);
 322         nv_wr32(dev, 0x404600, 0xc0000000);
 323         nv_wr32(dev, 0x408030, 0xc0000000);
 324         nv_wr32(dev, 0x404490, 0xc0000000);
 325         nv_wr32(dev, 0x406018, 0xc0000000);
 326         nv_wr32(dev, 0x407020, 0xc0000000);
 327         nv_wr32(dev, 0x405840, 0xc0000000);
 328         nv_wr32(dev, 0x405844, 0x00ffffff);
 329
 330         nv_mask(dev, 0x419cc0, 0x00000008, 0x00000008);
 331         nv_mask(dev, 0x419eb4, 0x00001000, 0x00001000);
 332
 333 }
 334
 335 static void
 336 nve0_graph_init_gpc_0(struct drm_device *dev)
 337 {
 338         struct nve0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
 339         const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, priv->tpc_total);
 340         u32 data[TPC_MAX / 8];
 341         u8  tpcnr[GPC_MAX];
 342         int i, gpc, tpc;
 343
 344         nv_wr32(dev, GPC_UNIT(0, 0x3018), 0x00000001);
 345
 346         memset(data, 0x00, sizeof(data));
 347         memcpy(tpcnr, priv->tpc_nr, sizeof(priv->tpc_nr));
 348         for (i = 0, gpc = -1; i < priv->tpc_total; i++) {
 349                 do {
 350                         gpc = (gpc + 1) % priv->gpc_nr;
 351                 } while (!tpcnr[gpc]);
 352                 tpc = priv->tpc_nr[gpc] - tpcnr[gpc]--;
 353
 354                 data[i / 8] |= tpc << ((i % 8) * 4);
 355         }
 356
 357         nv_wr32(dev, GPC_BCAST(0x0980), data[0]);
 358         nv_wr32(dev, GPC_BCAST(0x0984), data[1]);
 359         nv_wr32(dev, GPC_BCAST(0x0988), data[2]);
 360         nv_wr32(dev, GPC_BCAST(0x098c), data[3]);
 361
 362         for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
 363                 nv_wr32(dev, GPC_UNIT(gpc, 0x0914), priv->magic_not_rop_nr << 8 |
 364                                                   priv->tpc_nr[gpc]);
 365                 nv_wr32(dev, GPC_UNIT(gpc, 0x0910), 0x00040000 | priv->tpc_total);
 366                 nv_wr32(dev, GPC_UNIT(gpc, 0x0918), magicgpc918);
 367         }
 368
 369         nv_wr32(dev, GPC_BCAST(0x1bd4), magicgpc918);
 370         nv_wr32(dev, GPC_BCAST(0x08ac), nv_rd32(dev, 0x100800));
 371 }
 372
 373 static void
 374 nve0_graph_init_gpc_1(struct drm_device *dev)
 375 {
 376         struct nve0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
 377         int gpc, tpc;
 378
 379         for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
 380                 nv_wr32(dev, GPC_UNIT(gpc, 0x3038), 0xc0000000);
 381                 nv_wr32(dev, GPC_UNIT(gpc, 0x0420), 0xc0000000);
 382                 nv_wr32(dev, GPC_UNIT(gpc, 0x0900), 0xc0000000);
 383                 nv_wr32(dev, GPC_UNIT(gpc, 0x1028), 0xc0000000);
 384                 nv_wr32(dev, GPC_UNIT(gpc, 0x0824), 0xc0000000);
 385                 for (tpc = 0; tpc < priv->tpc_nr[gpc]; tpc++) {
 386                         nv_wr32(dev, TPC_UNIT(gpc, tpc, 0x508), 0xffffffff);
 387                         nv_wr32(dev, TPC_UNIT(gpc, tpc, 0x50c), 0xffffffff);
 388                         nv_wr32(dev, TPC_UNIT(gpc, tpc, 0x224), 0xc0000000);
 389                         nv_wr32(dev, TPC_UNIT(gpc, tpc, 0x48c), 0xc0000000);
 390                         nv_wr32(dev, TPC_UNIT(gpc, tpc, 0x084), 0xc0000000);
 391                         nv_wr32(dev, TPC_UNIT(gpc, tpc, 0x644), 0x001ffffe);
 392                         nv_wr32(dev, TPC_UNIT(gpc, tpc, 0x64c), 0x0000000f);
 393                 }
 394                 nv_wr32(dev, GPC_UNIT(gpc, 0x2c90), 0xffffffff);
 395                 nv_wr32(dev, GPC_UNIT(gpc, 0x2c94), 0xffffffff);
 396         }
 397 }
 398
 399 static void
 400 nve0_graph_init_rop(struct drm_device *dev)
 401 {
 402         struct nve0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
 403         int rop;
 404
 405         for (rop = 0; rop < priv->rop_nr; rop++) {
 406                 nv_wr32(dev, ROP_UNIT(rop, 0x144), 0xc0000000);
 407                 nv_wr32(dev, ROP_UNIT(rop, 0x070), 0xc0000000);
 408                 nv_wr32(dev, ROP_UNIT(rop, 0x204), 0xffffffff);
 409                 nv_wr32(dev, ROP_UNIT(rop, 0x208), 0xffffffff);
 410         }
 411 }
 412
 413 static void
 414 nve0_graph_init_fuc(struct drm_device *dev, u32 fuc_base,
 415                     struct nve0_graph_fuc *code, struct nve0_graph_fuc *data)
 416 {
 417         int i;
 418
 419         nv_wr32(dev, fuc_base + 0x01c0, 0x01000000);
 420         for (i = 0; i < data->size / 4; i++)
 421                 nv_wr32(dev, fuc_base + 0x01c4, data->data[i]);
 422
 423         nv_wr32(dev, fuc_base + 0x0180, 0x01000000);
 424         for (i = 0; i < code->size / 4; i++) {
 425                 if ((i & 0x3f) == 0)
 426                         nv_wr32(dev, fuc_base + 0x0188, i >> 6);
 427                 nv_wr32(dev, fuc_base + 0x0184, code->data[i]);
 428         }
 429 }
 430
 431 static int
 432 nve0_graph_init_ctxctl(struct drm_device *dev)
 433 {
 434         struct nve0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
 435         u32 r000260;
 436
 437         /* load fuc microcode */
 438         r000260 = nv_mask(dev, 0x000260, 0x00000001, 0x00000000);
 439         nve0_graph_init_fuc(dev, 0x409000, &priv->fuc409c, &priv->fuc409d);
 440         nve0_graph_init_fuc(dev, 0x41a000, &priv->fuc41ac, &priv->fuc41ad);
 441         nv_wr32(dev, 0x000260, r000260);
 442
 443         /* start both of them running */
 444         nv_wr32(dev, 0x409840, 0xffffffff);
 445         nv_wr32(dev, 0x41a10c, 0x00000000);
 446         nv_wr32(dev, 0x40910c, 0x00000000);
 447         nv_wr32(dev, 0x41a100, 0x00000002);
 448         nv_wr32(dev, 0x409100, 0x00000002);
 449         if (!nv_wait(dev, 0x409800, 0x00000001, 0x00000001))
 450                 NV_INFO(dev, "0x409800 wait failed\n");
 451
 452         nv_wr32(dev, 0x409840, 0xffffffff);
 453         nv_wr32(dev, 0x409500, 0x7fffffff);
 454         nv_wr32(dev, 0x409504, 0x00000021);
 455
 456         nv_wr32(dev, 0x409840, 0xffffffff);
 457         nv_wr32(dev, 0x409500, 0x00000000);
 458         nv_wr32(dev, 0x409504, 0x00000010);
 459         if (!nv_wait_ne(dev, 0x409800, 0xffffffff, 0x00000000)) {
 460                 NV_ERROR(dev, "fuc09 req 0x10 timeout\n");
 461                 return -EBUSY;
 462         }
 463         priv->grctx_size = nv_rd32(dev, 0x409800);
 464
 465         nv_wr32(dev, 0x409840, 0xffffffff);
 466         nv_wr32(dev, 0x409500, 0x00000000);
 467         nv_wr32(dev, 0x409504, 0x00000016);
 468         if (!nv_wait_ne(dev, 0x409800, 0xffffffff, 0x00000000)) {
 469                 NV_ERROR(dev, "fuc09 req 0x16 timeout\n");
 470                 return -EBUSY;
 471         }
 472
 473         nv_wr32(dev, 0x409840, 0xffffffff);
 474         nv_wr32(dev, 0x409500, 0x00000000);
 475         nv_wr32(dev, 0x409504, 0x00000025);
 476         if (!nv_wait_ne(dev, 0x409800, 0xffffffff, 0x00000000)) {
 477                 NV_ERROR(dev, "fuc09 req 0x25 timeout\n");
 478                 return -EBUSY;
 479         }
 480
 481         nv_wr32(dev, 0x409800, 0x00000000);
 482         nv_wr32(dev, 0x409500, 0x00000001);
 483         nv_wr32(dev, 0x409504, 0x00000030);
 484         if (!nv_wait_ne(dev, 0x409800, 0xffffffff, 0x00000000)) {
 485                 NV_ERROR(dev, "fuc09 req 0x30 timeout\n");
 486                 return -EBUSY;
 487         }
 488
 489         nv_wr32(dev, 0x409810, 0xb00095c8);
 490         nv_wr32(dev, 0x409800, 0x00000000);
 491         nv_wr32(dev, 0x409500, 0x00000001);
 492         nv_wr32(dev, 0x409504, 0x00000031);
 493         if (!nv_wait_ne(dev, 0x409800, 0xffffffff, 0x00000000)) {
 494                 NV_ERROR(dev, "fuc09 req 0x31 timeout\n");
 495                 return -EBUSY;
 496         }
 497
 498         nv_wr32(dev, 0x409810, 0x00080420);
 499         nv_wr32(dev, 0x409800, 0x00000000);
 500         nv_wr32(dev, 0x409500, 0x00000001);
 501         nv_wr32(dev, 0x409504, 0x00000032);
 502         if (!nv_wait_ne(dev, 0x409800, 0xffffffff, 0x00000000)) {
 503                 NV_ERROR(dev, "fuc09 req 0x32 timeout\n");
 504                 return -EBUSY;
 505         }
 506
 507         nv_wr32(dev, 0x409614, 0x00000070);
 508         nv_wr32(dev, 0x409614, 0x00000770);
 509         nv_wr32(dev, 0x40802c, 0x00000001);
 510         return 0;
 511 }
 512
 513 static int
 514 nve0_graph_init(struct drm_device *dev, int engine)
 515 {
 516         int ret;
 517
 518         nv_mask(dev, 0x000200, 0x18001000, 0x00000000);
 519         nv_mask(dev, 0x000200, 0x18001000, 0x18001000);
 520
 521         nve0_graph_init_obj418880(dev);
 522         nve0_graph_init_regs(dev);
 523         nve0_graph_init_gpc_0(dev);
 524
 525         nv_wr32(dev, 0x400500, 0x00010001);
 526         nv_wr32(dev, 0x400100, 0xffffffff);
 527         nv_wr32(dev, 0x40013c, 0xffffffff);
 528
 529         nve0_graph_init_units(dev);
 530         nve0_graph_init_gpc_1(dev);
 531         nve0_graph_init_rop(dev);
 532
 533         nv_wr32(dev, 0x400108, 0xffffffff);
 534         nv_wr32(dev, 0x400138, 0xffffffff);
 535         nv_wr32(dev, 0x400118, 0xffffffff);
 536         nv_wr32(dev, 0x400130, 0xffffffff);
 537         nv_wr32(dev, 0x40011c, 0xffffffff);
 538         nv_wr32(dev, 0x400134, 0xffffffff);
 539         nv_wr32(dev, 0x400054, 0x34ce3464);
 540
 541         ret = nve0_graph_init_ctxctl(dev);
 542         if (ret)
 543                 return ret;
 544
 545         return 0;
 546 }
 547
 548 int
 549 nve0_graph_isr_chid(struct drm_device *dev, u64 inst)
 550 {
 551         struct drm_nouveau_private *dev_priv = dev->dev_private;
 552         struct nouveau_channel *chan;
 553         unsigned long flags;
 554         int i;
 555
 556         spin_lock_irqsave(&dev_priv->channels.lock, flags);
 557         for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
 558                 chan = dev_priv->channels.ptr[i];
 559                 if (!chan || !chan->ramin)
 560                         continue;
 561
 562                 if (inst == chan->ramin->vinst)
 563                         break;
 564         }
 565         spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
 566         return i;
 567 }
 568
 569 static void
 570 nve0_graph_ctxctl_isr(struct drm_device *dev)
 571 {
 572         u32 ustat = nv_rd32(dev, 0x409c18);
 573
 574         if (ustat & 0x00000001)
 575                 NV_INFO(dev, "PGRAPH: CTXCTRL ucode error\n");
 576         if (ustat & 0x00080000)
 577                 NV_INFO(dev, "PGRAPH: CTXCTRL watchdog timeout\n");
 578         if (ustat & ~0x00080001)
 579                 NV_INFO(dev, "PGRAPH: CTXCTRL 0x%08x\n", ustat);
 580
 581         nve0_graph_ctxctl_debug(dev);
 582         nv_wr32(dev, 0x409c20, ustat);
 583 }
 584
 585 static void
 586 nve0_graph_trap_isr(struct drm_device *dev, int chid)
 587 {
 588         struct nve0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
 589         u32 trap = nv_rd32(dev, 0x400108);
 590         int rop;
 591
 592         if (trap & 0x00000001) {
 593                 u32 stat = nv_rd32(dev, 0x404000);
 594                 NV_INFO(dev, "PGRAPH: DISPATCH ch %d 0x%08x\n", chid, stat);
 595                 nv_wr32(dev, 0x404000, 0xc0000000);
 596                 nv_wr32(dev, 0x400108, 0x00000001);
 597                 trap &= ~0x00000001;
 598         }
 599
 600         if (trap & 0x00000010) {
 601                 u32 stat = nv_rd32(dev, 0x405840);
 602                 NV_INFO(dev, "PGRAPH: SHADER ch %d 0x%08x\n", chid, stat);
 603                 nv_wr32(dev, 0x405840, 0xc0000000);
 604                 nv_wr32(dev, 0x400108, 0x00000010);
 605                 trap &= ~0x00000010;
 606         }
 607
 608         if (trap & 0x02000000) {
 609                 for (rop = 0; rop < priv->rop_nr; rop++) {
 610                         u32 statz = nv_rd32(dev, ROP_UNIT(rop, 0x070));
 611                         u32 statc = nv_rd32(dev, ROP_UNIT(rop, 0x144));
 612                         NV_INFO(dev, "PGRAPH: ROP%d ch %d 0x%08x 0x%08x\n",
 613                                      rop, chid, statz, statc);
 614                         nv_wr32(dev, ROP_UNIT(rop, 0x070), 0xc0000000);
 615                         nv_wr32(dev, ROP_UNIT(rop, 0x144), 0xc0000000);
 616                 }
 617                 nv_wr32(dev, 0x400108, 0x02000000);
 618                 trap &= ~0x02000000;
 619         }
 620
 621         if (trap) {
 622                 NV_INFO(dev, "PGRAPH: TRAP ch %d 0x%08x\n", chid, trap);
 623                 nv_wr32(dev, 0x400108, trap);
 624         }
 625 }
 626
 627 static void
 628 nve0_graph_isr(struct drm_device *dev)
 629 {
 630         u64 inst = (u64)(nv_rd32(dev, 0x409b00) & 0x0fffffff) << 12;
 631         u32 chid = nve0_graph_isr_chid(dev, inst);
 632         u32 stat = nv_rd32(dev, 0x400100);
 633         u32 addr = nv_rd32(dev, 0x400704);
 634         u32 mthd = (addr & 0x00003ffc);
 635         u32 subc = (addr & 0x00070000) >> 16;
 636         u32 data = nv_rd32(dev, 0x400708);
 637         u32 code = nv_rd32(dev, 0x400110);
 638         u32 class = nv_rd32(dev, 0x404200 + (subc * 4));
 639
 640         if (stat & 0x00000010) {
 641                 if (nouveau_gpuobj_mthd_call2(dev, chid, class, mthd, data)) {
 642                         NV_INFO(dev, "PGRAPH: ILLEGAL_MTHD ch %d [0x%010llx] "
 643                                      "subc %d class 0x%04x mthd 0x%04x "
 644                                      "data 0x%08x\n",
 645                                 chid, inst, subc, class, mthd, data);
 646                 }
 647                 nv_wr32(dev, 0x400100, 0x00000010);
 648                 stat &= ~0x00000010;
 649         }
 650
 651         if (stat & 0x00000020) {
 652                 NV_INFO(dev, "PGRAPH: ILLEGAL_CLASS ch %d [0x%010llx] subc %d "
 653                              "class 0x%04x mthd 0x%04x data 0x%08x\n",
 654                         chid, inst, subc, class, mthd, data);
 655                 nv_wr32(dev, 0x400100, 0x00000020);
 656                 stat &= ~0x00000020;
 657         }
 658
 659         if (stat & 0x00100000) {
 660                 NV_INFO(dev, "PGRAPH: DATA_ERROR [");
 661                 nouveau_enum_print(nv50_data_error_names, code);
 662                 printk("] ch %d [0x%010llx] subc %d class 0x%04x "
 663                        "mthd 0x%04x data 0x%08x\n",
 664                        chid, inst, subc, class, mthd, data);
 665                 nv_wr32(dev, 0x400100, 0x00100000);
 666                 stat &= ~0x00100000;
 667         }
 668
 669         if (stat & 0x00200000) {
 670                 nve0_graph_trap_isr(dev, chid);
 671                 nv_wr32(dev, 0x400100, 0x00200000);
 672                 stat &= ~0x00200000;
 673         }
 674
 675         if (stat & 0x00080000) {
 676                 nve0_graph_ctxctl_isr(dev);
 677                 nv_wr32(dev, 0x400100, 0x00080000);
 678                 stat &= ~0x00080000;
 679         }
 680
 681         if (stat) {
 682                 NV_INFO(dev, "PGRAPH: unknown stat 0x%08x\n", stat);
 683                 nv_wr32(dev, 0x400100, stat);
 684         }
 685
 686         nv_wr32(dev, 0x400500, 0x00010001);
 687 }
 688
 689 static int
 690 nve0_graph_create_fw(struct drm_device *dev, const char *fwname,
 691                      struct nve0_graph_fuc *fuc)
 692 {
 693         struct drm_nouveau_private *dev_priv = dev->dev_private;
 694         const struct firmware *fw;
 695         char f[32];
 696         int ret;
 697
 698         snprintf(f, sizeof(f), "nouveau/nv%02x_%s", dev_priv->chipset, fwname);
 699         ret = request_firmware(&fw, f, &dev->pdev->dev);
 700         if (ret)
 701                 return ret;
 702
 703         fuc->size = fw->size;
 704         fuc->data = kmemdup(fw->data, fuc->size, GFP_KERNEL);
 705         release_firmware(fw);
 706         return (fuc->data != NULL) ? 0 : -ENOMEM;
 707 }
 708
 709 static void
 710 nve0_graph_destroy_fw(struct nve0_graph_fuc *fuc)
 711 {
 712         if (fuc->data) {
 713                 kfree(fuc->data);
 714                 fuc->data = NULL;
 715         }
 716 }
 717
 718 static void
 719 nve0_graph_destroy(struct drm_device *dev, int engine)
 720 {
 721         struct nve0_graph_priv *priv = nv_engine(dev, engine);
 722
 723         nve0_graph_destroy_fw(&priv->fuc409c);
 724         nve0_graph_destroy_fw(&priv->fuc409d);
 725         nve0_graph_destroy_fw(&priv->fuc41ac);
 726         nve0_graph_destroy_fw(&priv->fuc41ad);
 727
 728         nouveau_irq_unregister(dev, 12);
 729
 730         nouveau_gpuobj_ref(NULL, &priv->unk4188b8);
 731         nouveau_gpuobj_ref(NULL, &priv->unk4188b4);
 732
 733         if (priv->grctx_vals)
 734                 kfree(priv->grctx_vals);
 735
 736         NVOBJ_ENGINE_DEL(dev, GR);
 737         kfree(priv);
 738 }
 739
 740 int
 741 nve0_graph_create(struct drm_device *dev)
 742 {
 743         struct drm_nouveau_private *dev_priv = dev->dev_private;
 744         struct nve0_graph_priv *priv;
 745         int ret, gpc, i;
 746         u32 kepler;
 747
 748         kepler = nve0_graph_class(dev);
 749         if (!kepler) {
 750                 NV_ERROR(dev, "PGRAPH: unsupported chipset, please report!\n");
 751                 return 0;
 752         }
 753
 754         priv = kzalloc(sizeof(*priv), GFP_KERNEL);
 755         if (!priv)
 756                 return -ENOMEM;
 757
 758         priv->base.destroy = nve0_graph_destroy;
 759         priv->base.init = nve0_graph_init;
 760         priv->base.fini = nve0_graph_fini;
 761         priv->base.context_new = nve0_graph_context_new;
 762         priv->base.context_del = nve0_graph_context_del;
 763         priv->base.object_new = nve0_graph_object_new;
 764
 765         NVOBJ_ENGINE_ADD(dev, GR, &priv->base);
 766         nouveau_irq_register(dev, 12, nve0_graph_isr);
 767
 768         NV_INFO(dev, "PGRAPH: using external firmware\n");
 769         if (nve0_graph_create_fw(dev, "fuc409c", &priv->fuc409c) ||
 770             nve0_graph_create_fw(dev, "fuc409d", &priv->fuc409d) ||
 771             nve0_graph_create_fw(dev, "fuc41ac", &priv->fuc41ac) ||
 772             nve0_graph_create_fw(dev, "fuc41ad", &priv->fuc41ad)) {
 773                 ret = 0;
 774                 goto error;
 775         }
 776
 777         ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 256, 0, &priv->unk4188b4);
 778         if (ret)
 779                 goto error;
 780
 781         ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 256, 0, &priv->unk4188b8);
 782         if (ret)
 783                 goto error;
 784
 785         for (i = 0; i < 0x1000; i += 4) {
 786                 nv_wo32(priv->unk4188b4, i, 0x00000010);
 787                 nv_wo32(priv->unk4188b8, i, 0x00000010);
 788         }
 789
 790         priv->gpc_nr  =  nv_rd32(dev, 0x409604) & 0x0000001f;
 791         priv->rop_nr = (nv_rd32(dev, 0x409604) & 0x001f0000) >> 16;
 792         for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
 793                 priv->tpc_nr[gpc] = nv_rd32(dev, GPC_UNIT(gpc, 0x2608));
 794                 priv->tpc_total += priv->tpc_nr[gpc];
 795         }
 796
 797         switch (dev_priv->chipset) {
 798         case 0xe4:
 799                 if (priv->tpc_total == 8)
 800                         priv->magic_not_rop_nr = 3;
 801                 else
 802                 if (priv->tpc_total == 7)
 803                         priv->magic_not_rop_nr = 1;
 804                 break;
 805         case 0xe7:
 806                 priv->magic_not_rop_nr = 1;
 807                 break;
 808         default:
 809                 break;
 810         }
 811
 812         if (!priv->magic_not_rop_nr) {
 813                 NV_ERROR(dev, "PGRAPH: unknown config: %d/%d/%d/%d, %d\n",
 814                          priv->tpc_nr[0], priv->tpc_nr[1], priv->tpc_nr[2],
 815                          priv->tpc_nr[3], priv->rop_nr);
 816                 priv->magic_not_rop_nr = 0x00;
 817         }
 818
 819         NVOBJ_CLASS(dev, 0xa097, GR); /* subc 0: 3D */
 820         NVOBJ_CLASS(dev, 0xa0c0, GR); /* subc 1: COMPUTE */
 821         NVOBJ_CLASS(dev, 0xa040, GR); /* subc 2: P2MF */
 822         NVOBJ_CLASS(dev, 0x902d, GR); /* subc 3: 2D */
 823         //NVOBJ_CLASS(dev, 0xa0b5, GR); /* subc 4: COPY */
 824         return 0;
 825
 826 error:
 827         nve0_graph_destroy(dev, NVOBJ_ENGINE_GR);
 828         return ret;
 829 }