drivers/gpu/drm/nouveau/nouveau_irq.c

   1 /*
   2  * Copyright (C) 2006 Ben Skeggs.
   3  *
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining
   7  * a copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sublicense, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial
  16  * portions of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  */
  27
  28 /*
  29  * Authors:
  30  *   Ben Skeggs <darktama@iinet.net.au>
  31  */
  32
  33 #include "drmP.h"
  34 #include "drm.h"
  35 #include "nouveau_drm.h"
  36 #include "nouveau_drv.h"
  37 #include "nouveau_reg.h"
  38 #include "nouveau_ramht.h"
  39 #include "nouveau_util.h"
  40
  41 void
  42 nouveau_irq_preinstall(struct drm_device *dev)
  43 {
  44         struct drm_nouveau_private *dev_priv = dev->dev_private;
  45
  46         /* Master disable */
  47         nv_wr32(dev, NV03_PMC_INTR_EN_0, 0);
  48
  49         INIT_LIST_HEAD(&dev_priv->vbl_waiting);
  50 }
  51
  52 int
  53 nouveau_irq_postinstall(struct drm_device *dev)
  54 {
  55         struct drm_nouveau_private *dev_priv = dev->dev_private;
  56
  57         /* Master enable */
  58         nv_wr32(dev, NV03_PMC_INTR_EN_0, NV_PMC_INTR_EN_0_MASTER_ENABLE);
  59         if (dev_priv->msi_enabled)
  60                 nv_wr08(dev, 0x00088068, 0xff);
  61
  62         return 0;
  63 }
  64
  65 void
  66 nouveau_irq_uninstall(struct drm_device *dev)
  67 {
  68         /* Master disable */
  69         nv_wr32(dev, NV03_PMC_INTR_EN_0, 0);
  70 }
  71
  72 static bool
  73 nouveau_fifo_swmthd(struct drm_device *dev, u32 chid, u32 addr, u32 data)
  74 {
  75         struct drm_nouveau_private *dev_priv = dev->dev_private;
  76         struct nouveau_channel *chan = NULL;
  77         struct nouveau_gpuobj *obj;
  78         unsigned long flags;
  79         const int subc = (addr >> 13) & 0x7;
  80         const int mthd = addr & 0x1ffc;
  81         bool handled = false;
  82         u32 engine;
  83
  84         spin_lock_irqsave(&dev_priv->channels.lock, flags);
  85         if (likely(chid >= 0 && chid < dev_priv->engine.fifo.channels))
  86                 chan = dev_priv->channels.ptr[chid];
  87         if (unlikely(!chan))
  88                 goto out;
  89
  90         switch (mthd) {
  91         case 0x0000: /* bind object to subchannel */
  92                 obj = nouveau_ramht_find(chan, data);
  93                 if (unlikely(!obj || obj->engine != NVOBJ_ENGINE_SW))
  94                         break;
  95
  96                 chan->sw_subchannel[subc] = obj->class;
  97                 engine = 0x0000000f << (subc * 4);
  98
  99                 nv_mask(dev, NV04_PFIFO_CACHE1_ENGINE, engine, 0x00000000);
 100                 handled = true;
 101                 break;
 102         default:
 103                 engine = nv_rd32(dev, NV04_PFIFO_CACHE1_ENGINE);
 104                 if (unlikely(((engine >> (subc * 4)) & 0xf) != 0))
 105                         break;
 106
 107                 if (!nouveau_gpuobj_mthd_call(chan, chan->sw_subchannel[subc],
 108                                               mthd, data))
 109                         handled = true;
 110                 break;
 111         }
 112
 113 out:
 114         spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
 115         return handled;
 116 }
 117
 118 static void
 119 nouveau_fifo_irq_handler(struct drm_device *dev)
 120 {
 121         struct drm_nouveau_private *dev_priv = dev->dev_private;
 122         struct nouveau_engine *engine = &dev_priv->engine;
 123         uint32_t status, reassign;
 124         int cnt = 0;
 125
 126         reassign = nv_rd32(dev, NV03_PFIFO_CACHES) & 1;
 127         while ((status = nv_rd32(dev, NV03_PFIFO_INTR_0)) && (cnt++ < 100)) {
 128                 uint32_t chid, get;
 129
 130                 nv_wr32(dev, NV03_PFIFO_CACHES, 0);
 131
 132                 chid = engine->fifo.channel_id(dev);
 133                 get  = nv_rd32(dev, NV03_PFIFO_CACHE1_GET);
 134
 135                 if (status & NV_PFIFO_INTR_CACHE_ERROR) {
 136                         uint32_t mthd, data;
 137                         int ptr;
 138
 139                         /* NV_PFIFO_CACHE1_GET actually goes to 0xffc before
 140                          * wrapping on my G80 chips, but CACHE1 isn't big
 141                          * enough for this much data.. Tests show that it
 142                          * wraps around to the start at GET=0x800.. No clue
 143                          * as to why..
 144                          */
 145                         ptr = (get & 0x7ff) >> 2;
 146
 147                         if (dev_priv->card_type < NV_40) {
 148                                 mthd = nv_rd32(dev,
 149                                         NV04_PFIFO_CACHE1_METHOD(ptr));
 150                                 data = nv_rd32(dev,
 151                                         NV04_PFIFO_CACHE1_DATA(ptr));
 152                         } else {
 153                                 mthd = nv_rd32(dev,
 154                                         NV40_PFIFO_CACHE1_METHOD(ptr));
 155                                 data = nv_rd32(dev,
 156                                         NV40_PFIFO_CACHE1_DATA(ptr));
 157                         }
 158
 159                         if (!nouveau_fifo_swmthd(dev, chid, mthd, data)) {
 160                                 NV_INFO(dev, "PFIFO_CACHE_ERROR - Ch %d/%d "
 161                                              "Mthd 0x%04x Data 0x%08x\n",
 162                                         chid, (mthd >> 13) & 7, mthd & 0x1ffc,
 163                                         data);
 164                         }
 165
 166                         nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUSH, 0);
 167                         nv_wr32(dev, NV03_PFIFO_INTR_0,
 168                                                 NV_PFIFO_INTR_CACHE_ERROR);
 169
 170                         nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0,
 171                                 nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH0) & ~1);
 172                         nv_wr32(dev, NV03_PFIFO_CACHE1_GET, get + 4);
 173                         nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0,
 174                                 nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH0) | 1);
 175                         nv_wr32(dev, NV04_PFIFO_CACHE1_HASH, 0);
 176
 177                         nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUSH,
 178                                 nv_rd32(dev, NV04_PFIFO_CACHE1_DMA_PUSH) | 1);
 179                         nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
 180
 181                         status &= ~NV_PFIFO_INTR_CACHE_ERROR;
 182                 }
 183
 184                 if (status & NV_PFIFO_INTR_DMA_PUSHER) {
 185                         u32 dma_get = nv_rd32(dev, 0x003244);
 186                         u32 dma_put = nv_rd32(dev, 0x003240);
 187                         u32 push = nv_rd32(dev, 0x003220);
 188                         u32 state = nv_rd32(dev, 0x003228);
 189
 190                         if (dev_priv->card_type == NV_50) {
 191                                 u32 ho_get = nv_rd32(dev, 0x003328);
 192                                 u32 ho_put = nv_rd32(dev, 0x003320);
 193                                 u32 ib_get = nv_rd32(dev, 0x003334);
 194                                 u32 ib_put = nv_rd32(dev, 0x003330);
 195
 196                                 if (nouveau_ratelimit())
 197                                         NV_INFO(dev, "PFIFO_DMA_PUSHER - Ch %d Get 0x%02x%08x "
 198                                              "Put 0x%02x%08x IbGet 0x%08x IbPut 0x%08x "
 199                                              "State 0x%08x Push 0x%08x\n",
 200                                                 chid, ho_get, dma_get, ho_put,
 201                                                 dma_put, ib_get, ib_put, state,
 202                                                 push);
 203
 204                                 /* METHOD_COUNT, in DMA_STATE on earlier chipsets */
 205                                 nv_wr32(dev, 0x003364, 0x00000000);
 206                                 if (dma_get != dma_put || ho_get != ho_put) {
 207                                         nv_wr32(dev, 0x003244, dma_put);
 208                                         nv_wr32(dev, 0x003328, ho_put);
 209                                 } else
 210                                 if (ib_get != ib_put) {
 211                                         nv_wr32(dev, 0x003334, ib_put);
 212                                 }
 213                         } else {
 214                                 NV_INFO(dev, "PFIFO_DMA_PUSHER - Ch %d Get 0x%08x "
 215                                              "Put 0x%08x State 0x%08x Push 0x%08x\n",
 216                                         chid, dma_get, dma_put, state, push);
 217
 218                                 if (dma_get != dma_put)
 219                                         nv_wr32(dev, 0x003244, dma_put);
 220                         }
 221
 222                         nv_wr32(dev, 0x003228, 0x00000000);
 223                         nv_wr32(dev, 0x003220, 0x00000001);
 224                         nv_wr32(dev, 0x002100, NV_PFIFO_INTR_DMA_PUSHER);
 225                         status &= ~NV_PFIFO_INTR_DMA_PUSHER;
 226                 }
 227
 228                 if (status & NV_PFIFO_INTR_SEMAPHORE) {
 229                         uint32_t sem;
 230
 231                         status &= ~NV_PFIFO_INTR_SEMAPHORE;
 232                         nv_wr32(dev, NV03_PFIFO_INTR_0,
 233                                 NV_PFIFO_INTR_SEMAPHORE);
 234
 235                         sem = nv_rd32(dev, NV10_PFIFO_CACHE1_SEMAPHORE);
 236                         nv_wr32(dev, NV10_PFIFO_CACHE1_SEMAPHORE, sem | 0x1);
 237
 238                         nv_wr32(dev, NV03_PFIFO_CACHE1_GET, get + 4);
 239                         nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
 240                 }
 241
 242                 if (dev_priv->card_type == NV_50) {
 243                         if (status & 0x00000010) {
 244                                 nv50_fb_vm_trap(dev, 1, "PFIFO_BAR_FAULT");
 245                                 status &= ~0x00000010;
 246                                 nv_wr32(dev, 0x002100, 0x00000010);
 247                         }
 248                 }
 249
 250                 if (status) {
 251                         if (nouveau_ratelimit())
 252                                 NV_INFO(dev, "PFIFO_INTR 0x%08x - Ch %d\n",
 253                                         status, chid);
 254                         nv_wr32(dev, NV03_PFIFO_INTR_0, status);
 255                         status = 0;
 256                 }
 257
 258                 nv_wr32(dev, NV03_PFIFO_CACHES, reassign);
 259         }
 260
 261         if (status) {
 262                 NV_INFO(dev, "PFIFO still angry after %d spins, halt\n", cnt);
 263                 nv_wr32(dev, 0x2140, 0);
 264                 nv_wr32(dev, 0x140, 0);
 265         }
 266
 267         nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PFIFO_PENDING);
 268 }
 269
 270 static struct nouveau_bitfield nstatus_names[] =
 271 {
 272         { NV04_PGRAPH_NSTATUS_STATE_IN_USE,       "STATE_IN_USE" },
 273         { NV04_PGRAPH_NSTATUS_INVALID_STATE,      "INVALID_STATE" },
 274         { NV04_PGRAPH_NSTATUS_BAD_ARGUMENT,       "BAD_ARGUMENT" },
 275         { NV04_PGRAPH_NSTATUS_PROTECTION_FAULT,   "PROTECTION_FAULT" },
 276         {}
 277 };
 278
 279 static struct nouveau_bitfield nstatus_names_nv10[] =
 280 {
 281         { NV10_PGRAPH_NSTATUS_STATE_IN_USE,       "STATE_IN_USE" },
 282         { NV10_PGRAPH_NSTATUS_INVALID_STATE,      "INVALID_STATE" },
 283         { NV10_PGRAPH_NSTATUS_BAD_ARGUMENT,       "BAD_ARGUMENT" },
 284         { NV10_PGRAPH_NSTATUS_PROTECTION_FAULT,   "PROTECTION_FAULT" },
 285         {}
 286 };
 287
 288 static struct nouveau_bitfield nsource_names[] =
 289 {
 290         { NV03_PGRAPH_NSOURCE_NOTIFICATION,       "NOTIFICATION" },
 291         { NV03_PGRAPH_NSOURCE_DATA_ERROR,         "DATA_ERROR" },
 292         { NV03_PGRAPH_NSOURCE_PROTECTION_ERROR,   "PROTECTION_ERROR" },
 293         { NV03_PGRAPH_NSOURCE_RANGE_EXCEPTION,    "RANGE_EXCEPTION" },
 294         { NV03_PGRAPH_NSOURCE_LIMIT_COLOR,        "LIMIT_COLOR" },
 295         { NV03_PGRAPH_NSOURCE_LIMIT_ZETA,         "LIMIT_ZETA" },
 296         { NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD,       "ILLEGAL_MTHD" },
 297         { NV03_PGRAPH_NSOURCE_DMA_R_PROTECTION,   "DMA_R_PROTECTION" },
 298         { NV03_PGRAPH_NSOURCE_DMA_W_PROTECTION,   "DMA_W_PROTECTION" },
 299         { NV03_PGRAPH_NSOURCE_FORMAT_EXCEPTION,   "FORMAT_EXCEPTION" },
 300         { NV03_PGRAPH_NSOURCE_PATCH_EXCEPTION,    "PATCH_EXCEPTION" },
 301         { NV03_PGRAPH_NSOURCE_STATE_INVALID,      "STATE_INVALID" },
 302         { NV03_PGRAPH_NSOURCE_DOUBLE_NOTIFY,      "DOUBLE_NOTIFY" },
 303         { NV03_PGRAPH_NSOURCE_NOTIFY_IN_USE,      "NOTIFY_IN_USE" },
 304         { NV03_PGRAPH_NSOURCE_METHOD_CNT,         "METHOD_CNT" },
 305         { NV03_PGRAPH_NSOURCE_BFR_NOTIFICATION,   "BFR_NOTIFICATION" },
 306         { NV03_PGRAPH_NSOURCE_DMA_VTX_PROTECTION, "DMA_VTX_PROTECTION" },
 307         { NV03_PGRAPH_NSOURCE_DMA_WIDTH_A,        "DMA_WIDTH_A" },
 308         { NV03_PGRAPH_NSOURCE_DMA_WIDTH_B,        "DMA_WIDTH_B" },
 309         {}
 310 };
 311
 312 static int
 313 nouveau_graph_chid_from_grctx(struct drm_device *dev)
 314 {
 315         struct drm_nouveau_private *dev_priv = dev->dev_private;
 316         struct nouveau_channel *chan;
 317         unsigned long flags;
 318         uint32_t inst;
 319         int i;
 320
 321         if (dev_priv->card_type < NV_40)
 322                 return dev_priv->engine.fifo.channels;
 323         else
 324         if (dev_priv->card_type < NV_50) {
 325                 inst = (nv_rd32(dev, 0x40032c) & 0xfffff) << 4;
 326
 327                 spin_lock_irqsave(&dev_priv->channels.lock, flags);
 328                 for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
 329                         chan = dev_priv->channels.ptr[i];
 330                         if (!chan || !chan->ramin_grctx)
 331                                 continue;
 332
 333                         if (inst == chan->ramin_grctx->pinst)
 334                                 break;
 335                 }
 336                 spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
 337         } else {
 338                 inst = (nv_rd32(dev, 0x40032c) & 0xfffff) << 12;
 339
 340                 spin_lock_irqsave(&dev_priv->channels.lock, flags);
 341                 for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
 342                         chan = dev_priv->channels.ptr[i];
 343                         if (!chan || !chan->ramin)
 344                                 continue;
 345
 346                         if (inst == chan->ramin->vinst)
 347                                 break;
 348                 }
 349                 spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
 350         }
 351
 352
 353         return i;
 354 }
 355
 356 static int
 357 nouveau_graph_trapped_channel(struct drm_device *dev, int *channel_ret)
 358 {
 359         struct drm_nouveau_private *dev_priv = dev->dev_private;
 360         struct nouveau_engine *engine = &dev_priv->engine;
 361         int channel;
 362
 363         if (dev_priv->card_type < NV_10)
 364                 channel = (nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR) >> 24) & 0xf;
 365         else
 366         if (dev_priv->card_type < NV_40)
 367                 channel = (nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR) >> 20) & 0x1f;
 368         else
 369                 channel = nouveau_graph_chid_from_grctx(dev);
 370
 371         if (channel >= engine->fifo.channels ||
 372             !dev_priv->channels.ptr[channel]) {
 373                 NV_ERROR(dev, "AIII, invalid/inactive channel id %d\n", channel);
 374                 return -EINVAL;
 375         }
 376
 377         *channel_ret = channel;
 378         return 0;
 379 }
 380
 381 struct nouveau_pgraph_trap {
 382         int channel;
 383         int class;
 384         int subc, mthd, size;
 385         uint32_t data, data2;
 386         uint32_t nsource, nstatus;
 387 };
 388
 389 static void
 390 nouveau_graph_trap_info(struct drm_device *dev,
 391                         struct nouveau_pgraph_trap *trap)
 392 {
 393         struct drm_nouveau_private *dev_priv = dev->dev_private;
 394         uint32_t address;
 395
 396         trap->nsource = trap->nstatus = 0;
 397         if (dev_priv->card_type < NV_50) {
 398                 trap->nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
 399                 trap->nstatus = nv_rd32(dev, NV03_PGRAPH_NSTATUS);
 400         }
 401
 402         if (nouveau_graph_trapped_channel(dev, &trap->channel))
 403                 trap->channel = -1;
 404         address = nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR);
 405
 406         trap->mthd = address & 0x1FFC;
 407         trap->data = nv_rd32(dev, NV04_PGRAPH_TRAPPED_DATA);
 408         if (dev_priv->card_type < NV_10) {
 409                 trap->subc  = (address >> 13) & 0x7;
 410         } else {
 411                 trap->subc  = (address >> 16) & 0x7;
 412                 trap->data2 = nv_rd32(dev, NV10_PGRAPH_TRAPPED_DATA_HIGH);
 413         }
 414
 415         if (dev_priv->card_type < NV_10)
 416                 trap->class = nv_rd32(dev, 0x400180 + trap->subc*4) & 0xFF;
 417         else if (dev_priv->card_type < NV_40)
 418                 trap->class = nv_rd32(dev, 0x400160 + trap->subc*4) & 0xFFF;
 419         else if (dev_priv->card_type < NV_50)
 420                 trap->class = nv_rd32(dev, 0x400160 + trap->subc*4) & 0xFFFF;
 421         else
 422                 trap->class = nv_rd32(dev, 0x400814);
 423 }
 424
 425 static void
 426 nouveau_graph_dump_trap_info(struct drm_device *dev, const char *id,
 427                              struct nouveau_pgraph_trap *trap)
 428 {
 429         struct drm_nouveau_private *dev_priv = dev->dev_private;
 430         uint32_t nsource = trap->nsource, nstatus = trap->nstatus;
 431
 432         if (dev_priv->card_type < NV_50) {
 433                 NV_INFO(dev, "%s - nSource:", id);
 434                 nouveau_bitfield_print(nsource_names, nsource);
 435                 printk(", nStatus:");
 436                 if (dev_priv->card_type < NV_10)
 437                         nouveau_bitfield_print(nstatus_names, nstatus);
 438                 else
 439                         nouveau_bitfield_print(nstatus_names_nv10, nstatus);
 440                 printk("\n");
 441         }
 442
 443         NV_INFO(dev, "%s - Ch %d/%d Class 0x%04x Mthd 0x%04x "
 444                                         "Data 0x%08x:0x%08x\n",
 445                                         id, trap->channel, trap->subc,
 446                                         trap->class, trap->mthd,
 447                                         trap->data2, trap->data);
 448 }
 449
 450 static int
 451 nouveau_pgraph_intr_swmthd(struct drm_device *dev,
 452                            struct nouveau_pgraph_trap *trap)
 453 {
 454         struct drm_nouveau_private *dev_priv = dev->dev_private;
 455         struct nouveau_channel *chan;
 456         unsigned long flags;
 457         int ret = -EINVAL;
 458
 459         spin_lock_irqsave(&dev_priv->channels.lock, flags);
 460         if (trap->channel > 0 &&
 461             trap->channel < dev_priv->engine.fifo.channels &&
 462             dev_priv->channels.ptr[trap->channel]) {
 463                 chan = dev_priv->channels.ptr[trap->channel];
 464                 ret = nouveau_gpuobj_mthd_call(chan, trap->class, trap->mthd, trap->data);
 465         }
 466         spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
 467
 468         return ret;
 469 }
 470
 471 static inline void
 472 nouveau_pgraph_intr_notify(struct drm_device *dev, uint32_t nsource)
 473 {
 474         struct nouveau_pgraph_trap trap;
 475         int unhandled = 0;
 476
 477         nouveau_graph_trap_info(dev, &trap);
 478
 479         if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
 480                 if (nouveau_pgraph_intr_swmthd(dev, &trap))
 481                         unhandled = 1;
 482         } else {
 483                 unhandled = 1;
 484         }
 485
 486         if (unhandled)
 487                 nouveau_graph_dump_trap_info(dev, "PGRAPH_NOTIFY", &trap);
 488 }
 489
 490
 491 static inline void
 492 nouveau_pgraph_intr_error(struct drm_device *dev, uint32_t nsource)
 493 {
 494         struct nouveau_pgraph_trap trap;
 495         int unhandled = 0;
 496
 497         nouveau_graph_trap_info(dev, &trap);
 498         trap.nsource = nsource;
 499
 500         if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
 501                 if (nouveau_pgraph_intr_swmthd(dev, &trap))
 502                         unhandled = 1;
 503         } else if (nsource & NV03_PGRAPH_NSOURCE_DMA_VTX_PROTECTION) {
 504                 uint32_t v = nv_rd32(dev, 0x402000);
 505                 nv_wr32(dev, 0x402000, v);
 506
 507                 /* dump the error anyway for now: it's useful for
 508                    Gallium development */
 509                 unhandled = 1;
 510         } else {
 511                 unhandled = 1;
 512         }
 513
 514         if (unhandled && nouveau_ratelimit())
 515                 nouveau_graph_dump_trap_info(dev, "PGRAPH_ERROR", &trap);
 516 }
 517
 518 static inline void
 519 nouveau_pgraph_intr_context_switch(struct drm_device *dev)
 520 {
 521         struct drm_nouveau_private *dev_priv = dev->dev_private;
 522         struct nouveau_engine *engine = &dev_priv->engine;
 523         uint32_t chid;
 524
 525         chid = engine->fifo.channel_id(dev);
 526         NV_DEBUG(dev, "PGRAPH context switch interrupt channel %x\n", chid);
 527
 528         switch (dev_priv->card_type) {
 529         case NV_04:
 530                 nv04_graph_context_switch(dev);
 531                 break;
 532         case NV_10:
 533                 nv10_graph_context_switch(dev);
 534                 break;
 535         default:
 536                 NV_ERROR(dev, "Context switch not implemented\n");
 537                 break;
 538         }
 539 }
 540
 541 static void
 542 nouveau_pgraph_irq_handler(struct drm_device *dev)
 543 {
 544         uint32_t status;
 545
 546         while ((status = nv_rd32(dev, NV03_PGRAPH_INTR))) {
 547                 uint32_t nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
 548
 549                 if (status & NV_PGRAPH_INTR_NOTIFY) {
 550                         nouveau_pgraph_intr_notify(dev, nsource);
 551
 552                         status &= ~NV_PGRAPH_INTR_NOTIFY;
 553                         nv_wr32(dev, NV03_PGRAPH_INTR, NV_PGRAPH_INTR_NOTIFY);
 554                 }
 555
 556                 if (status & NV_PGRAPH_INTR_ERROR) {
 557                         nouveau_pgraph_intr_error(dev, nsource);
 558
 559                         status &= ~NV_PGRAPH_INTR_ERROR;
 560                         nv_wr32(dev, NV03_PGRAPH_INTR, NV_PGRAPH_INTR_ERROR);
 561                 }
 562
 563                 if (status & NV_PGRAPH_INTR_CONTEXT_SWITCH) {
 564                         status &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
 565                         nv_wr32(dev, NV03_PGRAPH_INTR,
 566                                  NV_PGRAPH_INTR_CONTEXT_SWITCH);
 567
 568                         nouveau_pgraph_intr_context_switch(dev);
 569                 }
 570
 571                 if (status) {
 572                         NV_INFO(dev, "Unhandled PGRAPH_INTR - 0x%08x\n", status);
 573                         nv_wr32(dev, NV03_PGRAPH_INTR, status);
 574                 }
 575
 576                 if ((nv_rd32(dev, NV04_PGRAPH_FIFO) & (1 << 0)) == 0)
 577                         nv_wr32(dev, NV04_PGRAPH_FIFO, 1);
 578         }
 579
 580         nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PGRAPH_PENDING);
 581 }
 582
 583 static struct nouveau_enum nv50_mp_exec_error_names[] =
 584 {
 585         { 3, "STACK_UNDERFLOW" },
 586         { 4, "QUADON_ACTIVE" },
 587         { 8, "TIMEOUT" },
 588         { 0x10, "INVALID_OPCODE" },
 589         { 0x40, "BREAKPOINT" },
 590         {}
 591 };
 592
 593 static void
 594 nv50_pgraph_mp_trap(struct drm_device *dev, int tpid, int display)
 595 {
 596         struct drm_nouveau_private *dev_priv = dev->dev_private;
 597         uint32_t units = nv_rd32(dev, 0x1540);
 598         uint32_t addr, mp10, status, pc, oplow, ophigh;
 599         int i;
 600         int mps = 0;
 601         for (i = 0; i < 4; i++) {
 602                 if (!(units & 1 << (i+24)))
 603                         continue;
 604                 if (dev_priv->chipset < 0xa0)
 605                         addr = 0x408200 + (tpid << 12) + (i << 7);
 606                 else
 607                         addr = 0x408100 + (tpid << 11) + (i << 7);
 608                 mp10 = nv_rd32(dev, addr + 0x10);
 609                 status = nv_rd32(dev, addr + 0x14);
 610                 if (!status)
 611                         continue;
 612                 if (display) {
 613                         nv_rd32(dev, addr + 0x20);
 614                         pc = nv_rd32(dev, addr + 0x24);
 615                         oplow = nv_rd32(dev, addr + 0x70);
 616                         ophigh= nv_rd32(dev, addr + 0x74);
 617                         NV_INFO(dev, "PGRAPH_TRAP_MP_EXEC - "
 618                                         "TP %d MP %d: ", tpid, i);
 619                         nouveau_enum_print(nv50_mp_exec_error_names, status);
 620                         printk(" at %06x warp %d, opcode %08x %08x\n",
 621                                         pc&0xffffff, pc >> 24,
 622                                         oplow, ophigh);
 623                 }
 624                 nv_wr32(dev, addr + 0x10, mp10);
 625                 nv_wr32(dev, addr + 0x14, 0);
 626                 mps++;
 627         }
 628         if (!mps && display)
 629                 NV_INFO(dev, "PGRAPH_TRAP_MP_EXEC - TP %d: "
 630                                 "No MPs claiming errors?\n", tpid);
 631 }
 632
 633 static void
 634 nv50_pgraph_tp_trap(struct drm_device *dev, int type, uint32_t ustatus_old,
 635                 uint32_t ustatus_new, int display, const char *name)
 636 {
 637         struct drm_nouveau_private *dev_priv = dev->dev_private;
 638         int tps = 0;
 639         uint32_t units = nv_rd32(dev, 0x1540);
 640         int i, r;
 641         uint32_t ustatus_addr, ustatus;
 642         for (i = 0; i < 16; i++) {
 643                 if (!(units & (1 << i)))
 644                         continue;
 645                 if (dev_priv->chipset < 0xa0)
 646                         ustatus_addr = ustatus_old + (i << 12);
 647                 else
 648                         ustatus_addr = ustatus_new + (i << 11);
 649                 ustatus = nv_rd32(dev, ustatus_addr) & 0x7fffffff;
 650                 if (!ustatus)
 651                         continue;
 652                 tps++;
 653                 switch (type) {
 654                 case 6: /* texture error... unknown for now */
 655                         nv50_fb_vm_trap(dev, display, name);
 656                         if (display) {
 657                                 NV_ERROR(dev, "magic set %d:\n", i);
 658                                 for (r = ustatus_addr + 4; r <= ustatus_addr + 0x10; r += 4)
 659                                         NV_ERROR(dev, "\t0x%08x: 0x%08x\n", r,
 660                                                 nv_rd32(dev, r));
 661                         }
 662                         break;
 663                 case 7: /* MP error */
 664                         if (ustatus & 0x00010000) {
 665                                 nv50_pgraph_mp_trap(dev, i, display);
 666                                 ustatus &= ~0x00010000;
 667                         }
 668                         break;
 669                 case 8: /* TPDMA error */
 670                         {
 671                         uint32_t e0c = nv_rd32(dev, ustatus_addr + 4);
 672                         uint32_t e10 = nv_rd32(dev, ustatus_addr + 8);
 673                         uint32_t e14 = nv_rd32(dev, ustatus_addr + 0xc);
 674                         uint32_t e18 = nv_rd32(dev, ustatus_addr + 0x10);
 675                         uint32_t e1c = nv_rd32(dev, ustatus_addr + 0x14);
 676                         uint32_t e20 = nv_rd32(dev, ustatus_addr + 0x18);
 677                         uint32_t e24 = nv_rd32(dev, ustatus_addr + 0x1c);
 678                         nv50_fb_vm_trap(dev, display, name);
 679                         /* 2d engine destination */
 680                         if (ustatus & 0x00000010) {
 681                                 if (display) {
 682                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA_2D - TP %d - Unknown fault at address %02x%08x\n",
 683                                                         i, e14, e10);
 684                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA_2D - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
 685                                                         i, e0c, e18, e1c, e20, e24);
 686                                 }
 687                                 ustatus &= ~0x00000010;
 688                         }
 689                         /* Render target */
 690                         if (ustatus & 0x00000040) {
 691                                 if (display) {
 692                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA_RT - TP %d - Unknown fault at address %02x%08x\n",
 693                                                         i, e14, e10);
 694                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA_RT - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
 695                                                         i, e0c, e18, e1c, e20, e24);
 696                                 }
 697                                 ustatus &= ~0x00000040;
 698                         }
 699                         /* CUDA memory: l[], g[] or stack. */
 700                         if (ustatus & 0x00000080) {
 701                                 if (display) {
 702                                         if (e18 & 0x80000000) {
 703                                                 /* g[] read fault? */
 704                                                 NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Global read fault at address %02x%08x\n",
 705                                                                 i, e14, e10 | ((e18 >> 24) & 0x1f));
 706                                                 e18 &= ~0x1f000000;
 707                                         } else if (e18 & 0xc) {
 708                                                 /* g[] write fault? */
 709                                                 NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Global write fault at address %02x%08x\n",
 710                                                                 i, e14, e10 | ((e18 >> 7) & 0x1f));
 711                                                 e18 &= ~0x00000f80;
 712                                         } else {
 713                                                 NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Unknown CUDA fault at address %02x%08x\n",
 714                                                                 i, e14, e10);
 715                                         }
 716                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
 717                                                         i, e0c, e18, e1c, e20, e24);
 718                                 }
 719                                 ustatus &= ~0x00000080;
 720                         }
 721                         }
 722                         break;
 723                 }
 724                 if (ustatus) {
 725                         if (display)
 726                                 NV_INFO(dev, "%s - TP%d: Unhandled ustatus 0x%08x\n", name, i, ustatus);
 727                 }
 728                 nv_wr32(dev, ustatus_addr, 0xc0000000);
 729         }
 730
 731         if (!tps && display)
 732                 NV_INFO(dev, "%s - No TPs claiming errors?\n", name);
 733 }
 734
 735 static void
 736 nv50_pgraph_trap_handler(struct drm_device *dev)
 737 {
 738         struct nouveau_pgraph_trap trap;
 739         uint32_t status = nv_rd32(dev, 0x400108);
 740         uint32_t ustatus;
 741         int display = nouveau_ratelimit();
 742
 743
 744         if (!status && display) {
 745                 nouveau_graph_trap_info(dev, &trap);
 746                 nouveau_graph_dump_trap_info(dev, "PGRAPH_TRAP", &trap);
 747                 NV_INFO(dev, "PGRAPH_TRAP - no units reporting traps?\n");
 748         }
 749
 750         /* DISPATCH: Relays commands to other units and handles NOTIFY,
 751          * COND, QUERY. If you get a trap from it, the command is still stuck
 752          * in DISPATCH and you need to do something about it. */
 753         if (status & 0x001) {
 754                 ustatus = nv_rd32(dev, 0x400804) & 0x7fffffff;
 755                 if (!ustatus && display) {
 756                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH - no ustatus?\n");
 757                 }
 758
 759                 /* Known to be triggered by screwed up NOTIFY and COND... */
 760                 if (ustatus & 0x00000001) {
 761                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_DISPATCH_FAULT");
 762                         nv_wr32(dev, 0x400500, 0);
 763                         if (nv_rd32(dev, 0x400808) & 0x80000000) {
 764                                 if (display) {
 765                                         if (nouveau_graph_trapped_channel(dev, &trap.channel))
 766                                                 trap.channel = -1;
 767                                         trap.class = nv_rd32(dev, 0x400814);
 768                                         trap.mthd = nv_rd32(dev, 0x400808) & 0x1ffc;
 769                                         trap.subc = (nv_rd32(dev, 0x400808) >> 16) & 0x7;
 770                                         trap.data = nv_rd32(dev, 0x40080c);
 771                                         trap.data2 = nv_rd32(dev, 0x400810);
 772                                         nouveau_graph_dump_trap_info(dev,
 773                                                         "PGRAPH_TRAP_DISPATCH_FAULT", &trap);
 774                                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_FAULT - 400808: %08x\n", nv_rd32(dev, 0x400808));
 775                                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_FAULT - 400848: %08x\n", nv_rd32(dev, 0x400848));
 776                                 }
 777                                 nv_wr32(dev, 0x400808, 0);
 778                         } else if (display) {
 779                                 NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_FAULT - No stuck command?\n");
 780                         }
 781                         nv_wr32(dev, 0x4008e8, nv_rd32(dev, 0x4008e8) & 3);
 782                         nv_wr32(dev, 0x400848, 0);
 783                         ustatus &= ~0x00000001;
 784                 }
 785                 if (ustatus & 0x00000002) {
 786                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_DISPATCH_QUERY");
 787                         nv_wr32(dev, 0x400500, 0);
 788                         if (nv_rd32(dev, 0x40084c) & 0x80000000) {
 789                                 if (display) {
 790                                         if (nouveau_graph_trapped_channel(dev, &trap.channel))
 791                                                 trap.channel = -1;
 792                                         trap.class = nv_rd32(dev, 0x400814);
 793                                         trap.mthd = nv_rd32(dev, 0x40084c) & 0x1ffc;
 794                                         trap.subc = (nv_rd32(dev, 0x40084c) >> 16) & 0x7;
 795                                         trap.data = nv_rd32(dev, 0x40085c);
 796                                         trap.data2 = 0;
 797                                         nouveau_graph_dump_trap_info(dev,
 798                                                         "PGRAPH_TRAP_DISPATCH_QUERY", &trap);
 799                                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_QUERY - 40084c: %08x\n", nv_rd32(dev, 0x40084c));
 800                                 }
 801                                 nv_wr32(dev, 0x40084c, 0);
 802                         } else if (display) {
 803                                 NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_QUERY - No stuck command?\n");
 804                         }
 805                         ustatus &= ~0x00000002;
 806                 }
 807                 if (ustatus && display)
 808                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH - Unhandled ustatus 0x%08x\n", ustatus);
 809                 nv_wr32(dev, 0x400804, 0xc0000000);
 810                 nv_wr32(dev, 0x400108, 0x001);
 811                 status &= ~0x001;
 812         }
 813
 814         /* TRAPs other than dispatch use the "normal" trap regs. */
 815         if (status && display) {
 816                 nouveau_graph_trap_info(dev, &trap);
 817                 nouveau_graph_dump_trap_info(dev,
 818                                 "PGRAPH_TRAP", &trap);
 819         }
 820
 821         /* M2MF: Memory to memory copy engine. */
 822         if (status & 0x002) {
 823                 ustatus = nv_rd32(dev, 0x406800) & 0x7fffffff;
 824                 if (!ustatus && display) {
 825                         NV_INFO(dev, "PGRAPH_TRAP_M2MF - no ustatus?\n");
 826                 }
 827                 if (ustatus & 0x00000001) {
 828                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_M2MF_NOTIFY");
 829                         ustatus &= ~0x00000001;
 830                 }
 831                 if (ustatus & 0x00000002) {
 832                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_M2MF_IN");
 833                         ustatus &= ~0x00000002;
 834                 }
 835                 if (ustatus & 0x00000004) {
 836                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_M2MF_OUT");
 837                         ustatus &= ~0x00000004;
 838                 }
 839                 NV_INFO (dev, "PGRAPH_TRAP_M2MF - %08x %08x %08x %08x\n",
 840                                 nv_rd32(dev, 0x406804),
 841                                 nv_rd32(dev, 0x406808),
 842                                 nv_rd32(dev, 0x40680c),
 843                                 nv_rd32(dev, 0x406810));
 844                 if (ustatus && display)
 845                         NV_INFO(dev, "PGRAPH_TRAP_M2MF - Unhandled ustatus 0x%08x\n", ustatus);
 846                 /* No sane way found yet -- just reset the bugger. */
 847                 nv_wr32(dev, 0x400040, 2);
 848                 nv_wr32(dev, 0x400040, 0);
 849                 nv_wr32(dev, 0x406800, 0xc0000000);
 850                 nv_wr32(dev, 0x400108, 0x002);
 851                 status &= ~0x002;
 852         }
 853
 854         /* VFETCH: Fetches data from vertex buffers. */
 855         if (status & 0x004) {
 856                 ustatus = nv_rd32(dev, 0x400c04) & 0x7fffffff;
 857                 if (!ustatus && display) {
 858                         NV_INFO(dev, "PGRAPH_TRAP_VFETCH - no ustatus?\n");
 859                 }
 860                 if (ustatus & 0x00000001) {
 861                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_VFETCH_FAULT");
 862                         NV_INFO (dev, "PGRAPH_TRAP_VFETCH_FAULT - %08x %08x %08x %08x\n",
 863                                         nv_rd32(dev, 0x400c00),
 864                                         nv_rd32(dev, 0x400c08),
 865                                         nv_rd32(dev, 0x400c0c),
 866                                         nv_rd32(dev, 0x400c10));
 867                         ustatus &= ~0x00000001;
 868                 }
 869                 if (ustatus && display)
 870                         NV_INFO(dev, "PGRAPH_TRAP_VFETCH - Unhandled ustatus 0x%08x\n", ustatus);
 871                 nv_wr32(dev, 0x400c04, 0xc0000000);
 872                 nv_wr32(dev, 0x400108, 0x004);
 873                 status &= ~0x004;
 874         }
 875
 876         /* STRMOUT: DirectX streamout / OpenGL transform feedback. */
 877         if (status & 0x008) {
 878                 ustatus = nv_rd32(dev, 0x401800) & 0x7fffffff;
 879                 if (!ustatus && display) {
 880                         NV_INFO(dev, "PGRAPH_TRAP_STRMOUT - no ustatus?\n");
 881                 }
 882                 if (ustatus & 0x00000001) {
 883                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_STRMOUT_FAULT");
 884                         NV_INFO (dev, "PGRAPH_TRAP_STRMOUT_FAULT - %08x %08x %08x %08x\n",
 885                                         nv_rd32(dev, 0x401804),
 886                                         nv_rd32(dev, 0x401808),
 887                                         nv_rd32(dev, 0x40180c),
 888                                         nv_rd32(dev, 0x401810));
 889                         ustatus &= ~0x00000001;
 890                 }
 891                 if (ustatus && display)
 892                         NV_INFO(dev, "PGRAPH_TRAP_STRMOUT - Unhandled ustatus 0x%08x\n", ustatus);
 893                 /* No sane way found yet -- just reset the bugger. */
 894                 nv_wr32(dev, 0x400040, 0x80);
 895                 nv_wr32(dev, 0x400040, 0);
 896                 nv_wr32(dev, 0x401800, 0xc0000000);
 897                 nv_wr32(dev, 0x400108, 0x008);
 898                 status &= ~0x008;
 899         }
 900
 901         /* CCACHE: Handles code and c[] caches and fills them. */
 902         if (status & 0x010) {
 903                 ustatus = nv_rd32(dev, 0x405018) & 0x7fffffff;
 904                 if (!ustatus && display) {
 905                         NV_INFO(dev, "PGRAPH_TRAP_CCACHE - no ustatus?\n");
 906                 }
 907                 if (ustatus & 0x00000001) {
 908                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_CCACHE_FAULT");
 909                         NV_INFO (dev, "PGRAPH_TRAP_CCACHE_FAULT - %08x %08x %08x %08x %08x %08x %08x\n",
 910                                         nv_rd32(dev, 0x405800),
 911                                         nv_rd32(dev, 0x405804),
 912                                         nv_rd32(dev, 0x405808),
 913                                         nv_rd32(dev, 0x40580c),
 914                                         nv_rd32(dev, 0x405810),
 915                                         nv_rd32(dev, 0x405814),
 916                                         nv_rd32(dev, 0x40581c));
 917                         ustatus &= ~0x00000001;
 918                 }
 919                 if (ustatus && display)
 920                         NV_INFO(dev, "PGRAPH_TRAP_CCACHE - Unhandled ustatus 0x%08x\n", ustatus);
 921                 nv_wr32(dev, 0x405018, 0xc0000000);
 922                 nv_wr32(dev, 0x400108, 0x010);
 923                 status &= ~0x010;
 924         }
 925
 926         /* Unknown, not seen yet... 0x402000 is the only trap status reg
 927          * remaining, so try to handle it anyway. Perhaps related to that
 928          * unknown DMA slot on tesla? */
 929         if (status & 0x20) {
 930                 nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_UNKC04");
 931                 ustatus = nv_rd32(dev, 0x402000) & 0x7fffffff;
 932                 if (display)
 933                         NV_INFO(dev, "PGRAPH_TRAP_UNKC04 - Unhandled ustatus 0x%08x\n", ustatus);
 934                 nv_wr32(dev, 0x402000, 0xc0000000);
 935                 /* no status modifiction on purpose */
 936         }
 937
 938         /* TEXTURE: CUDA texturing units */
 939         if (status & 0x040) {
 940                 nv50_pgraph_tp_trap (dev, 6, 0x408900, 0x408600, display,
 941                                 "PGRAPH_TRAP_TEXTURE");
 942                 nv_wr32(dev, 0x400108, 0x040);
 943                 status &= ~0x040;
 944         }
 945
 946         /* MP: CUDA execution engines. */
 947         if (status & 0x080) {
 948                 nv50_pgraph_tp_trap (dev, 7, 0x408314, 0x40831c, display,
 949                                 "PGRAPH_TRAP_MP");
 950                 nv_wr32(dev, 0x400108, 0x080);
 951                 status &= ~0x080;
 952         }
 953
 954         /* TPDMA:  Handles TP-initiated uncached memory accesses:
 955          * l[], g[], stack, 2d surfaces, render targets. */
 956         if (status & 0x100) {
 957                 nv50_pgraph_tp_trap (dev, 8, 0x408e08, 0x408708, display,
 958                                 "PGRAPH_TRAP_TPDMA");
 959                 nv_wr32(dev, 0x400108, 0x100);
 960                 status &= ~0x100;
 961         }
 962
 963         if (status) {
 964                 if (display)
 965                         NV_INFO(dev, "PGRAPH_TRAP - Unknown trap 0x%08x\n",
 966                                 status);
 967                 nv_wr32(dev, 0x400108, status);
 968         }
 969 }
 970
 971 /* There must be a *lot* of these. Will take some time to gather them up. */
 972 static struct nouveau_enum nv50_data_error_names[] =
 973 {
 974         { 4,    "INVALID_VALUE" },
 975         { 5,    "INVALID_ENUM" },
 976         { 8,    "INVALID_OBJECT" },
 977         { 0xc,  "INVALID_BITFIELD" },
 978         { 0x28, "MP_NO_REG_SPACE" },
 979         { 0x2b, "MP_BLOCK_SIZE_MISMATCH" },
 980         {}
 981 };
 982
 983 static void
 984 nv50_pgraph_irq_handler(struct drm_device *dev)
 985 {
 986         struct nouveau_pgraph_trap trap;
 987         int unhandled = 0;
 988         uint32_t status;
 989
 990         while ((status = nv_rd32(dev, NV03_PGRAPH_INTR))) {
 991                 /* NOTIFY: You've set a NOTIFY an a command and it's done. */
 992                 if (status & 0x00000001) {
 993                         nouveau_graph_trap_info(dev, &trap);
 994                         if (nouveau_ratelimit())
 995                                 nouveau_graph_dump_trap_info(dev,
 996                                                 "PGRAPH_NOTIFY", &trap);
 997                         status &= ~0x00000001;
 998                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000001);
 999                 }
1000
1001                 /* COMPUTE_QUERY: Purpose and exact cause unknown, happens
1002                  * when you write 0x200 to 0x50c0 method 0x31c. */
1003                 if (status & 0x00000002) {
1004                         nouveau_graph_trap_info(dev, &trap);
1005                         if (nouveau_ratelimit())
1006                                 nouveau_graph_dump_trap_info(dev,
1007                                                 "PGRAPH_COMPUTE_QUERY", &trap);
1008                         status &= ~0x00000002;
1009                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000002);
1010                 }
1011
1012                 /* Unknown, never seen: 0x4 */
1013
1014                 /* ILLEGAL_MTHD: You used a wrong method for this class. */
1015                 if (status & 0x00000010) {
1016                         nouveau_graph_trap_info(dev, &trap);
1017                         if (nouveau_pgraph_intr_swmthd(dev, &trap))
1018                                 unhandled = 1;
1019                         if (unhandled && nouveau_ratelimit())
1020                                 nouveau_graph_dump_trap_info(dev,
1021                                                 "PGRAPH_ILLEGAL_MTHD", &trap);
1022                         status &= ~0x00000010;
1023                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000010);
1024                 }
1025
1026                 /* ILLEGAL_CLASS: You used a wrong class. */
1027                 if (status & 0x00000020) {
1028                         nouveau_graph_trap_info(dev, &trap);
1029                         if (nouveau_ratelimit())
1030                                 nouveau_graph_dump_trap_info(dev,
1031                                                 "PGRAPH_ILLEGAL_CLASS", &trap);
1032                         status &= ~0x00000020;
1033                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000020);
1034                 }
1035
1036                 /* DOUBLE_NOTIFY: You tried to set a NOTIFY on another NOTIFY. */
1037                 if (status & 0x00000040) {
1038                         nouveau_graph_trap_info(dev, &trap);
1039                         if (nouveau_ratelimit())
1040                                 nouveau_graph_dump_trap_info(dev,
1041                                                 "PGRAPH_DOUBLE_NOTIFY", &trap);
1042                         status &= ~0x00000040;
1043                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000040);
1044                 }
1045
1046                 /* CONTEXT_SWITCH: PGRAPH needs us to load a new context */
1047                 if (status & 0x00001000) {
1048                         nv_wr32(dev, 0x400500, 0x00000000);
1049                         nv_wr32(dev, NV03_PGRAPH_INTR,
1050                                 NV_PGRAPH_INTR_CONTEXT_SWITCH);
1051                         nv_wr32(dev, NV40_PGRAPH_INTR_EN, nv_rd32(dev,
1052                                 NV40_PGRAPH_INTR_EN) &
1053                                 ~NV_PGRAPH_INTR_CONTEXT_SWITCH);
1054                         nv_wr32(dev, 0x400500, 0x00010001);
1055
1056                         nv50_graph_context_switch(dev);
1057
1058                         status &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
1059                 }
1060
1061                 /* BUFFER_NOTIFY: Your m2mf transfer finished */
1062                 if (status & 0x00010000) {
1063                         nouveau_graph_trap_info(dev, &trap);
1064                         if (nouveau_ratelimit())
1065                                 nouveau_graph_dump_trap_info(dev,
1066                                                 "PGRAPH_BUFFER_NOTIFY", &trap);
1067                         status &= ~0x00010000;
1068                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00010000);
1069                 }
1070
1071                 /* DATA_ERROR: Invalid value for this method, or invalid
1072                  * state in current PGRAPH context for this operation */
1073                 if (status & 0x00100000) {
1074                         nouveau_graph_trap_info(dev, &trap);
1075                         if (nouveau_ratelimit()) {
1076                                 nouveau_graph_dump_trap_info(dev,
1077                                                 "PGRAPH_DATA_ERROR", &trap);
1078                                 NV_INFO (dev, "PGRAPH_DATA_ERROR - ");
1079                                 nouveau_enum_print(nv50_data_error_names,
1080                                                    nv_rd32(dev, 0x400110));
1081                                 printk("\n");
1082                         }
1083                         status &= ~0x00100000;
1084                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00100000);
1085                 }
1086
1087                 /* TRAP: Something bad happened in the middle of command
1088                  * execution.  Has a billion types, subtypes, and even
1089                  * subsubtypes. */
1090                 if (status & 0x00200000) {
1091                         nv50_pgraph_trap_handler(dev);
1092                         status &= ~0x00200000;
1093                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00200000);
1094                 }
1095
1096                 /* Unknown, never seen: 0x00400000 */
1097
1098                 /* SINGLE_STEP: Happens on every method if you turned on
1099                  * single stepping in 40008c */
1100                 if (status & 0x01000000) {
1101                         nouveau_graph_trap_info(dev, &trap);
1102                         if (nouveau_ratelimit())
1103                                 nouveau_graph_dump_trap_info(dev,
1104                                                 "PGRAPH_SINGLE_STEP", &trap);
1105                         status &= ~0x01000000;
1106                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x01000000);
1107                 }
1108
1109                 /* 0x02000000 happens when you pause a ctxprog...
1110                  * but the only way this can happen that I know is by
1111                  * poking the relevant MMIO register, and we don't
1112                  * do that. */
1113
1114                 if (status) {
1115                         NV_INFO(dev, "Unhandled PGRAPH_INTR - 0x%08x\n",
1116                                 status);
1117                         nv_wr32(dev, NV03_PGRAPH_INTR, status);
1118                 }
1119
1120                 {
1121                         const int isb = (1 << 16) | (1 << 0);
1122
1123                         if ((nv_rd32(dev, 0x400500) & isb) != isb)
1124                                 nv_wr32(dev, 0x400500,
1125                                         nv_rd32(dev, 0x400500) | isb);
1126                 }
1127         }
1128
1129         nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PGRAPH_PENDING);
1130         if (nv_rd32(dev, 0x400824) & (1 << 31))
1131                 nv_wr32(dev, 0x400824, nv_rd32(dev, 0x400824) & ~(1 << 31));
1132 }
1133
1134 irqreturn_t
1135 nouveau_irq_handler(DRM_IRQ_ARGS)
1136 {
1137         struct drm_device *dev = (struct drm_device *)arg;
1138         struct drm_nouveau_private *dev_priv = dev->dev_private;
1139         unsigned long flags;
1140         u32 status;
1141         int i;
1142
1143         status = nv_rd32(dev, NV03_PMC_INTR_0);
1144         if (!status)
1145                 return IRQ_NONE;
1146
1147         spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
1148
1149         if (status & NV_PMC_INTR_0_PFIFO_PENDING) {
1150                 nouveau_fifo_irq_handler(dev);
1151                 status &= ~NV_PMC_INTR_0_PFIFO_PENDING;
1152         }
1153
1154         if (status & NV_PMC_INTR_0_PGRAPH_PENDING) {
1155                 if (dev_priv->card_type >= NV_50)
1156                         nv50_pgraph_irq_handler(dev);
1157                 else
1158                         nouveau_pgraph_irq_handler(dev);
1159
1160                 status &= ~NV_PMC_INTR_0_PGRAPH_PENDING;
1161         }
1162
1163         for (i = 0; i < 32 && status; i++) {
1164                 if (!(status & (1 << i)) || !dev_priv->irq_handler[i])
1165                         continue;
1166
1167                 dev_priv->irq_handler[i](dev);
1168                 status &= ~(1 << i);
1169         }
1170
1171         if (status)
1172                 NV_ERROR(dev, "Unhandled PMC INTR status bits 0x%08x\n", status);
1173
1174         spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
1175
1176         if (dev_priv->msi_enabled)
1177                 nv_wr08(dev, 0x00088068, 0xff);
1178
1179         return IRQ_HANDLED;
1180 }
1181
1182 int
1183 nouveau_irq_init(struct drm_device *dev)
1184 {
1185         struct drm_nouveau_private *dev_priv = dev->dev_private;
1186         int ret;
1187
1188         if (nouveau_msi != 0 && dev_priv->card_type >= NV_50) {
1189                 ret = pci_enable_msi(dev->pdev);
1190                 if (ret == 0) {
1191                         NV_INFO(dev, "enabled MSI\n");
1192                         dev_priv->msi_enabled = true;
1193                 }
1194         }
1195
1196         return drm_irq_install(dev);
1197 }
1198
1199 void
1200 nouveau_irq_fini(struct drm_device *dev)
1201 {
1202         struct drm_nouveau_private *dev_priv = dev->dev_private;
1203
1204         drm_irq_uninstall(dev);
1205         if (dev_priv->msi_enabled)
1206                 pci_disable_msi(dev->pdev);
1207 }
1208
1209 void
1210 nouveau_irq_register(struct drm_device *dev, int status_bit,
1211                      void (*handler)(struct drm_device *))
1212 {
1213         struct drm_nouveau_private *dev_priv = dev->dev_private;
1214         unsigned long flags;
1215
1216         spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
1217         dev_priv->irq_handler[status_bit] = handler;
1218         spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
1219 }
1220
1221 void
1222 nouveau_irq_unregister(struct drm_device *dev, int status_bit)
1223 {
1224         struct drm_nouveau_private *dev_priv = dev->dev_private;
1225         unsigned long flags;
1226
1227         spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
1228         dev_priv->irq_handler[status_bit] = NULL;
1229         spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
1230 }