drivers/gpu/drm/nouveau/nouveau_irq.c

   1 /*
   2  * Copyright (C) 2006 Ben Skeggs.
   3  *
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining
   7  * a copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sublicense, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial
  16  * portions of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  */
  27
  28 /*
  29  * Authors:
  30  *   Ben Skeggs <darktama@iinet.net.au>
  31  */
  32
  33 #include "drmP.h"
  34 #include "drm.h"
  35 #include "nouveau_drm.h"
  36 #include "nouveau_drv.h"
  37 #include "nouveau_reg.h"
  38 #include "nouveau_ramht.h"
  39 #include "nouveau_util.h"
  40
  41 void
  42 nouveau_irq_preinstall(struct drm_device *dev)
  43 {
  44         struct drm_nouveau_private *dev_priv = dev->dev_private;
  45
  46         /* Master disable */
  47         nv_wr32(dev, NV03_PMC_INTR_EN_0, 0);
  48
  49         INIT_LIST_HEAD(&dev_priv->vbl_waiting);
  50 }
  51
  52 int
  53 nouveau_irq_postinstall(struct drm_device *dev)
  54 {
  55         struct drm_nouveau_private *dev_priv = dev->dev_private;
  56
  57         /* Master enable */
  58         nv_wr32(dev, NV03_PMC_INTR_EN_0, NV_PMC_INTR_EN_0_MASTER_ENABLE);
  59         if (dev_priv->msi_enabled)
  60                 nv_wr08(dev, 0x00088068, 0xff);
  61
  62         return 0;
  63 }
  64
  65 void
  66 nouveau_irq_uninstall(struct drm_device *dev)
  67 {
  68         /* Master disable */
  69         nv_wr32(dev, NV03_PMC_INTR_EN_0, 0);
  70 }
  71
  72 static bool
  73 nouveau_fifo_swmthd(struct drm_device *dev, u32 chid, u32 addr, u32 data)
  74 {
  75         struct drm_nouveau_private *dev_priv = dev->dev_private;
  76         struct nouveau_channel *chan = NULL;
  77         struct nouveau_gpuobj *obj;
  78         unsigned long flags;
  79         const int subc = (addr >> 13) & 0x7;
  80         const int mthd = addr & 0x1ffc;
  81         bool handled = false;
  82         u32 engine;
  83
  84         spin_lock_irqsave(&dev_priv->channels.lock, flags);
  85         if (likely(chid >= 0 && chid < dev_priv->engine.fifo.channels))
  86                 chan = dev_priv->channels.ptr[chid];
  87         if (unlikely(!chan))
  88                 goto out;
  89
  90         switch (mthd) {
  91         case 0x0000: /* bind object to subchannel */
  92                 obj = nouveau_ramht_find(chan, data);
  93                 if (unlikely(!obj || obj->engine != NVOBJ_ENGINE_SW))
  94                         break;
  95
  96                 chan->sw_subchannel[subc] = obj->class;
  97                 engine = 0x0000000f << (subc * 4);
  98
  99                 nv_mask(dev, NV04_PFIFO_CACHE1_ENGINE, engine, 0x00000000);
 100                 handled = true;
 101                 break;
 102         default:
 103                 engine = nv_rd32(dev, NV04_PFIFO_CACHE1_ENGINE);
 104                 if (unlikely(((engine >> (subc * 4)) & 0xf) != 0))
 105                         break;
 106
 107                 if (!nouveau_gpuobj_mthd_call(chan, chan->sw_subchannel[subc],
 108                                               mthd, data))
 109                         handled = true;
 110                 break;
 111         }
 112
 113 out:
 114         spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
 115         return handled;
 116 }
 117
 118 static void
 119 nouveau_fifo_irq_handler(struct drm_device *dev)
 120 {
 121         struct drm_nouveau_private *dev_priv = dev->dev_private;
 122         struct nouveau_engine *engine = &dev_priv->engine;
 123         uint32_t status, reassign;
 124         int cnt = 0;
 125
 126         reassign = nv_rd32(dev, NV03_PFIFO_CACHES) & 1;
 127         while ((status = nv_rd32(dev, NV03_PFIFO_INTR_0)) && (cnt++ < 100)) {
 128                 uint32_t chid, get;
 129
 130                 nv_wr32(dev, NV03_PFIFO_CACHES, 0);
 131
 132                 chid = engine->fifo.channel_id(dev);
 133                 get  = nv_rd32(dev, NV03_PFIFO_CACHE1_GET);
 134
 135                 if (status & NV_PFIFO_INTR_CACHE_ERROR) {
 136                         uint32_t mthd, data;
 137                         int ptr;
 138
 139                         /* NV_PFIFO_CACHE1_GET actually goes to 0xffc before
 140                          * wrapping on my G80 chips, but CACHE1 isn't big
 141                          * enough for this much data.. Tests show that it
 142                          * wraps around to the start at GET=0x800.. No clue
 143                          * as to why..
 144                          */
 145                         ptr = (get & 0x7ff) >> 2;
 146
 147                         if (dev_priv->card_type < NV_40) {
 148                                 mthd = nv_rd32(dev,
 149                                         NV04_PFIFO_CACHE1_METHOD(ptr));
 150                                 data = nv_rd32(dev,
 151                                         NV04_PFIFO_CACHE1_DATA(ptr));
 152                         } else {
 153                                 mthd = nv_rd32(dev,
 154                                         NV40_PFIFO_CACHE1_METHOD(ptr));
 155                                 data = nv_rd32(dev,
 156                                         NV40_PFIFO_CACHE1_DATA(ptr));
 157                         }
 158
 159                         if (!nouveau_fifo_swmthd(dev, chid, mthd, data)) {
 160                                 NV_INFO(dev, "PFIFO_CACHE_ERROR - Ch %d/%d "
 161                                              "Mthd 0x%04x Data 0x%08x\n",
 162                                         chid, (mthd >> 13) & 7, mthd & 0x1ffc,
 163                                         data);
 164                         }
 165
 166                         nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUSH, 0);
 167                         nv_wr32(dev, NV03_PFIFO_INTR_0,
 168                                                 NV_PFIFO_INTR_CACHE_ERROR);
 169
 170                         nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0,
 171                                 nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH0) & ~1);
 172                         nv_wr32(dev, NV03_PFIFO_CACHE1_GET, get + 4);
 173                         nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0,
 174                                 nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH0) | 1);
 175                         nv_wr32(dev, NV04_PFIFO_CACHE1_HASH, 0);
 176
 177                         nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUSH,
 178                                 nv_rd32(dev, NV04_PFIFO_CACHE1_DMA_PUSH) | 1);
 179                         nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
 180
 181                         status &= ~NV_PFIFO_INTR_CACHE_ERROR;
 182                 }
 183
 184                 if (status & NV_PFIFO_INTR_DMA_PUSHER) {
 185                         u32 dma_get = nv_rd32(dev, 0x003244);
 186                         u32 dma_put = nv_rd32(dev, 0x003240);
 187                         u32 push = nv_rd32(dev, 0x003220);
 188                         u32 state = nv_rd32(dev, 0x003228);
 189
 190                         if (dev_priv->card_type == NV_50) {
 191                                 u32 ho_get = nv_rd32(dev, 0x003328);
 192                                 u32 ho_put = nv_rd32(dev, 0x003320);
 193                                 u32 ib_get = nv_rd32(dev, 0x003334);
 194                                 u32 ib_put = nv_rd32(dev, 0x003330);
 195
 196                                 if (nouveau_ratelimit())
 197                                         NV_INFO(dev, "PFIFO_DMA_PUSHER - Ch %d Get 0x%02x%08x "
 198                                              "Put 0x%02x%08x IbGet 0x%08x IbPut 0x%08x "
 199                                              "State 0x%08x Push 0x%08x\n",
 200                                                 chid, ho_get, dma_get, ho_put,
 201                                                 dma_put, ib_get, ib_put, state,
 202                                                 push);
 203
 204                                 /* METHOD_COUNT, in DMA_STATE on earlier chipsets */
 205                                 nv_wr32(dev, 0x003364, 0x00000000);
 206                                 if (dma_get != dma_put || ho_get != ho_put) {
 207                                         nv_wr32(dev, 0x003244, dma_put);
 208                                         nv_wr32(dev, 0x003328, ho_put);
 209                                 } else
 210                                 if (ib_get != ib_put) {
 211                                         nv_wr32(dev, 0x003334, ib_put);
 212                                 }
 213                         } else {
 214                                 NV_INFO(dev, "PFIFO_DMA_PUSHER - Ch %d Get 0x%08x "
 215                                              "Put 0x%08x State 0x%08x Push 0x%08x\n",
 216                                         chid, dma_get, dma_put, state, push);
 217
 218                                 if (dma_get != dma_put)
 219                                         nv_wr32(dev, 0x003244, dma_put);
 220                         }
 221
 222                         nv_wr32(dev, 0x003228, 0x00000000);
 223                         nv_wr32(dev, 0x003220, 0x00000001);
 224                         nv_wr32(dev, 0x002100, NV_PFIFO_INTR_DMA_PUSHER);
 225                         status &= ~NV_PFIFO_INTR_DMA_PUSHER;
 226                 }
 227
 228                 if (status & NV_PFIFO_INTR_SEMAPHORE) {
 229                         uint32_t sem;
 230
 231                         status &= ~NV_PFIFO_INTR_SEMAPHORE;
 232                         nv_wr32(dev, NV03_PFIFO_INTR_0,
 233                                 NV_PFIFO_INTR_SEMAPHORE);
 234
 235                         sem = nv_rd32(dev, NV10_PFIFO_CACHE1_SEMAPHORE);
 236                         nv_wr32(dev, NV10_PFIFO_CACHE1_SEMAPHORE, sem | 0x1);
 237
 238                         nv_wr32(dev, NV03_PFIFO_CACHE1_GET, get + 4);
 239                         nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
 240                 }
 241
 242                 if (dev_priv->card_type == NV_50) {
 243                         if (status & 0x00000010) {
 244                                 nv50_fb_vm_trap(dev, 1, "PFIFO_BAR_FAULT");
 245                                 status &= ~0x00000010;
 246                                 nv_wr32(dev, 0x002100, 0x00000010);
 247                         }
 248                 }
 249
 250                 if (status) {
 251                         if (nouveau_ratelimit())
 252                                 NV_INFO(dev, "PFIFO_INTR 0x%08x - Ch %d\n",
 253                                         status, chid);
 254                         nv_wr32(dev, NV03_PFIFO_INTR_0, status);
 255                         status = 0;
 256                 }
 257
 258                 nv_wr32(dev, NV03_PFIFO_CACHES, reassign);
 259         }
 260
 261         if (status) {
 262                 NV_INFO(dev, "PFIFO still angry after %d spins, halt\n", cnt);
 263                 nv_wr32(dev, 0x2140, 0);
 264                 nv_wr32(dev, 0x140, 0);
 265         }
 266
 267         nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PFIFO_PENDING);
 268 }
 269
 270 struct nouveau_bitfield_names {
 271         uint32_t mask;
 272         const char *name;
 273 };
 274
 275 static struct nouveau_bitfield_names nstatus_names[] =
 276 {
 277         { NV04_PGRAPH_NSTATUS_STATE_IN_USE,       "STATE_IN_USE" },
 278         { NV04_PGRAPH_NSTATUS_INVALID_STATE,      "INVALID_STATE" },
 279         { NV04_PGRAPH_NSTATUS_BAD_ARGUMENT,       "BAD_ARGUMENT" },
 280         { NV04_PGRAPH_NSTATUS_PROTECTION_FAULT,   "PROTECTION_FAULT" }
 281 };
 282
 283 static struct nouveau_bitfield_names nstatus_names_nv10[] =
 284 {
 285         { NV10_PGRAPH_NSTATUS_STATE_IN_USE,       "STATE_IN_USE" },
 286         { NV10_PGRAPH_NSTATUS_INVALID_STATE,      "INVALID_STATE" },
 287         { NV10_PGRAPH_NSTATUS_BAD_ARGUMENT,       "BAD_ARGUMENT" },
 288         { NV10_PGRAPH_NSTATUS_PROTECTION_FAULT,   "PROTECTION_FAULT" }
 289 };
 290
 291 static struct nouveau_bitfield_names nsource_names[] =
 292 {
 293         { NV03_PGRAPH_NSOURCE_NOTIFICATION,       "NOTIFICATION" },
 294         { NV03_PGRAPH_NSOURCE_DATA_ERROR,         "DATA_ERROR" },
 295         { NV03_PGRAPH_NSOURCE_PROTECTION_ERROR,   "PROTECTION_ERROR" },
 296         { NV03_PGRAPH_NSOURCE_RANGE_EXCEPTION,    "RANGE_EXCEPTION" },
 297         { NV03_PGRAPH_NSOURCE_LIMIT_COLOR,        "LIMIT_COLOR" },
 298         { NV03_PGRAPH_NSOURCE_LIMIT_ZETA,         "LIMIT_ZETA" },
 299         { NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD,       "ILLEGAL_MTHD" },
 300         { NV03_PGRAPH_NSOURCE_DMA_R_PROTECTION,   "DMA_R_PROTECTION" },
 301         { NV03_PGRAPH_NSOURCE_DMA_W_PROTECTION,   "DMA_W_PROTECTION" },
 302         { NV03_PGRAPH_NSOURCE_FORMAT_EXCEPTION,   "FORMAT_EXCEPTION" },
 303         { NV03_PGRAPH_NSOURCE_PATCH_EXCEPTION,    "PATCH_EXCEPTION" },
 304         { NV03_PGRAPH_NSOURCE_STATE_INVALID,      "STATE_INVALID" },
 305         { NV03_PGRAPH_NSOURCE_DOUBLE_NOTIFY,      "DOUBLE_NOTIFY" },
 306         { NV03_PGRAPH_NSOURCE_NOTIFY_IN_USE,      "NOTIFY_IN_USE" },
 307         { NV03_PGRAPH_NSOURCE_METHOD_CNT,         "METHOD_CNT" },
 308         { NV03_PGRAPH_NSOURCE_BFR_NOTIFICATION,   "BFR_NOTIFICATION" },
 309         { NV03_PGRAPH_NSOURCE_DMA_VTX_PROTECTION, "DMA_VTX_PROTECTION" },
 310         { NV03_PGRAPH_NSOURCE_DMA_WIDTH_A,        "DMA_WIDTH_A" },
 311         { NV03_PGRAPH_NSOURCE_DMA_WIDTH_B,        "DMA_WIDTH_B" },
 312 };
 313
 314 static void
 315 nouveau_print_bitfield_names_(uint32_t value,
 316                                 const struct nouveau_bitfield_names *namelist,
 317                                 const int namelist_len)
 318 {
 319         /*
 320          * Caller must have already printed the KERN_* log level for us.
 321          * Also the caller is responsible for adding the newline.
 322          */
 323         int i;
 324         for (i = 0; i < namelist_len; ++i) {
 325                 uint32_t mask = namelist[i].mask;
 326                 if (value & mask) {
 327                         printk(" %s", namelist[i].name);
 328                         value &= ~mask;
 329                 }
 330         }
 331         if (value)
 332                 printk(" (unknown bits 0x%08x)", value);
 333 }
 334 #define nouveau_print_bitfield_names(val, namelist) \
 335         nouveau_print_bitfield_names_((val), (namelist), ARRAY_SIZE(namelist))
 336
 337 struct nouveau_enum_names {
 338         uint32_t value;
 339         const char *name;
 340 };
 341
 342 static void
 343 nouveau_print_enum_names_(uint32_t value,
 344                                 const struct nouveau_enum_names *namelist,
 345                                 const int namelist_len)
 346 {
 347         /*
 348          * Caller must have already printed the KERN_* log level for us.
 349          * Also the caller is responsible for adding the newline.
 350          */
 351         int i;
 352         for (i = 0; i < namelist_len; ++i) {
 353                 if (value == namelist[i].value) {
 354                         printk("%s", namelist[i].name);
 355                         return;
 356                 }
 357         }
 358         printk("unknown value 0x%08x", value);
 359 }
 360 #define nouveau_print_enum_names(val, namelist) \
 361         nouveau_print_enum_names_((val), (namelist), ARRAY_SIZE(namelist))
 362
 363 static int
 364 nouveau_graph_chid_from_grctx(struct drm_device *dev)
 365 {
 366         struct drm_nouveau_private *dev_priv = dev->dev_private;
 367         struct nouveau_channel *chan;
 368         unsigned long flags;
 369         uint32_t inst;
 370         int i;
 371
 372         if (dev_priv->card_type < NV_40)
 373                 return dev_priv->engine.fifo.channels;
 374         else
 375         if (dev_priv->card_type < NV_50) {
 376                 inst = (nv_rd32(dev, 0x40032c) & 0xfffff) << 4;
 377
 378                 spin_lock_irqsave(&dev_priv->channels.lock, flags);
 379                 for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
 380                         chan = dev_priv->channels.ptr[i];
 381                         if (!chan || !chan->ramin_grctx)
 382                                 continue;
 383
 384                         if (inst == chan->ramin_grctx->pinst)
 385                                 break;
 386                 }
 387                 spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
 388         } else {
 389                 inst = (nv_rd32(dev, 0x40032c) & 0xfffff) << 12;
 390
 391                 spin_lock_irqsave(&dev_priv->channels.lock, flags);
 392                 for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
 393                         chan = dev_priv->channels.ptr[i];
 394                         if (!chan || !chan->ramin)
 395                                 continue;
 396
 397                         if (inst == chan->ramin->vinst)
 398                                 break;
 399                 }
 400                 spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
 401         }
 402
 403
 404         return i;
 405 }
 406
 407 static int
 408 nouveau_graph_trapped_channel(struct drm_device *dev, int *channel_ret)
 409 {
 410         struct drm_nouveau_private *dev_priv = dev->dev_private;
 411         struct nouveau_engine *engine = &dev_priv->engine;
 412         int channel;
 413
 414         if (dev_priv->card_type < NV_10)
 415                 channel = (nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR) >> 24) & 0xf;
 416         else
 417         if (dev_priv->card_type < NV_40)
 418                 channel = (nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR) >> 20) & 0x1f;
 419         else
 420                 channel = nouveau_graph_chid_from_grctx(dev);
 421
 422         if (channel >= engine->fifo.channels ||
 423             !dev_priv->channels.ptr[channel]) {
 424                 NV_ERROR(dev, "AIII, invalid/inactive channel id %d\n", channel);
 425                 return -EINVAL;
 426         }
 427
 428         *channel_ret = channel;
 429         return 0;
 430 }
 431
 432 struct nouveau_pgraph_trap {
 433         int channel;
 434         int class;
 435         int subc, mthd, size;
 436         uint32_t data, data2;
 437         uint32_t nsource, nstatus;
 438 };
 439
 440 static void
 441 nouveau_graph_trap_info(struct drm_device *dev,
 442                         struct nouveau_pgraph_trap *trap)
 443 {
 444         struct drm_nouveau_private *dev_priv = dev->dev_private;
 445         uint32_t address;
 446
 447         trap->nsource = trap->nstatus = 0;
 448         if (dev_priv->card_type < NV_50) {
 449                 trap->nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
 450                 trap->nstatus = nv_rd32(dev, NV03_PGRAPH_NSTATUS);
 451         }
 452
 453         if (nouveau_graph_trapped_channel(dev, &trap->channel))
 454                 trap->channel = -1;
 455         address = nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR);
 456
 457         trap->mthd = address & 0x1FFC;
 458         trap->data = nv_rd32(dev, NV04_PGRAPH_TRAPPED_DATA);
 459         if (dev_priv->card_type < NV_10) {
 460                 trap->subc  = (address >> 13) & 0x7;
 461         } else {
 462                 trap->subc  = (address >> 16) & 0x7;
 463                 trap->data2 = nv_rd32(dev, NV10_PGRAPH_TRAPPED_DATA_HIGH);
 464         }
 465
 466         if (dev_priv->card_type < NV_10)
 467                 trap->class = nv_rd32(dev, 0x400180 + trap->subc*4) & 0xFF;
 468         else if (dev_priv->card_type < NV_40)
 469                 trap->class = nv_rd32(dev, 0x400160 + trap->subc*4) & 0xFFF;
 470         else if (dev_priv->card_type < NV_50)
 471                 trap->class = nv_rd32(dev, 0x400160 + trap->subc*4) & 0xFFFF;
 472         else
 473                 trap->class = nv_rd32(dev, 0x400814);
 474 }
 475
 476 static void
 477 nouveau_graph_dump_trap_info(struct drm_device *dev, const char *id,
 478                              struct nouveau_pgraph_trap *trap)
 479 {
 480         struct drm_nouveau_private *dev_priv = dev->dev_private;
 481         uint32_t nsource = trap->nsource, nstatus = trap->nstatus;
 482
 483         if (dev_priv->card_type < NV_50) {
 484                 NV_INFO(dev, "%s - nSource:", id);
 485                 nouveau_print_bitfield_names(nsource, nsource_names);
 486                 printk(", nStatus:");
 487                 if (dev_priv->card_type < NV_10)
 488                         nouveau_print_bitfield_names(nstatus, nstatus_names);
 489                 else
 490                         nouveau_print_bitfield_names(nstatus, nstatus_names_nv10);
 491                 printk("\n");
 492         }
 493
 494         NV_INFO(dev, "%s - Ch %d/%d Class 0x%04x Mthd 0x%04x "
 495                                         "Data 0x%08x:0x%08x\n",
 496                                         id, trap->channel, trap->subc,
 497                                         trap->class, trap->mthd,
 498                                         trap->data2, trap->data);
 499 }
 500
 501 static int
 502 nouveau_pgraph_intr_swmthd(struct drm_device *dev,
 503                            struct nouveau_pgraph_trap *trap)
 504 {
 505         struct drm_nouveau_private *dev_priv = dev->dev_private;
 506         struct nouveau_channel *chan;
 507         unsigned long flags;
 508         int ret = -EINVAL;
 509
 510         spin_lock_irqsave(&dev_priv->channels.lock, flags);
 511         if (trap->channel > 0 &&
 512             trap->channel < dev_priv->engine.fifo.channels &&
 513             dev_priv->channels.ptr[trap->channel]) {
 514                 chan = dev_priv->channels.ptr[trap->channel];
 515                 ret = nouveau_gpuobj_mthd_call(chan, trap->class, trap->mthd, trap->data);
 516         }
 517         spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
 518
 519         return ret;
 520 }
 521
 522 static inline void
 523 nouveau_pgraph_intr_notify(struct drm_device *dev, uint32_t nsource)
 524 {
 525         struct nouveau_pgraph_trap trap;
 526         int unhandled = 0;
 527
 528         nouveau_graph_trap_info(dev, &trap);
 529
 530         if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
 531                 if (nouveau_pgraph_intr_swmthd(dev, &trap))
 532                         unhandled = 1;
 533         } else {
 534                 unhandled = 1;
 535         }
 536
 537         if (unhandled)
 538                 nouveau_graph_dump_trap_info(dev, "PGRAPH_NOTIFY", &trap);
 539 }
 540
 541
 542 static inline void
 543 nouveau_pgraph_intr_error(struct drm_device *dev, uint32_t nsource)
 544 {
 545         struct nouveau_pgraph_trap trap;
 546         int unhandled = 0;
 547
 548         nouveau_graph_trap_info(dev, &trap);
 549         trap.nsource = nsource;
 550
 551         if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
 552                 if (nouveau_pgraph_intr_swmthd(dev, &trap))
 553                         unhandled = 1;
 554         } else if (nsource & NV03_PGRAPH_NSOURCE_DMA_VTX_PROTECTION) {
 555                 uint32_t v = nv_rd32(dev, 0x402000);
 556                 nv_wr32(dev, 0x402000, v);
 557
 558                 /* dump the error anyway for now: it's useful for
 559                    Gallium development */
 560                 unhandled = 1;
 561         } else {
 562                 unhandled = 1;
 563         }
 564
 565         if (unhandled && nouveau_ratelimit())
 566                 nouveau_graph_dump_trap_info(dev, "PGRAPH_ERROR", &trap);
 567 }
 568
 569 static inline void
 570 nouveau_pgraph_intr_context_switch(struct drm_device *dev)
 571 {
 572         struct drm_nouveau_private *dev_priv = dev->dev_private;
 573         struct nouveau_engine *engine = &dev_priv->engine;
 574         uint32_t chid;
 575
 576         chid = engine->fifo.channel_id(dev);
 577         NV_DEBUG(dev, "PGRAPH context switch interrupt channel %x\n", chid);
 578
 579         switch (dev_priv->card_type) {
 580         case NV_04:
 581                 nv04_graph_context_switch(dev);
 582                 break;
 583         case NV_10:
 584                 nv10_graph_context_switch(dev);
 585                 break;
 586         default:
 587                 NV_ERROR(dev, "Context switch not implemented\n");
 588                 break;
 589         }
 590 }
 591
 592 static void
 593 nouveau_pgraph_irq_handler(struct drm_device *dev)
 594 {
 595         uint32_t status;
 596
 597         while ((status = nv_rd32(dev, NV03_PGRAPH_INTR))) {
 598                 uint32_t nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
 599
 600                 if (status & NV_PGRAPH_INTR_NOTIFY) {
 601                         nouveau_pgraph_intr_notify(dev, nsource);
 602
 603                         status &= ~NV_PGRAPH_INTR_NOTIFY;
 604                         nv_wr32(dev, NV03_PGRAPH_INTR, NV_PGRAPH_INTR_NOTIFY);
 605                 }
 606
 607                 if (status & NV_PGRAPH_INTR_ERROR) {
 608                         nouveau_pgraph_intr_error(dev, nsource);
 609
 610                         status &= ~NV_PGRAPH_INTR_ERROR;
 611                         nv_wr32(dev, NV03_PGRAPH_INTR, NV_PGRAPH_INTR_ERROR);
 612                 }
 613
 614                 if (status & NV_PGRAPH_INTR_CONTEXT_SWITCH) {
 615                         status &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
 616                         nv_wr32(dev, NV03_PGRAPH_INTR,
 617                                  NV_PGRAPH_INTR_CONTEXT_SWITCH);
 618
 619                         nouveau_pgraph_intr_context_switch(dev);
 620                 }
 621
 622                 if (status) {
 623                         NV_INFO(dev, "Unhandled PGRAPH_INTR - 0x%08x\n", status);
 624                         nv_wr32(dev, NV03_PGRAPH_INTR, status);
 625                 }
 626
 627                 if ((nv_rd32(dev, NV04_PGRAPH_FIFO) & (1 << 0)) == 0)
 628                         nv_wr32(dev, NV04_PGRAPH_FIFO, 1);
 629         }
 630
 631         nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PGRAPH_PENDING);
 632 }
 633
 634 static struct nouveau_enum_names nv50_mp_exec_error_names[] =
 635 {
 636         { 3, "STACK_UNDERFLOW" },
 637         { 4, "QUADON_ACTIVE" },
 638         { 8, "TIMEOUT" },
 639         { 0x10, "INVALID_OPCODE" },
 640         { 0x40, "BREAKPOINT" },
 641 };
 642
 643 static void
 644 nv50_pgraph_mp_trap(struct drm_device *dev, int tpid, int display)
 645 {
 646         struct drm_nouveau_private *dev_priv = dev->dev_private;
 647         uint32_t units = nv_rd32(dev, 0x1540);
 648         uint32_t addr, mp10, status, pc, oplow, ophigh;
 649         int i;
 650         int mps = 0;
 651         for (i = 0; i < 4; i++) {
 652                 if (!(units & 1 << (i+24)))
 653                         continue;
 654                 if (dev_priv->chipset < 0xa0)
 655                         addr = 0x408200 + (tpid << 12) + (i << 7);
 656                 else
 657                         addr = 0x408100 + (tpid << 11) + (i << 7);
 658                 mp10 = nv_rd32(dev, addr + 0x10);
 659                 status = nv_rd32(dev, addr + 0x14);
 660                 if (!status)
 661                         continue;
 662                 if (display) {
 663                         nv_rd32(dev, addr + 0x20);
 664                         pc = nv_rd32(dev, addr + 0x24);
 665                         oplow = nv_rd32(dev, addr + 0x70);
 666                         ophigh= nv_rd32(dev, addr + 0x74);
 667                         NV_INFO(dev, "PGRAPH_TRAP_MP_EXEC - "
 668                                         "TP %d MP %d: ", tpid, i);
 669                         nouveau_print_enum_names(status,
 670                                         nv50_mp_exec_error_names);
 671                         printk(" at %06x warp %d, opcode %08x %08x\n",
 672                                         pc&0xffffff, pc >> 24,
 673                                         oplow, ophigh);
 674                 }
 675                 nv_wr32(dev, addr + 0x10, mp10);
 676                 nv_wr32(dev, addr + 0x14, 0);
 677                 mps++;
 678         }
 679         if (!mps && display)
 680                 NV_INFO(dev, "PGRAPH_TRAP_MP_EXEC - TP %d: "
 681                                 "No MPs claiming errors?\n", tpid);
 682 }
 683
 684 static void
 685 nv50_pgraph_tp_trap(struct drm_device *dev, int type, uint32_t ustatus_old,
 686                 uint32_t ustatus_new, int display, const char *name)
 687 {
 688         struct drm_nouveau_private *dev_priv = dev->dev_private;
 689         int tps = 0;
 690         uint32_t units = nv_rd32(dev, 0x1540);
 691         int i, r;
 692         uint32_t ustatus_addr, ustatus;
 693         for (i = 0; i < 16; i++) {
 694                 if (!(units & (1 << i)))
 695                         continue;
 696                 if (dev_priv->chipset < 0xa0)
 697                         ustatus_addr = ustatus_old + (i << 12);
 698                 else
 699                         ustatus_addr = ustatus_new + (i << 11);
 700                 ustatus = nv_rd32(dev, ustatus_addr) & 0x7fffffff;
 701                 if (!ustatus)
 702                         continue;
 703                 tps++;
 704                 switch (type) {
 705                 case 6: /* texture error... unknown for now */
 706                         nv50_fb_vm_trap(dev, display, name);
 707                         if (display) {
 708                                 NV_ERROR(dev, "magic set %d:\n", i);
 709                                 for (r = ustatus_addr + 4; r <= ustatus_addr + 0x10; r += 4)
 710                                         NV_ERROR(dev, "\t0x%08x: 0x%08x\n", r,
 711                                                 nv_rd32(dev, r));
 712                         }
 713                         break;
 714                 case 7: /* MP error */
 715                         if (ustatus & 0x00010000) {
 716                                 nv50_pgraph_mp_trap(dev, i, display);
 717                                 ustatus &= ~0x00010000;
 718                         }
 719                         break;
 720                 case 8: /* TPDMA error */
 721                         {
 722                         uint32_t e0c = nv_rd32(dev, ustatus_addr + 4);
 723                         uint32_t e10 = nv_rd32(dev, ustatus_addr + 8);
 724                         uint32_t e14 = nv_rd32(dev, ustatus_addr + 0xc);
 725                         uint32_t e18 = nv_rd32(dev, ustatus_addr + 0x10);
 726                         uint32_t e1c = nv_rd32(dev, ustatus_addr + 0x14);
 727                         uint32_t e20 = nv_rd32(dev, ustatus_addr + 0x18);
 728                         uint32_t e24 = nv_rd32(dev, ustatus_addr + 0x1c);
 729                         nv50_fb_vm_trap(dev, display, name);
 730                         /* 2d engine destination */
 731                         if (ustatus & 0x00000010) {
 732                                 if (display) {
 733                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA_2D - TP %d - Unknown fault at address %02x%08x\n",
 734                                                         i, e14, e10);
 735                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA_2D - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
 736                                                         i, e0c, e18, e1c, e20, e24);
 737                                 }
 738                                 ustatus &= ~0x00000010;
 739                         }
 740                         /* Render target */
 741                         if (ustatus & 0x00000040) {
 742                                 if (display) {
 743                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA_RT - TP %d - Unknown fault at address %02x%08x\n",
 744                                                         i, e14, e10);
 745                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA_RT - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
 746                                                         i, e0c, e18, e1c, e20, e24);
 747                                 }
 748                                 ustatus &= ~0x00000040;
 749                         }
 750                         /* CUDA memory: l[], g[] or stack. */
 751                         if (ustatus & 0x00000080) {
 752                                 if (display) {
 753                                         if (e18 & 0x80000000) {
 754                                                 /* g[] read fault? */
 755                                                 NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Global read fault at address %02x%08x\n",
 756                                                                 i, e14, e10 | ((e18 >> 24) & 0x1f));
 757                                                 e18 &= ~0x1f000000;
 758                                         } else if (e18 & 0xc) {
 759                                                 /* g[] write fault? */
 760                                                 NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Global write fault at address %02x%08x\n",
 761                                                                 i, e14, e10 | ((e18 >> 7) & 0x1f));
 762                                                 e18 &= ~0x00000f80;
 763                                         } else {
 764                                                 NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Unknown CUDA fault at address %02x%08x\n",
 765                                                                 i, e14, e10);
 766                                         }
 767                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
 768                                                         i, e0c, e18, e1c, e20, e24);
 769                                 }
 770                                 ustatus &= ~0x00000080;
 771                         }
 772                         }
 773                         break;
 774                 }
 775                 if (ustatus) {
 776                         if (display)
 777                                 NV_INFO(dev, "%s - TP%d: Unhandled ustatus 0x%08x\n", name, i, ustatus);
 778                 }
 779                 nv_wr32(dev, ustatus_addr, 0xc0000000);
 780         }
 781
 782         if (!tps && display)
 783                 NV_INFO(dev, "%s - No TPs claiming errors?\n", name);
 784 }
 785
 786 static void
 787 nv50_pgraph_trap_handler(struct drm_device *dev)
 788 {
 789         struct nouveau_pgraph_trap trap;
 790         uint32_t status = nv_rd32(dev, 0x400108);
 791         uint32_t ustatus;
 792         int display = nouveau_ratelimit();
 793
 794
 795         if (!status && display) {
 796                 nouveau_graph_trap_info(dev, &trap);
 797                 nouveau_graph_dump_trap_info(dev, "PGRAPH_TRAP", &trap);
 798                 NV_INFO(dev, "PGRAPH_TRAP - no units reporting traps?\n");
 799         }
 800
 801         /* DISPATCH: Relays commands to other units and handles NOTIFY,
 802          * COND, QUERY. If you get a trap from it, the command is still stuck
 803          * in DISPATCH and you need to do something about it. */
 804         if (status & 0x001) {
 805                 ustatus = nv_rd32(dev, 0x400804) & 0x7fffffff;
 806                 if (!ustatus && display) {
 807                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH - no ustatus?\n");
 808                 }
 809
 810                 /* Known to be triggered by screwed up NOTIFY and COND... */
 811                 if (ustatus & 0x00000001) {
 812                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_DISPATCH_FAULT");
 813                         nv_wr32(dev, 0x400500, 0);
 814                         if (nv_rd32(dev, 0x400808) & 0x80000000) {
 815                                 if (display) {
 816                                         if (nouveau_graph_trapped_channel(dev, &trap.channel))
 817                                                 trap.channel = -1;
 818                                         trap.class = nv_rd32(dev, 0x400814);
 819                                         trap.mthd = nv_rd32(dev, 0x400808) & 0x1ffc;
 820                                         trap.subc = (nv_rd32(dev, 0x400808) >> 16) & 0x7;
 821                                         trap.data = nv_rd32(dev, 0x40080c);
 822                                         trap.data2 = nv_rd32(dev, 0x400810);
 823                                         nouveau_graph_dump_trap_info(dev,
 824                                                         "PGRAPH_TRAP_DISPATCH_FAULT", &trap);
 825                                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_FAULT - 400808: %08x\n", nv_rd32(dev, 0x400808));
 826                                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_FAULT - 400848: %08x\n", nv_rd32(dev, 0x400848));
 827                                 }
 828                                 nv_wr32(dev, 0x400808, 0);
 829                         } else if (display) {
 830                                 NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_FAULT - No stuck command?\n");
 831                         }
 832                         nv_wr32(dev, 0x4008e8, nv_rd32(dev, 0x4008e8) & 3);
 833                         nv_wr32(dev, 0x400848, 0);
 834                         ustatus &= ~0x00000001;
 835                 }
 836                 if (ustatus & 0x00000002) {
 837                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_DISPATCH_QUERY");
 838                         nv_wr32(dev, 0x400500, 0);
 839                         if (nv_rd32(dev, 0x40084c) & 0x80000000) {
 840                                 if (display) {
 841                                         if (nouveau_graph_trapped_channel(dev, &trap.channel))
 842                                                 trap.channel = -1;
 843                                         trap.class = nv_rd32(dev, 0x400814);
 844                                         trap.mthd = nv_rd32(dev, 0x40084c) & 0x1ffc;
 845                                         trap.subc = (nv_rd32(dev, 0x40084c) >> 16) & 0x7;
 846                                         trap.data = nv_rd32(dev, 0x40085c);
 847                                         trap.data2 = 0;
 848                                         nouveau_graph_dump_trap_info(dev,
 849                                                         "PGRAPH_TRAP_DISPATCH_QUERY", &trap);
 850                                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_QUERY - 40084c: %08x\n", nv_rd32(dev, 0x40084c));
 851                                 }
 852                                 nv_wr32(dev, 0x40084c, 0);
 853                         } else if (display) {
 854                                 NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_QUERY - No stuck command?\n");
 855                         }
 856                         ustatus &= ~0x00000002;
 857                 }
 858                 if (ustatus && display)
 859                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH - Unhandled ustatus 0x%08x\n", ustatus);
 860                 nv_wr32(dev, 0x400804, 0xc0000000);
 861                 nv_wr32(dev, 0x400108, 0x001);
 862                 status &= ~0x001;
 863         }
 864
 865         /* TRAPs other than dispatch use the "normal" trap regs. */
 866         if (status && display) {
 867                 nouveau_graph_trap_info(dev, &trap);
 868                 nouveau_graph_dump_trap_info(dev,
 869                                 "PGRAPH_TRAP", &trap);
 870         }
 871
 872         /* M2MF: Memory to memory copy engine. */
 873         if (status & 0x002) {
 874                 ustatus = nv_rd32(dev, 0x406800) & 0x7fffffff;
 875                 if (!ustatus && display) {
 876                         NV_INFO(dev, "PGRAPH_TRAP_M2MF - no ustatus?\n");
 877                 }
 878                 if (ustatus & 0x00000001) {
 879                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_M2MF_NOTIFY");
 880                         ustatus &= ~0x00000001;
 881                 }
 882                 if (ustatus & 0x00000002) {
 883                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_M2MF_IN");
 884                         ustatus &= ~0x00000002;
 885                 }
 886                 if (ustatus & 0x00000004) {
 887                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_M2MF_OUT");
 888                         ustatus &= ~0x00000004;
 889                 }
 890                 NV_INFO (dev, "PGRAPH_TRAP_M2MF - %08x %08x %08x %08x\n",
 891                                 nv_rd32(dev, 0x406804),
 892                                 nv_rd32(dev, 0x406808),
 893                                 nv_rd32(dev, 0x40680c),
 894                                 nv_rd32(dev, 0x406810));
 895                 if (ustatus && display)
 896                         NV_INFO(dev, "PGRAPH_TRAP_M2MF - Unhandled ustatus 0x%08x\n", ustatus);
 897                 /* No sane way found yet -- just reset the bugger. */
 898                 nv_wr32(dev, 0x400040, 2);
 899                 nv_wr32(dev, 0x400040, 0);
 900                 nv_wr32(dev, 0x406800, 0xc0000000);
 901                 nv_wr32(dev, 0x400108, 0x002);
 902                 status &= ~0x002;
 903         }
 904
 905         /* VFETCH: Fetches data from vertex buffers. */
 906         if (status & 0x004) {
 907                 ustatus = nv_rd32(dev, 0x400c04) & 0x7fffffff;
 908                 if (!ustatus && display) {
 909                         NV_INFO(dev, "PGRAPH_TRAP_VFETCH - no ustatus?\n");
 910                 }
 911                 if (ustatus & 0x00000001) {
 912                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_VFETCH_FAULT");
 913                         NV_INFO (dev, "PGRAPH_TRAP_VFETCH_FAULT - %08x %08x %08x %08x\n",
 914                                         nv_rd32(dev, 0x400c00),
 915                                         nv_rd32(dev, 0x400c08),
 916                                         nv_rd32(dev, 0x400c0c),
 917                                         nv_rd32(dev, 0x400c10));
 918                         ustatus &= ~0x00000001;
 919                 }
 920                 if (ustatus && display)
 921                         NV_INFO(dev, "PGRAPH_TRAP_VFETCH - Unhandled ustatus 0x%08x\n", ustatus);
 922                 nv_wr32(dev, 0x400c04, 0xc0000000);
 923                 nv_wr32(dev, 0x400108, 0x004);
 924                 status &= ~0x004;
 925         }
 926
 927         /* STRMOUT: DirectX streamout / OpenGL transform feedback. */
 928         if (status & 0x008) {
 929                 ustatus = nv_rd32(dev, 0x401800) & 0x7fffffff;
 930                 if (!ustatus && display) {
 931                         NV_INFO(dev, "PGRAPH_TRAP_STRMOUT - no ustatus?\n");
 932                 }
 933                 if (ustatus & 0x00000001) {
 934                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_STRMOUT_FAULT");
 935                         NV_INFO (dev, "PGRAPH_TRAP_STRMOUT_FAULT - %08x %08x %08x %08x\n",
 936                                         nv_rd32(dev, 0x401804),
 937                                         nv_rd32(dev, 0x401808),
 938                                         nv_rd32(dev, 0x40180c),
 939                                         nv_rd32(dev, 0x401810));
 940                         ustatus &= ~0x00000001;
 941                 }
 942                 if (ustatus && display)
 943                         NV_INFO(dev, "PGRAPH_TRAP_STRMOUT - Unhandled ustatus 0x%08x\n", ustatus);
 944                 /* No sane way found yet -- just reset the bugger. */
 945                 nv_wr32(dev, 0x400040, 0x80);
 946                 nv_wr32(dev, 0x400040, 0);
 947                 nv_wr32(dev, 0x401800, 0xc0000000);
 948                 nv_wr32(dev, 0x400108, 0x008);
 949                 status &= ~0x008;
 950         }
 951
 952         /* CCACHE: Handles code and c[] caches and fills them. */
 953         if (status & 0x010) {
 954                 ustatus = nv_rd32(dev, 0x405018) & 0x7fffffff;
 955                 if (!ustatus && display) {
 956                         NV_INFO(dev, "PGRAPH_TRAP_CCACHE - no ustatus?\n");
 957                 }
 958                 if (ustatus & 0x00000001) {
 959                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_CCACHE_FAULT");
 960                         NV_INFO (dev, "PGRAPH_TRAP_CCACHE_FAULT - %08x %08x %08x %08x %08x %08x %08x\n",
 961                                         nv_rd32(dev, 0x405800),
 962                                         nv_rd32(dev, 0x405804),
 963                                         nv_rd32(dev, 0x405808),
 964                                         nv_rd32(dev, 0x40580c),
 965                                         nv_rd32(dev, 0x405810),
 966                                         nv_rd32(dev, 0x405814),
 967                                         nv_rd32(dev, 0x40581c));
 968                         ustatus &= ~0x00000001;
 969                 }
 970                 if (ustatus && display)
 971                         NV_INFO(dev, "PGRAPH_TRAP_CCACHE - Unhandled ustatus 0x%08x\n", ustatus);
 972                 nv_wr32(dev, 0x405018, 0xc0000000);
 973                 nv_wr32(dev, 0x400108, 0x010);
 974                 status &= ~0x010;
 975         }
 976
 977         /* Unknown, not seen yet... 0x402000 is the only trap status reg
 978          * remaining, so try to handle it anyway. Perhaps related to that
 979          * unknown DMA slot on tesla? */
 980         if (status & 0x20) {
 981                 nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_UNKC04");
 982                 ustatus = nv_rd32(dev, 0x402000) & 0x7fffffff;
 983                 if (display)
 984                         NV_INFO(dev, "PGRAPH_TRAP_UNKC04 - Unhandled ustatus 0x%08x\n", ustatus);
 985                 nv_wr32(dev, 0x402000, 0xc0000000);
 986                 /* no status modifiction on purpose */
 987         }
 988
 989         /* TEXTURE: CUDA texturing units */
 990         if (status & 0x040) {
 991                 nv50_pgraph_tp_trap (dev, 6, 0x408900, 0x408600, display,
 992                                 "PGRAPH_TRAP_TEXTURE");
 993                 nv_wr32(dev, 0x400108, 0x040);
 994                 status &= ~0x040;
 995         }
 996
 997         /* MP: CUDA execution engines. */
 998         if (status & 0x080) {
 999                 nv50_pgraph_tp_trap (dev, 7, 0x408314, 0x40831c, display,
1000                                 "PGRAPH_TRAP_MP");
1001                 nv_wr32(dev, 0x400108, 0x080);
1002                 status &= ~0x080;
1003         }
1004
1005         /* TPDMA:  Handles TP-initiated uncached memory accesses:
1006          * l[], g[], stack, 2d surfaces, render targets. */
1007         if (status & 0x100) {
1008                 nv50_pgraph_tp_trap (dev, 8, 0x408e08, 0x408708, display,
1009                                 "PGRAPH_TRAP_TPDMA");
1010                 nv_wr32(dev, 0x400108, 0x100);
1011                 status &= ~0x100;
1012         }
1013
1014         if (status) {
1015                 if (display)
1016                         NV_INFO(dev, "PGRAPH_TRAP - Unknown trap 0x%08x\n",
1017                                 status);
1018                 nv_wr32(dev, 0x400108, status);
1019         }
1020 }
1021
1022 /* There must be a *lot* of these. Will take some time to gather them up. */
1023 static struct nouveau_enum_names nv50_data_error_names[] =
1024 {
1025         { 4,    "INVALID_VALUE" },
1026         { 5,    "INVALID_ENUM" },
1027         { 8,    "INVALID_OBJECT" },
1028         { 0xc,  "INVALID_BITFIELD" },
1029         { 0x28, "MP_NO_REG_SPACE" },
1030         { 0x2b, "MP_BLOCK_SIZE_MISMATCH" },
1031 };
1032
1033 static void
1034 nv50_pgraph_irq_handler(struct drm_device *dev)
1035 {
1036         struct nouveau_pgraph_trap trap;
1037         int unhandled = 0;
1038         uint32_t status;
1039
1040         while ((status = nv_rd32(dev, NV03_PGRAPH_INTR))) {
1041                 /* NOTIFY: You've set a NOTIFY an a command and it's done. */
1042                 if (status & 0x00000001) {
1043                         nouveau_graph_trap_info(dev, &trap);
1044                         if (nouveau_ratelimit())
1045                                 nouveau_graph_dump_trap_info(dev,
1046                                                 "PGRAPH_NOTIFY", &trap);
1047                         status &= ~0x00000001;
1048                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000001);
1049                 }
1050
1051                 /* COMPUTE_QUERY: Purpose and exact cause unknown, happens
1052                  * when you write 0x200 to 0x50c0 method 0x31c. */
1053                 if (status & 0x00000002) {
1054                         nouveau_graph_trap_info(dev, &trap);
1055                         if (nouveau_ratelimit())
1056                                 nouveau_graph_dump_trap_info(dev,
1057                                                 "PGRAPH_COMPUTE_QUERY", &trap);
1058                         status &= ~0x00000002;
1059                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000002);
1060                 }
1061
1062                 /* Unknown, never seen: 0x4 */
1063
1064                 /* ILLEGAL_MTHD: You used a wrong method for this class. */
1065                 if (status & 0x00000010) {
1066                         nouveau_graph_trap_info(dev, &trap);
1067                         if (nouveau_pgraph_intr_swmthd(dev, &trap))
1068                                 unhandled = 1;
1069                         if (unhandled && nouveau_ratelimit())
1070                                 nouveau_graph_dump_trap_info(dev,
1071                                                 "PGRAPH_ILLEGAL_MTHD", &trap);
1072                         status &= ~0x00000010;
1073                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000010);
1074                 }
1075
1076                 /* ILLEGAL_CLASS: You used a wrong class. */
1077                 if (status & 0x00000020) {
1078                         nouveau_graph_trap_info(dev, &trap);
1079                         if (nouveau_ratelimit())
1080                                 nouveau_graph_dump_trap_info(dev,
1081                                                 "PGRAPH_ILLEGAL_CLASS", &trap);
1082                         status &= ~0x00000020;
1083                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000020);
1084                 }
1085
1086                 /* DOUBLE_NOTIFY: You tried to set a NOTIFY on another NOTIFY. */
1087                 if (status & 0x00000040) {
1088                         nouveau_graph_trap_info(dev, &trap);
1089                         if (nouveau_ratelimit())
1090                                 nouveau_graph_dump_trap_info(dev,
1091                                                 "PGRAPH_DOUBLE_NOTIFY", &trap);
1092                         status &= ~0x00000040;
1093                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000040);
1094                 }
1095
1096                 /* CONTEXT_SWITCH: PGRAPH needs us to load a new context */
1097                 if (status & 0x00001000) {
1098                         nv_wr32(dev, 0x400500, 0x00000000);
1099                         nv_wr32(dev, NV03_PGRAPH_INTR,
1100                                 NV_PGRAPH_INTR_CONTEXT_SWITCH);
1101                         nv_wr32(dev, NV40_PGRAPH_INTR_EN, nv_rd32(dev,
1102                                 NV40_PGRAPH_INTR_EN) &
1103                                 ~NV_PGRAPH_INTR_CONTEXT_SWITCH);
1104                         nv_wr32(dev, 0x400500, 0x00010001);
1105
1106                         nv50_graph_context_switch(dev);
1107
1108                         status &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
1109                 }
1110
1111                 /* BUFFER_NOTIFY: Your m2mf transfer finished */
1112                 if (status & 0x00010000) {
1113                         nouveau_graph_trap_info(dev, &trap);
1114                         if (nouveau_ratelimit())
1115                                 nouveau_graph_dump_trap_info(dev,
1116                                                 "PGRAPH_BUFFER_NOTIFY", &trap);
1117                         status &= ~0x00010000;
1118                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00010000);
1119                 }
1120
1121                 /* DATA_ERROR: Invalid value for this method, or invalid
1122                  * state in current PGRAPH context for this operation */
1123                 if (status & 0x00100000) {
1124                         nouveau_graph_trap_info(dev, &trap);
1125                         if (nouveau_ratelimit()) {
1126                                 nouveau_graph_dump_trap_info(dev,
1127                                                 "PGRAPH_DATA_ERROR", &trap);
1128                                 NV_INFO (dev, "PGRAPH_DATA_ERROR - ");
1129                                 nouveau_print_enum_names(nv_rd32(dev, 0x400110),
1130                                                 nv50_data_error_names);
1131                                 printk("\n");
1132                         }
1133                         status &= ~0x00100000;
1134                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00100000);
1135                 }
1136
1137                 /* TRAP: Something bad happened in the middle of command
1138                  * execution.  Has a billion types, subtypes, and even
1139                  * subsubtypes. */
1140                 if (status & 0x00200000) {
1141                         nv50_pgraph_trap_handler(dev);
1142                         status &= ~0x00200000;
1143                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00200000);
1144                 }
1145
1146                 /* Unknown, never seen: 0x00400000 */
1147
1148                 /* SINGLE_STEP: Happens on every method if you turned on
1149                  * single stepping in 40008c */
1150                 if (status & 0x01000000) {
1151                         nouveau_graph_trap_info(dev, &trap);
1152                         if (nouveau_ratelimit())
1153                                 nouveau_graph_dump_trap_info(dev,
1154                                                 "PGRAPH_SINGLE_STEP", &trap);
1155                         status &= ~0x01000000;
1156                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x01000000);
1157                 }
1158
1159                 /* 0x02000000 happens when you pause a ctxprog...
1160                  * but the only way this can happen that I know is by
1161                  * poking the relevant MMIO register, and we don't
1162                  * do that. */
1163
1164                 if (status) {
1165                         NV_INFO(dev, "Unhandled PGRAPH_INTR - 0x%08x\n",
1166                                 status);
1167                         nv_wr32(dev, NV03_PGRAPH_INTR, status);
1168                 }
1169
1170                 {
1171                         const int isb = (1 << 16) | (1 << 0);
1172
1173                         if ((nv_rd32(dev, 0x400500) & isb) != isb)
1174                                 nv_wr32(dev, 0x400500,
1175                                         nv_rd32(dev, 0x400500) | isb);
1176                 }
1177         }
1178
1179         nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PGRAPH_PENDING);
1180         if (nv_rd32(dev, 0x400824) & (1 << 31))
1181                 nv_wr32(dev, 0x400824, nv_rd32(dev, 0x400824) & ~(1 << 31));
1182 }
1183
1184 static void
1185 nouveau_crtc_irq_handler(struct drm_device *dev, int crtc)
1186 {
1187         if (crtc & 1) {
1188                 nv_wr32(dev, NV_CRTC0_INTSTAT, NV_CRTC_INTR_VBLANK);
1189                 drm_handle_vblank(dev, 0);
1190         }
1191
1192         if (crtc & 2) {
1193                 nv_wr32(dev, NV_CRTC1_INTSTAT, NV_CRTC_INTR_VBLANK);
1194                 drm_handle_vblank(dev, 1);
1195         }
1196 }
1197
1198 irqreturn_t
1199 nouveau_irq_handler(DRM_IRQ_ARGS)
1200 {
1201         struct drm_device *dev = (struct drm_device *)arg;
1202         struct drm_nouveau_private *dev_priv = dev->dev_private;
1203         unsigned long flags;
1204         u32 status;
1205         int i;
1206
1207         status = nv_rd32(dev, NV03_PMC_INTR_0);
1208         if (!status)
1209                 return IRQ_NONE;
1210
1211         spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
1212
1213         if (status & NV_PMC_INTR_0_PFIFO_PENDING) {
1214                 nouveau_fifo_irq_handler(dev);
1215                 status &= ~NV_PMC_INTR_0_PFIFO_PENDING;
1216         }
1217
1218         if (status & NV_PMC_INTR_0_PGRAPH_PENDING) {
1219                 if (dev_priv->card_type >= NV_50)
1220                         nv50_pgraph_irq_handler(dev);
1221                 else
1222                         nouveau_pgraph_irq_handler(dev);
1223
1224                 status &= ~NV_PMC_INTR_0_PGRAPH_PENDING;
1225         }
1226
1227         if (status & NV_PMC_INTR_0_CRTCn_PENDING) {
1228                 nouveau_crtc_irq_handler(dev, (status>>24)&3);
1229                 status &= ~NV_PMC_INTR_0_CRTCn_PENDING;
1230         }
1231
1232         for (i = 0; i < 32 && status; i++) {
1233                 if (!(status & (1 << i)) || !dev_priv->irq_handler[i])
1234                         continue;
1235
1236                 dev_priv->irq_handler[i](dev);
1237                 status &= ~(1 << i);
1238         }
1239
1240         if (status)
1241                 NV_ERROR(dev, "Unhandled PMC INTR status bits 0x%08x\n", status);
1242
1243         spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
1244
1245         if (dev_priv->msi_enabled)
1246                 nv_wr08(dev, 0x00088068, 0xff);
1247
1248         return IRQ_HANDLED;
1249 }
1250
1251 int
1252 nouveau_irq_init(struct drm_device *dev)
1253 {
1254         struct drm_nouveau_private *dev_priv = dev->dev_private;
1255         int ret;
1256
1257         if (nouveau_msi != 0 && dev_priv->card_type >= NV_50) {
1258                 ret = pci_enable_msi(dev->pdev);
1259                 if (ret == 0) {
1260                         NV_INFO(dev, "enabled MSI\n");
1261                         dev_priv->msi_enabled = true;
1262                 }
1263         }
1264
1265         return drm_irq_install(dev);
1266 }
1267
1268 void
1269 nouveau_irq_fini(struct drm_device *dev)
1270 {
1271         struct drm_nouveau_private *dev_priv = dev->dev_private;
1272
1273         drm_irq_uninstall(dev);
1274         if (dev_priv->msi_enabled)
1275                 pci_disable_msi(dev->pdev);
1276 }
1277
1278 void
1279 nouveau_irq_register(struct drm_device *dev, int status_bit,
1280                      void (*handler)(struct drm_device *))
1281 {
1282         struct drm_nouveau_private *dev_priv = dev->dev_private;
1283         unsigned long flags;
1284
1285         spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
1286         dev_priv->irq_handler[status_bit] = handler;
1287         spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
1288 }
1289
1290 void
1291 nouveau_irq_unregister(struct drm_device *dev, int status_bit)
1292 {
1293         struct drm_nouveau_private *dev_priv = dev->dev_private;
1294         unsigned long flags;
1295
1296         spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
1297         dev_priv->irq_handler[status_bit] = NULL;
1298         spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
1299 }