drivers/gpu/drm/nouveau/nouveau_irq.c

   1 /*
   2  * Copyright (C) 2006 Ben Skeggs.
   3  *
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining
   7  * a copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sublicense, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial
  16  * portions of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  */
  27
  28 /*
  29  * Authors:
  30  *   Ben Skeggs <darktama@iinet.net.au>
  31  */
  32
  33 #include "drmP.h"
  34 #include "drm.h"
  35 #include "nouveau_drm.h"
  36 #include "nouveau_drv.h"
  37 #include "nouveau_reg.h"
  38 #include "nouveau_ramht.h"
  39 #include <linux/ratelimit.h>
  40
  41 /* needed for hotplug irq */
  42 #include "nouveau_connector.h"
  43 #include "nv50_display.h"
  44
  45 static DEFINE_RATELIMIT_STATE(nouveau_ratelimit_state, 3 * HZ, 20);
  46
  47 static int nouveau_ratelimit(void)
  48 {
  49         return __ratelimit(&nouveau_ratelimit_state);
  50 }
  51
  52 void
  53 nouveau_irq_preinstall(struct drm_device *dev)
  54 {
  55         struct drm_nouveau_private *dev_priv = dev->dev_private;
  56
  57         /* Master disable */
  58         nv_wr32(dev, NV03_PMC_INTR_EN_0, 0);
  59
  60         if (dev_priv->card_type >= NV_50) {
  61                 INIT_WORK(&dev_priv->irq_work, nv50_display_irq_handler_bh);
  62                 INIT_WORK(&dev_priv->hpd_work, nv50_display_irq_hotplug_bh);
  63                 spin_lock_init(&dev_priv->hpd_state.lock);
  64                 INIT_LIST_HEAD(&dev_priv->vbl_waiting);
  65         }
  66 }
  67
  68 int
  69 nouveau_irq_postinstall(struct drm_device *dev)
  70 {
  71         struct drm_nouveau_private *dev_priv = dev->dev_private;
  72
  73         /* Master enable */
  74         nv_wr32(dev, NV03_PMC_INTR_EN_0, NV_PMC_INTR_EN_0_MASTER_ENABLE);
  75         if (dev_priv->msi_enabled)
  76                 nv_wr08(dev, 0x00088068, 0xff);
  77
  78         return 0;
  79 }
  80
  81 void
  82 nouveau_irq_uninstall(struct drm_device *dev)
  83 {
  84         /* Master disable */
  85         nv_wr32(dev, NV03_PMC_INTR_EN_0, 0);
  86 }
  87
  88 static bool
  89 nouveau_fifo_swmthd(struct drm_device *dev, u32 chid, u32 addr, u32 data)
  90 {
  91         struct drm_nouveau_private *dev_priv = dev->dev_private;
  92         struct nouveau_channel *chan = NULL;
  93         struct nouveau_gpuobj *obj;
  94         unsigned long flags;
  95         const int subc = (addr >> 13) & 0x7;
  96         const int mthd = addr & 0x1ffc;
  97         bool handled = false;
  98         u32 engine;
  99
 100         spin_lock_irqsave(&dev_priv->channels.lock, flags);
 101         if (likely(chid >= 0 && chid < dev_priv->engine.fifo.channels))
 102                 chan = dev_priv->channels.ptr[chid];
 103         if (unlikely(!chan))
 104                 goto out;
 105
 106         switch (mthd) {
 107         case 0x0000: /* bind object to subchannel */
 108                 obj = nouveau_ramht_find(chan, data);
 109                 if (unlikely(!obj || obj->engine != NVOBJ_ENGINE_SW))
 110                         break;
 111
 112                 chan->sw_subchannel[subc] = obj->class;
 113                 engine = 0x0000000f << (subc * 4);
 114
 115                 nv_mask(dev, NV04_PFIFO_CACHE1_ENGINE, engine, 0x00000000);
 116                 handled = true;
 117                 break;
 118         default:
 119                 engine = nv_rd32(dev, NV04_PFIFO_CACHE1_ENGINE);
 120                 if (unlikely(((engine >> (subc * 4)) & 0xf) != 0))
 121                         break;
 122
 123                 if (!nouveau_gpuobj_mthd_call(chan, chan->sw_subchannel[subc],
 124                                               mthd, data))
 125                         handled = true;
 126                 break;
 127         }
 128
 129 out:
 130         spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
 131         return handled;
 132 }
 133
 134 static void
 135 nouveau_fifo_irq_handler(struct drm_device *dev)
 136 {
 137         struct drm_nouveau_private *dev_priv = dev->dev_private;
 138         struct nouveau_engine *engine = &dev_priv->engine;
 139         uint32_t status, reassign;
 140         int cnt = 0;
 141
 142         reassign = nv_rd32(dev, NV03_PFIFO_CACHES) & 1;
 143         while ((status = nv_rd32(dev, NV03_PFIFO_INTR_0)) && (cnt++ < 100)) {
 144                 uint32_t chid, get;
 145
 146                 nv_wr32(dev, NV03_PFIFO_CACHES, 0);
 147
 148                 chid = engine->fifo.channel_id(dev);
 149                 get  = nv_rd32(dev, NV03_PFIFO_CACHE1_GET);
 150
 151                 if (status & NV_PFIFO_INTR_CACHE_ERROR) {
 152                         uint32_t mthd, data;
 153                         int ptr;
 154
 155                         /* NV_PFIFO_CACHE1_GET actually goes to 0xffc before
 156                          * wrapping on my G80 chips, but CACHE1 isn't big
 157                          * enough for this much data.. Tests show that it
 158                          * wraps around to the start at GET=0x800.. No clue
 159                          * as to why..
 160                          */
 161                         ptr = (get & 0x7ff) >> 2;
 162
 163                         if (dev_priv->card_type < NV_40) {
 164                                 mthd = nv_rd32(dev,
 165                                         NV04_PFIFO_CACHE1_METHOD(ptr));
 166                                 data = nv_rd32(dev,
 167                                         NV04_PFIFO_CACHE1_DATA(ptr));
 168                         } else {
 169                                 mthd = nv_rd32(dev,
 170                                         NV40_PFIFO_CACHE1_METHOD(ptr));
 171                                 data = nv_rd32(dev,
 172                                         NV40_PFIFO_CACHE1_DATA(ptr));
 173                         }
 174
 175                         if (!nouveau_fifo_swmthd(dev, chid, mthd, data)) {
 176                                 NV_INFO(dev, "PFIFO_CACHE_ERROR - Ch %d/%d "
 177                                              "Mthd 0x%04x Data 0x%08x\n",
 178                                         chid, (mthd >> 13) & 7, mthd & 0x1ffc,
 179                                         data);
 180                         }
 181
 182                         nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUSH, 0);
 183                         nv_wr32(dev, NV03_PFIFO_INTR_0,
 184                                                 NV_PFIFO_INTR_CACHE_ERROR);
 185
 186                         nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0,
 187                                 nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH0) & ~1);
 188                         nv_wr32(dev, NV03_PFIFO_CACHE1_GET, get + 4);
 189                         nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0,
 190                                 nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH0) | 1);
 191                         nv_wr32(dev, NV04_PFIFO_CACHE1_HASH, 0);
 192
 193                         nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUSH,
 194                                 nv_rd32(dev, NV04_PFIFO_CACHE1_DMA_PUSH) | 1);
 195                         nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
 196
 197                         status &= ~NV_PFIFO_INTR_CACHE_ERROR;
 198                 }
 199
 200                 if (status & NV_PFIFO_INTR_DMA_PUSHER) {
 201                         u32 dma_get = nv_rd32(dev, 0x003244);
 202                         u32 dma_put = nv_rd32(dev, 0x003240);
 203                         u32 push = nv_rd32(dev, 0x003220);
 204                         u32 state = nv_rd32(dev, 0x003228);
 205
 206                         if (dev_priv->card_type == NV_50) {
 207                                 u32 ho_get = nv_rd32(dev, 0x003328);
 208                                 u32 ho_put = nv_rd32(dev, 0x003320);
 209                                 u32 ib_get = nv_rd32(dev, 0x003334);
 210                                 u32 ib_put = nv_rd32(dev, 0x003330);
 211
 212                                 if (nouveau_ratelimit())
 213                                         NV_INFO(dev, "PFIFO_DMA_PUSHER - Ch %d Get 0x%02x%08x "
 214                                              "Put 0x%02x%08x IbGet 0x%08x IbPut 0x%08x "
 215                                              "State 0x%08x Push 0x%08x\n",
 216                                                 chid, ho_get, dma_get, ho_put,
 217                                                 dma_put, ib_get, ib_put, state,
 218                                                 push);
 219
 220                                 /* METHOD_COUNT, in DMA_STATE on earlier chipsets */
 221                                 nv_wr32(dev, 0x003364, 0x00000000);
 222                                 if (dma_get != dma_put || ho_get != ho_put) {
 223                                         nv_wr32(dev, 0x003244, dma_put);
 224                                         nv_wr32(dev, 0x003328, ho_put);
 225                                 } else
 226                                 if (ib_get != ib_put) {
 227                                         nv_wr32(dev, 0x003334, ib_put);
 228                                 }
 229                         } else {
 230                                 NV_INFO(dev, "PFIFO_DMA_PUSHER - Ch %d Get 0x%08x "
 231                                              "Put 0x%08x State 0x%08x Push 0x%08x\n",
 232                                         chid, dma_get, dma_put, state, push);
 233
 234                                 if (dma_get != dma_put)
 235                                         nv_wr32(dev, 0x003244, dma_put);
 236                         }
 237
 238                         nv_wr32(dev, 0x003228, 0x00000000);
 239                         nv_wr32(dev, 0x003220, 0x00000001);
 240                         nv_wr32(dev, 0x002100, NV_PFIFO_INTR_DMA_PUSHER);
 241                         status &= ~NV_PFIFO_INTR_DMA_PUSHER;
 242                 }
 243
 244                 if (status & NV_PFIFO_INTR_SEMAPHORE) {
 245                         uint32_t sem;
 246
 247                         status &= ~NV_PFIFO_INTR_SEMAPHORE;
 248                         nv_wr32(dev, NV03_PFIFO_INTR_0,
 249                                 NV_PFIFO_INTR_SEMAPHORE);
 250
 251                         sem = nv_rd32(dev, NV10_PFIFO_CACHE1_SEMAPHORE);
 252                         nv_wr32(dev, NV10_PFIFO_CACHE1_SEMAPHORE, sem | 0x1);
 253
 254                         nv_wr32(dev, NV03_PFIFO_CACHE1_GET, get + 4);
 255                         nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
 256                 }
 257
 258                 if (dev_priv->card_type == NV_50) {
 259                         if (status & 0x00000010) {
 260                                 nv50_fb_vm_trap(dev, 1, "PFIFO_BAR_FAULT");
 261                                 status &= ~0x00000010;
 262                                 nv_wr32(dev, 0x002100, 0x00000010);
 263                         }
 264                 }
 265
 266                 if (status) {
 267                         if (nouveau_ratelimit())
 268                                 NV_INFO(dev, "PFIFO_INTR 0x%08x - Ch %d\n",
 269                                         status, chid);
 270                         nv_wr32(dev, NV03_PFIFO_INTR_0, status);
 271                         status = 0;
 272                 }
 273
 274                 nv_wr32(dev, NV03_PFIFO_CACHES, reassign);
 275         }
 276
 277         if (status) {
 278                 NV_INFO(dev, "PFIFO still angry after %d spins, halt\n", cnt);
 279                 nv_wr32(dev, 0x2140, 0);
 280                 nv_wr32(dev, 0x140, 0);
 281         }
 282
 283         nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PFIFO_PENDING);
 284 }
 285
 286 struct nouveau_bitfield_names {
 287         uint32_t mask;
 288         const char *name;
 289 };
 290
 291 static struct nouveau_bitfield_names nstatus_names[] =
 292 {
 293         { NV04_PGRAPH_NSTATUS_STATE_IN_USE,       "STATE_IN_USE" },
 294         { NV04_PGRAPH_NSTATUS_INVALID_STATE,      "INVALID_STATE" },
 295         { NV04_PGRAPH_NSTATUS_BAD_ARGUMENT,       "BAD_ARGUMENT" },
 296         { NV04_PGRAPH_NSTATUS_PROTECTION_FAULT,   "PROTECTION_FAULT" }
 297 };
 298
 299 static struct nouveau_bitfield_names nstatus_names_nv10[] =
 300 {
 301         { NV10_PGRAPH_NSTATUS_STATE_IN_USE,       "STATE_IN_USE" },
 302         { NV10_PGRAPH_NSTATUS_INVALID_STATE,      "INVALID_STATE" },
 303         { NV10_PGRAPH_NSTATUS_BAD_ARGUMENT,       "BAD_ARGUMENT" },
 304         { NV10_PGRAPH_NSTATUS_PROTECTION_FAULT,   "PROTECTION_FAULT" }
 305 };
 306
 307 static struct nouveau_bitfield_names nsource_names[] =
 308 {
 309         { NV03_PGRAPH_NSOURCE_NOTIFICATION,       "NOTIFICATION" },
 310         { NV03_PGRAPH_NSOURCE_DATA_ERROR,         "DATA_ERROR" },
 311         { NV03_PGRAPH_NSOURCE_PROTECTION_ERROR,   "PROTECTION_ERROR" },
 312         { NV03_PGRAPH_NSOURCE_RANGE_EXCEPTION,    "RANGE_EXCEPTION" },
 313         { NV03_PGRAPH_NSOURCE_LIMIT_COLOR,        "LIMIT_COLOR" },
 314         { NV03_PGRAPH_NSOURCE_LIMIT_ZETA,         "LIMIT_ZETA" },
 315         { NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD,       "ILLEGAL_MTHD" },
 316         { NV03_PGRAPH_NSOURCE_DMA_R_PROTECTION,   "DMA_R_PROTECTION" },
 317         { NV03_PGRAPH_NSOURCE_DMA_W_PROTECTION,   "DMA_W_PROTECTION" },
 318         { NV03_PGRAPH_NSOURCE_FORMAT_EXCEPTION,   "FORMAT_EXCEPTION" },
 319         { NV03_PGRAPH_NSOURCE_PATCH_EXCEPTION,    "PATCH_EXCEPTION" },
 320         { NV03_PGRAPH_NSOURCE_STATE_INVALID,      "STATE_INVALID" },
 321         { NV03_PGRAPH_NSOURCE_DOUBLE_NOTIFY,      "DOUBLE_NOTIFY" },
 322         { NV03_PGRAPH_NSOURCE_NOTIFY_IN_USE,      "NOTIFY_IN_USE" },
 323         { NV03_PGRAPH_NSOURCE_METHOD_CNT,         "METHOD_CNT" },
 324         { NV03_PGRAPH_NSOURCE_BFR_NOTIFICATION,   "BFR_NOTIFICATION" },
 325         { NV03_PGRAPH_NSOURCE_DMA_VTX_PROTECTION, "DMA_VTX_PROTECTION" },
 326         { NV03_PGRAPH_NSOURCE_DMA_WIDTH_A,        "DMA_WIDTH_A" },
 327         { NV03_PGRAPH_NSOURCE_DMA_WIDTH_B,        "DMA_WIDTH_B" },
 328 };
 329
 330 static void
 331 nouveau_print_bitfield_names_(uint32_t value,
 332                                 const struct nouveau_bitfield_names *namelist,
 333                                 const int namelist_len)
 334 {
 335         /*
 336          * Caller must have already printed the KERN_* log level for us.
 337          * Also the caller is responsible for adding the newline.
 338          */
 339         int i;
 340         for (i = 0; i < namelist_len; ++i) {
 341                 uint32_t mask = namelist[i].mask;
 342                 if (value & mask) {
 343                         printk(" %s", namelist[i].name);
 344                         value &= ~mask;
 345                 }
 346         }
 347         if (value)
 348                 printk(" (unknown bits 0x%08x)", value);
 349 }
 350 #define nouveau_print_bitfield_names(val, namelist) \
 351         nouveau_print_bitfield_names_((val), (namelist), ARRAY_SIZE(namelist))
 352
 353 struct nouveau_enum_names {
 354         uint32_t value;
 355         const char *name;
 356 };
 357
 358 static void
 359 nouveau_print_enum_names_(uint32_t value,
 360                                 const struct nouveau_enum_names *namelist,
 361                                 const int namelist_len)
 362 {
 363         /*
 364          * Caller must have already printed the KERN_* log level for us.
 365          * Also the caller is responsible for adding the newline.
 366          */
 367         int i;
 368         for (i = 0; i < namelist_len; ++i) {
 369                 if (value == namelist[i].value) {
 370                         printk("%s", namelist[i].name);
 371                         return;
 372                 }
 373         }
 374         printk("unknown value 0x%08x", value);
 375 }
 376 #define nouveau_print_enum_names(val, namelist) \
 377         nouveau_print_enum_names_((val), (namelist), ARRAY_SIZE(namelist))
 378
 379 static int
 380 nouveau_graph_chid_from_grctx(struct drm_device *dev)
 381 {
 382         struct drm_nouveau_private *dev_priv = dev->dev_private;
 383         struct nouveau_channel *chan;
 384         unsigned long flags;
 385         uint32_t inst;
 386         int i;
 387
 388         if (dev_priv->card_type < NV_40)
 389                 return dev_priv->engine.fifo.channels;
 390         else
 391         if (dev_priv->card_type < NV_50) {
 392                 inst = (nv_rd32(dev, 0x40032c) & 0xfffff) << 4;
 393
 394                 spin_lock_irqsave(&dev_priv->channels.lock, flags);
 395                 for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
 396                         chan = dev_priv->channels.ptr[i];
 397                         if (!chan || !chan->ramin_grctx)
 398                                 continue;
 399
 400                         if (inst == chan->ramin_grctx->pinst)
 401                                 break;
 402                 }
 403                 spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
 404         } else {
 405                 inst = (nv_rd32(dev, 0x40032c) & 0xfffff) << 12;
 406
 407                 spin_lock_irqsave(&dev_priv->channels.lock, flags);
 408                 for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
 409                         chan = dev_priv->channels.ptr[i];
 410                         if (!chan || !chan->ramin)
 411                                 continue;
 412
 413                         if (inst == chan->ramin->vinst)
 414                                 break;
 415                 }
 416                 spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
 417         }
 418
 419
 420         return i;
 421 }
 422
 423 static int
 424 nouveau_graph_trapped_channel(struct drm_device *dev, int *channel_ret)
 425 {
 426         struct drm_nouveau_private *dev_priv = dev->dev_private;
 427         struct nouveau_engine *engine = &dev_priv->engine;
 428         int channel;
 429
 430         if (dev_priv->card_type < NV_10)
 431                 channel = (nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR) >> 24) & 0xf;
 432         else
 433         if (dev_priv->card_type < NV_40)
 434                 channel = (nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR) >> 20) & 0x1f;
 435         else
 436                 channel = nouveau_graph_chid_from_grctx(dev);
 437
 438         if (channel >= engine->fifo.channels ||
 439             !dev_priv->channels.ptr[channel]) {
 440                 NV_ERROR(dev, "AIII, invalid/inactive channel id %d\n", channel);
 441                 return -EINVAL;
 442         }
 443
 444         *channel_ret = channel;
 445         return 0;
 446 }
 447
 448 struct nouveau_pgraph_trap {
 449         int channel;
 450         int class;
 451         int subc, mthd, size;
 452         uint32_t data, data2;
 453         uint32_t nsource, nstatus;
 454 };
 455
 456 static void
 457 nouveau_graph_trap_info(struct drm_device *dev,
 458                         struct nouveau_pgraph_trap *trap)
 459 {
 460         struct drm_nouveau_private *dev_priv = dev->dev_private;
 461         uint32_t address;
 462
 463         trap->nsource = trap->nstatus = 0;
 464         if (dev_priv->card_type < NV_50) {
 465                 trap->nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
 466                 trap->nstatus = nv_rd32(dev, NV03_PGRAPH_NSTATUS);
 467         }
 468
 469         if (nouveau_graph_trapped_channel(dev, &trap->channel))
 470                 trap->channel = -1;
 471         address = nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR);
 472
 473         trap->mthd = address & 0x1FFC;
 474         trap->data = nv_rd32(dev, NV04_PGRAPH_TRAPPED_DATA);
 475         if (dev_priv->card_type < NV_10) {
 476                 trap->subc  = (address >> 13) & 0x7;
 477         } else {
 478                 trap->subc  = (address >> 16) & 0x7;
 479                 trap->data2 = nv_rd32(dev, NV10_PGRAPH_TRAPPED_DATA_HIGH);
 480         }
 481
 482         if (dev_priv->card_type < NV_10)
 483                 trap->class = nv_rd32(dev, 0x400180 + trap->subc*4) & 0xFF;
 484         else if (dev_priv->card_type < NV_40)
 485                 trap->class = nv_rd32(dev, 0x400160 + trap->subc*4) & 0xFFF;
 486         else if (dev_priv->card_type < NV_50)
 487                 trap->class = nv_rd32(dev, 0x400160 + trap->subc*4) & 0xFFFF;
 488         else
 489                 trap->class = nv_rd32(dev, 0x400814);
 490 }
 491
 492 static void
 493 nouveau_graph_dump_trap_info(struct drm_device *dev, const char *id,
 494                              struct nouveau_pgraph_trap *trap)
 495 {
 496         struct drm_nouveau_private *dev_priv = dev->dev_private;
 497         uint32_t nsource = trap->nsource, nstatus = trap->nstatus;
 498
 499         if (dev_priv->card_type < NV_50) {
 500                 NV_INFO(dev, "%s - nSource:", id);
 501                 nouveau_print_bitfield_names(nsource, nsource_names);
 502                 printk(", nStatus:");
 503                 if (dev_priv->card_type < NV_10)
 504                         nouveau_print_bitfield_names(nstatus, nstatus_names);
 505                 else
 506                         nouveau_print_bitfield_names(nstatus, nstatus_names_nv10);
 507                 printk("\n");
 508         }
 509
 510         NV_INFO(dev, "%s - Ch %d/%d Class 0x%04x Mthd 0x%04x "
 511                                         "Data 0x%08x:0x%08x\n",
 512                                         id, trap->channel, trap->subc,
 513                                         trap->class, trap->mthd,
 514                                         trap->data2, trap->data);
 515 }
 516
 517 static int
 518 nouveau_pgraph_intr_swmthd(struct drm_device *dev,
 519                            struct nouveau_pgraph_trap *trap)
 520 {
 521         struct drm_nouveau_private *dev_priv = dev->dev_private;
 522         struct nouveau_channel *chan;
 523         unsigned long flags;
 524         int ret = -EINVAL;
 525
 526         spin_lock_irqsave(&dev_priv->channels.lock, flags);
 527         if (trap->channel > 0 &&
 528             trap->channel < dev_priv->engine.fifo.channels &&
 529             dev_priv->channels.ptr[trap->channel]) {
 530                 chan = dev_priv->channels.ptr[trap->channel];
 531                 ret = nouveau_gpuobj_mthd_call(chan, trap->class, trap->mthd, trap->data);
 532         }
 533         spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
 534
 535         return ret;
 536 }
 537
 538 static inline void
 539 nouveau_pgraph_intr_notify(struct drm_device *dev, uint32_t nsource)
 540 {
 541         struct nouveau_pgraph_trap trap;
 542         int unhandled = 0;
 543
 544         nouveau_graph_trap_info(dev, &trap);
 545
 546         if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
 547                 if (nouveau_pgraph_intr_swmthd(dev, &trap))
 548                         unhandled = 1;
 549         } else {
 550                 unhandled = 1;
 551         }
 552
 553         if (unhandled)
 554                 nouveau_graph_dump_trap_info(dev, "PGRAPH_NOTIFY", &trap);
 555 }
 556
 557
 558 static inline void
 559 nouveau_pgraph_intr_error(struct drm_device *dev, uint32_t nsource)
 560 {
 561         struct nouveau_pgraph_trap trap;
 562         int unhandled = 0;
 563
 564         nouveau_graph_trap_info(dev, &trap);
 565         trap.nsource = nsource;
 566
 567         if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
 568                 if (nouveau_pgraph_intr_swmthd(dev, &trap))
 569                         unhandled = 1;
 570         } else if (nsource & NV03_PGRAPH_NSOURCE_DMA_VTX_PROTECTION) {
 571                 uint32_t v = nv_rd32(dev, 0x402000);
 572                 nv_wr32(dev, 0x402000, v);
 573
 574                 /* dump the error anyway for now: it's useful for
 575                    Gallium development */
 576                 unhandled = 1;
 577         } else {
 578                 unhandled = 1;
 579         }
 580
 581         if (unhandled && nouveau_ratelimit())
 582                 nouveau_graph_dump_trap_info(dev, "PGRAPH_ERROR", &trap);
 583 }
 584
 585 static inline void
 586 nouveau_pgraph_intr_context_switch(struct drm_device *dev)
 587 {
 588         struct drm_nouveau_private *dev_priv = dev->dev_private;
 589         struct nouveau_engine *engine = &dev_priv->engine;
 590         uint32_t chid;
 591
 592         chid = engine->fifo.channel_id(dev);
 593         NV_DEBUG(dev, "PGRAPH context switch interrupt channel %x\n", chid);
 594
 595         switch (dev_priv->card_type) {
 596         case NV_04:
 597                 nv04_graph_context_switch(dev);
 598                 break;
 599         case NV_10:
 600                 nv10_graph_context_switch(dev);
 601                 break;
 602         default:
 603                 NV_ERROR(dev, "Context switch not implemented\n");
 604                 break;
 605         }
 606 }
 607
 608 static void
 609 nouveau_pgraph_irq_handler(struct drm_device *dev)
 610 {
 611         uint32_t status;
 612
 613         while ((status = nv_rd32(dev, NV03_PGRAPH_INTR))) {
 614                 uint32_t nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
 615
 616                 if (status & NV_PGRAPH_INTR_NOTIFY) {
 617                         nouveau_pgraph_intr_notify(dev, nsource);
 618
 619                         status &= ~NV_PGRAPH_INTR_NOTIFY;
 620                         nv_wr32(dev, NV03_PGRAPH_INTR, NV_PGRAPH_INTR_NOTIFY);
 621                 }
 622
 623                 if (status & NV_PGRAPH_INTR_ERROR) {
 624                         nouveau_pgraph_intr_error(dev, nsource);
 625
 626                         status &= ~NV_PGRAPH_INTR_ERROR;
 627                         nv_wr32(dev, NV03_PGRAPH_INTR, NV_PGRAPH_INTR_ERROR);
 628                 }
 629
 630                 if (status & NV_PGRAPH_INTR_CONTEXT_SWITCH) {
 631                         status &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
 632                         nv_wr32(dev, NV03_PGRAPH_INTR,
 633                                  NV_PGRAPH_INTR_CONTEXT_SWITCH);
 634
 635                         nouveau_pgraph_intr_context_switch(dev);
 636                 }
 637
 638                 if (status) {
 639                         NV_INFO(dev, "Unhandled PGRAPH_INTR - 0x%08x\n", status);
 640                         nv_wr32(dev, NV03_PGRAPH_INTR, status);
 641                 }
 642
 643                 if ((nv_rd32(dev, NV04_PGRAPH_FIFO) & (1 << 0)) == 0)
 644                         nv_wr32(dev, NV04_PGRAPH_FIFO, 1);
 645         }
 646
 647         nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PGRAPH_PENDING);
 648 }
 649
 650 static struct nouveau_enum_names nv50_mp_exec_error_names[] =
 651 {
 652         { 3, "STACK_UNDERFLOW" },
 653         { 4, "QUADON_ACTIVE" },
 654         { 8, "TIMEOUT" },
 655         { 0x10, "INVALID_OPCODE" },
 656         { 0x40, "BREAKPOINT" },
 657 };
 658
 659 static void
 660 nv50_pgraph_mp_trap(struct drm_device *dev, int tpid, int display)
 661 {
 662         struct drm_nouveau_private *dev_priv = dev->dev_private;
 663         uint32_t units = nv_rd32(dev, 0x1540);
 664         uint32_t addr, mp10, status, pc, oplow, ophigh;
 665         int i;
 666         int mps = 0;
 667         for (i = 0; i < 4; i++) {
 668                 if (!(units & 1 << (i+24)))
 669                         continue;
 670                 if (dev_priv->chipset < 0xa0)
 671                         addr = 0x408200 + (tpid << 12) + (i << 7);
 672                 else
 673                         addr = 0x408100 + (tpid << 11) + (i << 7);
 674                 mp10 = nv_rd32(dev, addr + 0x10);
 675                 status = nv_rd32(dev, addr + 0x14);
 676                 if (!status)
 677                         continue;
 678                 if (display) {
 679                         nv_rd32(dev, addr + 0x20);
 680                         pc = nv_rd32(dev, addr + 0x24);
 681                         oplow = nv_rd32(dev, addr + 0x70);
 682                         ophigh= nv_rd32(dev, addr + 0x74);
 683                         NV_INFO(dev, "PGRAPH_TRAP_MP_EXEC - "
 684                                         "TP %d MP %d: ", tpid, i);
 685                         nouveau_print_enum_names(status,
 686                                         nv50_mp_exec_error_names);
 687                         printk(" at %06x warp %d, opcode %08x %08x\n",
 688                                         pc&0xffffff, pc >> 24,
 689                                         oplow, ophigh);
 690                 }
 691                 nv_wr32(dev, addr + 0x10, mp10);
 692                 nv_wr32(dev, addr + 0x14, 0);
 693                 mps++;
 694         }
 695         if (!mps && display)
 696                 NV_INFO(dev, "PGRAPH_TRAP_MP_EXEC - TP %d: "
 697                                 "No MPs claiming errors?\n", tpid);
 698 }
 699
 700 static void
 701 nv50_pgraph_tp_trap(struct drm_device *dev, int type, uint32_t ustatus_old,
 702                 uint32_t ustatus_new, int display, const char *name)
 703 {
 704         struct drm_nouveau_private *dev_priv = dev->dev_private;
 705         int tps = 0;
 706         uint32_t units = nv_rd32(dev, 0x1540);
 707         int i, r;
 708         uint32_t ustatus_addr, ustatus;
 709         for (i = 0; i < 16; i++) {
 710                 if (!(units & (1 << i)))
 711                         continue;
 712                 if (dev_priv->chipset < 0xa0)
 713                         ustatus_addr = ustatus_old + (i << 12);
 714                 else
 715                         ustatus_addr = ustatus_new + (i << 11);
 716                 ustatus = nv_rd32(dev, ustatus_addr) & 0x7fffffff;
 717                 if (!ustatus)
 718                         continue;
 719                 tps++;
 720                 switch (type) {
 721                 case 6: /* texture error... unknown for now */
 722                         nv50_fb_vm_trap(dev, display, name);
 723                         if (display) {
 724                                 NV_ERROR(dev, "magic set %d:\n", i);
 725                                 for (r = ustatus_addr + 4; r <= ustatus_addr + 0x10; r += 4)
 726                                         NV_ERROR(dev, "\t0x%08x: 0x%08x\n", r,
 727                                                 nv_rd32(dev, r));
 728                         }
 729                         break;
 730                 case 7: /* MP error */
 731                         if (ustatus & 0x00010000) {
 732                                 nv50_pgraph_mp_trap(dev, i, display);
 733                                 ustatus &= ~0x00010000;
 734                         }
 735                         break;
 736                 case 8: /* TPDMA error */
 737                         {
 738                         uint32_t e0c = nv_rd32(dev, ustatus_addr + 4);
 739                         uint32_t e10 = nv_rd32(dev, ustatus_addr + 8);
 740                         uint32_t e14 = nv_rd32(dev, ustatus_addr + 0xc);
 741                         uint32_t e18 = nv_rd32(dev, ustatus_addr + 0x10);
 742                         uint32_t e1c = nv_rd32(dev, ustatus_addr + 0x14);
 743                         uint32_t e20 = nv_rd32(dev, ustatus_addr + 0x18);
 744                         uint32_t e24 = nv_rd32(dev, ustatus_addr + 0x1c);
 745                         nv50_fb_vm_trap(dev, display, name);
 746                         /* 2d engine destination */
 747                         if (ustatus & 0x00000010) {
 748                                 if (display) {
 749                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA_2D - TP %d - Unknown fault at address %02x%08x\n",
 750                                                         i, e14, e10);
 751                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA_2D - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
 752                                                         i, e0c, e18, e1c, e20, e24);
 753                                 }
 754                                 ustatus &= ~0x00000010;
 755                         }
 756                         /* Render target */
 757                         if (ustatus & 0x00000040) {
 758                                 if (display) {
 759                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA_RT - TP %d - Unknown fault at address %02x%08x\n",
 760                                                         i, e14, e10);
 761                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA_RT - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
 762                                                         i, e0c, e18, e1c, e20, e24);
 763                                 }
 764                                 ustatus &= ~0x00000040;
 765                         }
 766                         /* CUDA memory: l[], g[] or stack. */
 767                         if (ustatus & 0x00000080) {
 768                                 if (display) {
 769                                         if (e18 & 0x80000000) {
 770                                                 /* g[] read fault? */
 771                                                 NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Global read fault at address %02x%08x\n",
 772                                                                 i, e14, e10 | ((e18 >> 24) & 0x1f));
 773                                                 e18 &= ~0x1f000000;
 774                                         } else if (e18 & 0xc) {
 775                                                 /* g[] write fault? */
 776                                                 NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Global write fault at address %02x%08x\n",
 777                                                                 i, e14, e10 | ((e18 >> 7) & 0x1f));
 778                                                 e18 &= ~0x00000f80;
 779                                         } else {
 780                                                 NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Unknown CUDA fault at address %02x%08x\n",
 781                                                                 i, e14, e10);
 782                                         }
 783                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
 784                                                         i, e0c, e18, e1c, e20, e24);
 785                                 }
 786                                 ustatus &= ~0x00000080;
 787                         }
 788                         }
 789                         break;
 790                 }
 791                 if (ustatus) {
 792                         if (display)
 793                                 NV_INFO(dev, "%s - TP%d: Unhandled ustatus 0x%08x\n", name, i, ustatus);
 794                 }
 795                 nv_wr32(dev, ustatus_addr, 0xc0000000);
 796         }
 797
 798         if (!tps && display)
 799                 NV_INFO(dev, "%s - No TPs claiming errors?\n", name);
 800 }
 801
 802 static void
 803 nv50_pgraph_trap_handler(struct drm_device *dev)
 804 {
 805         struct nouveau_pgraph_trap trap;
 806         uint32_t status = nv_rd32(dev, 0x400108);
 807         uint32_t ustatus;
 808         int display = nouveau_ratelimit();
 809
 810
 811         if (!status && display) {
 812                 nouveau_graph_trap_info(dev, &trap);
 813                 nouveau_graph_dump_trap_info(dev, "PGRAPH_TRAP", &trap);
 814                 NV_INFO(dev, "PGRAPH_TRAP - no units reporting traps?\n");
 815         }
 816
 817         /* DISPATCH: Relays commands to other units and handles NOTIFY,
 818          * COND, QUERY. If you get a trap from it, the command is still stuck
 819          * in DISPATCH and you need to do something about it. */
 820         if (status & 0x001) {
 821                 ustatus = nv_rd32(dev, 0x400804) & 0x7fffffff;
 822                 if (!ustatus && display) {
 823                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH - no ustatus?\n");
 824                 }
 825
 826                 /* Known to be triggered by screwed up NOTIFY and COND... */
 827                 if (ustatus & 0x00000001) {
 828                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_DISPATCH_FAULT");
 829                         nv_wr32(dev, 0x400500, 0);
 830                         if (nv_rd32(dev, 0x400808) & 0x80000000) {
 831                                 if (display) {
 832                                         if (nouveau_graph_trapped_channel(dev, &trap.channel))
 833                                                 trap.channel = -1;
 834                                         trap.class = nv_rd32(dev, 0x400814);
 835                                         trap.mthd = nv_rd32(dev, 0x400808) & 0x1ffc;
 836                                         trap.subc = (nv_rd32(dev, 0x400808) >> 16) & 0x7;
 837                                         trap.data = nv_rd32(dev, 0x40080c);
 838                                         trap.data2 = nv_rd32(dev, 0x400810);
 839                                         nouveau_graph_dump_trap_info(dev,
 840                                                         "PGRAPH_TRAP_DISPATCH_FAULT", &trap);
 841                                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_FAULT - 400808: %08x\n", nv_rd32(dev, 0x400808));
 842                                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_FAULT - 400848: %08x\n", nv_rd32(dev, 0x400848));
 843                                 }
 844                                 nv_wr32(dev, 0x400808, 0);
 845                         } else if (display) {
 846                                 NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_FAULT - No stuck command?\n");
 847                         }
 848                         nv_wr32(dev, 0x4008e8, nv_rd32(dev, 0x4008e8) & 3);
 849                         nv_wr32(dev, 0x400848, 0);
 850                         ustatus &= ~0x00000001;
 851                 }
 852                 if (ustatus & 0x00000002) {
 853                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_DISPATCH_QUERY");
 854                         nv_wr32(dev, 0x400500, 0);
 855                         if (nv_rd32(dev, 0x40084c) & 0x80000000) {
 856                                 if (display) {
 857                                         if (nouveau_graph_trapped_channel(dev, &trap.channel))
 858                                                 trap.channel = -1;
 859                                         trap.class = nv_rd32(dev, 0x400814);
 860                                         trap.mthd = nv_rd32(dev, 0x40084c) & 0x1ffc;
 861                                         trap.subc = (nv_rd32(dev, 0x40084c) >> 16) & 0x7;
 862                                         trap.data = nv_rd32(dev, 0x40085c);
 863                                         trap.data2 = 0;
 864                                         nouveau_graph_dump_trap_info(dev,
 865                                                         "PGRAPH_TRAP_DISPATCH_QUERY", &trap);
 866                                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_QUERY - 40084c: %08x\n", nv_rd32(dev, 0x40084c));
 867                                 }
 868                                 nv_wr32(dev, 0x40084c, 0);
 869                         } else if (display) {
 870                                 NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_QUERY - No stuck command?\n");
 871                         }
 872                         ustatus &= ~0x00000002;
 873                 }
 874                 if (ustatus && display)
 875                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH - Unhandled ustatus 0x%08x\n", ustatus);
 876                 nv_wr32(dev, 0x400804, 0xc0000000);
 877                 nv_wr32(dev, 0x400108, 0x001);
 878                 status &= ~0x001;
 879         }
 880
 881         /* TRAPs other than dispatch use the "normal" trap regs. */
 882         if (status && display) {
 883                 nouveau_graph_trap_info(dev, &trap);
 884                 nouveau_graph_dump_trap_info(dev,
 885                                 "PGRAPH_TRAP", &trap);
 886         }
 887
 888         /* M2MF: Memory to memory copy engine. */
 889         if (status & 0x002) {
 890                 ustatus = nv_rd32(dev, 0x406800) & 0x7fffffff;
 891                 if (!ustatus && display) {
 892                         NV_INFO(dev, "PGRAPH_TRAP_M2MF - no ustatus?\n");
 893                 }
 894                 if (ustatus & 0x00000001) {
 895                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_M2MF_NOTIFY");
 896                         ustatus &= ~0x00000001;
 897                 }
 898                 if (ustatus & 0x00000002) {
 899                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_M2MF_IN");
 900                         ustatus &= ~0x00000002;
 901                 }
 902                 if (ustatus & 0x00000004) {
 903                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_M2MF_OUT");
 904                         ustatus &= ~0x00000004;
 905                 }
 906                 NV_INFO (dev, "PGRAPH_TRAP_M2MF - %08x %08x %08x %08x\n",
 907                                 nv_rd32(dev, 0x406804),
 908                                 nv_rd32(dev, 0x406808),
 909                                 nv_rd32(dev, 0x40680c),
 910                                 nv_rd32(dev, 0x406810));
 911                 if (ustatus && display)
 912                         NV_INFO(dev, "PGRAPH_TRAP_M2MF - Unhandled ustatus 0x%08x\n", ustatus);
 913                 /* No sane way found yet -- just reset the bugger. */
 914                 nv_wr32(dev, 0x400040, 2);
 915                 nv_wr32(dev, 0x400040, 0);
 916                 nv_wr32(dev, 0x406800, 0xc0000000);
 917                 nv_wr32(dev, 0x400108, 0x002);
 918                 status &= ~0x002;
 919         }
 920
 921         /* VFETCH: Fetches data from vertex buffers. */
 922         if (status & 0x004) {
 923                 ustatus = nv_rd32(dev, 0x400c04) & 0x7fffffff;
 924                 if (!ustatus && display) {
 925                         NV_INFO(dev, "PGRAPH_TRAP_VFETCH - no ustatus?\n");
 926                 }
 927                 if (ustatus & 0x00000001) {
 928                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_VFETCH_FAULT");
 929                         NV_INFO (dev, "PGRAPH_TRAP_VFETCH_FAULT - %08x %08x %08x %08x\n",
 930                                         nv_rd32(dev, 0x400c00),
 931                                         nv_rd32(dev, 0x400c08),
 932                                         nv_rd32(dev, 0x400c0c),
 933                                         nv_rd32(dev, 0x400c10));
 934                         ustatus &= ~0x00000001;
 935                 }
 936                 if (ustatus && display)
 937                         NV_INFO(dev, "PGRAPH_TRAP_VFETCH - Unhandled ustatus 0x%08x\n", ustatus);
 938                 nv_wr32(dev, 0x400c04, 0xc0000000);
 939                 nv_wr32(dev, 0x400108, 0x004);
 940                 status &= ~0x004;
 941         }
 942
 943         /* STRMOUT: DirectX streamout / OpenGL transform feedback. */
 944         if (status & 0x008) {
 945                 ustatus = nv_rd32(dev, 0x401800) & 0x7fffffff;
 946                 if (!ustatus && display) {
 947                         NV_INFO(dev, "PGRAPH_TRAP_STRMOUT - no ustatus?\n");
 948                 }
 949                 if (ustatus & 0x00000001) {
 950                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_STRMOUT_FAULT");
 951                         NV_INFO (dev, "PGRAPH_TRAP_STRMOUT_FAULT - %08x %08x %08x %08x\n",
 952                                         nv_rd32(dev, 0x401804),
 953                                         nv_rd32(dev, 0x401808),
 954                                         nv_rd32(dev, 0x40180c),
 955                                         nv_rd32(dev, 0x401810));
 956                         ustatus &= ~0x00000001;
 957                 }
 958                 if (ustatus && display)
 959                         NV_INFO(dev, "PGRAPH_TRAP_STRMOUT - Unhandled ustatus 0x%08x\n", ustatus);
 960                 /* No sane way found yet -- just reset the bugger. */
 961                 nv_wr32(dev, 0x400040, 0x80);
 962                 nv_wr32(dev, 0x400040, 0);
 963                 nv_wr32(dev, 0x401800, 0xc0000000);
 964                 nv_wr32(dev, 0x400108, 0x008);
 965                 status &= ~0x008;
 966         }
 967
 968         /* CCACHE: Handles code and c[] caches and fills them. */
 969         if (status & 0x010) {
 970                 ustatus = nv_rd32(dev, 0x405018) & 0x7fffffff;
 971                 if (!ustatus && display) {
 972                         NV_INFO(dev, "PGRAPH_TRAP_CCACHE - no ustatus?\n");
 973                 }
 974                 if (ustatus & 0x00000001) {
 975                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_CCACHE_FAULT");
 976                         NV_INFO (dev, "PGRAPH_TRAP_CCACHE_FAULT - %08x %08x %08x %08x %08x %08x %08x\n",
 977                                         nv_rd32(dev, 0x405800),
 978                                         nv_rd32(dev, 0x405804),
 979                                         nv_rd32(dev, 0x405808),
 980                                         nv_rd32(dev, 0x40580c),
 981                                         nv_rd32(dev, 0x405810),
 982                                         nv_rd32(dev, 0x405814),
 983                                         nv_rd32(dev, 0x40581c));
 984                         ustatus &= ~0x00000001;
 985                 }
 986                 if (ustatus && display)
 987                         NV_INFO(dev, "PGRAPH_TRAP_CCACHE - Unhandled ustatus 0x%08x\n", ustatus);
 988                 nv_wr32(dev, 0x405018, 0xc0000000);
 989                 nv_wr32(dev, 0x400108, 0x010);
 990                 status &= ~0x010;
 991         }
 992
 993         /* Unknown, not seen yet... 0x402000 is the only trap status reg
 994          * remaining, so try to handle it anyway. Perhaps related to that
 995          * unknown DMA slot on tesla? */
 996         if (status & 0x20) {
 997                 nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_UNKC04");
 998                 ustatus = nv_rd32(dev, 0x402000) & 0x7fffffff;
 999                 if (display)
1000                         NV_INFO(dev, "PGRAPH_TRAP_UNKC04 - Unhandled ustatus 0x%08x\n", ustatus);
1001                 nv_wr32(dev, 0x402000, 0xc0000000);
1002                 /* no status modifiction on purpose */
1003         }
1004
1005         /* TEXTURE: CUDA texturing units */
1006         if (status & 0x040) {
1007                 nv50_pgraph_tp_trap (dev, 6, 0x408900, 0x408600, display,
1008                                 "PGRAPH_TRAP_TEXTURE");
1009                 nv_wr32(dev, 0x400108, 0x040);
1010                 status &= ~0x040;
1011         }
1012
1013         /* MP: CUDA execution engines. */
1014         if (status & 0x080) {
1015                 nv50_pgraph_tp_trap (dev, 7, 0x408314, 0x40831c, display,
1016                                 "PGRAPH_TRAP_MP");
1017                 nv_wr32(dev, 0x400108, 0x080);
1018                 status &= ~0x080;
1019         }
1020
1021         /* TPDMA:  Handles TP-initiated uncached memory accesses:
1022          * l[], g[], stack, 2d surfaces, render targets. */
1023         if (status & 0x100) {
1024                 nv50_pgraph_tp_trap (dev, 8, 0x408e08, 0x408708, display,
1025                                 "PGRAPH_TRAP_TPDMA");
1026                 nv_wr32(dev, 0x400108, 0x100);
1027                 status &= ~0x100;
1028         }
1029
1030         if (status) {
1031                 if (display)
1032                         NV_INFO(dev, "PGRAPH_TRAP - Unknown trap 0x%08x\n",
1033                                 status);
1034                 nv_wr32(dev, 0x400108, status);
1035         }
1036 }
1037
1038 /* There must be a *lot* of these. Will take some time to gather them up. */
1039 static struct nouveau_enum_names nv50_data_error_names[] =
1040 {
1041         { 4,    "INVALID_VALUE" },
1042         { 5,    "INVALID_ENUM" },
1043         { 8,    "INVALID_OBJECT" },
1044         { 0xc,  "INVALID_BITFIELD" },
1045         { 0x28, "MP_NO_REG_SPACE" },
1046         { 0x2b, "MP_BLOCK_SIZE_MISMATCH" },
1047 };
1048
1049 static void
1050 nv50_pgraph_irq_handler(struct drm_device *dev)
1051 {
1052         struct nouveau_pgraph_trap trap;
1053         int unhandled = 0;
1054         uint32_t status;
1055
1056         while ((status = nv_rd32(dev, NV03_PGRAPH_INTR))) {
1057                 /* NOTIFY: You've set a NOTIFY an a command and it's done. */
1058                 if (status & 0x00000001) {
1059                         nouveau_graph_trap_info(dev, &trap);
1060                         if (nouveau_ratelimit())
1061                                 nouveau_graph_dump_trap_info(dev,
1062                                                 "PGRAPH_NOTIFY", &trap);
1063                         status &= ~0x00000001;
1064                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000001);
1065                 }
1066
1067                 /* COMPUTE_QUERY: Purpose and exact cause unknown, happens
1068                  * when you write 0x200 to 0x50c0 method 0x31c. */
1069                 if (status & 0x00000002) {
1070                         nouveau_graph_trap_info(dev, &trap);
1071                         if (nouveau_ratelimit())
1072                                 nouveau_graph_dump_trap_info(dev,
1073                                                 "PGRAPH_COMPUTE_QUERY", &trap);
1074                         status &= ~0x00000002;
1075                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000002);
1076                 }
1077
1078                 /* Unknown, never seen: 0x4 */
1079
1080                 /* ILLEGAL_MTHD: You used a wrong method for this class. */
1081                 if (status & 0x00000010) {
1082                         nouveau_graph_trap_info(dev, &trap);
1083                         if (nouveau_pgraph_intr_swmthd(dev, &trap))
1084                                 unhandled = 1;
1085                         if (unhandled && nouveau_ratelimit())
1086                                 nouveau_graph_dump_trap_info(dev,
1087                                                 "PGRAPH_ILLEGAL_MTHD", &trap);
1088                         status &= ~0x00000010;
1089                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000010);
1090                 }
1091
1092                 /* ILLEGAL_CLASS: You used a wrong class. */
1093                 if (status & 0x00000020) {
1094                         nouveau_graph_trap_info(dev, &trap);
1095                         if (nouveau_ratelimit())
1096                                 nouveau_graph_dump_trap_info(dev,
1097                                                 "PGRAPH_ILLEGAL_CLASS", &trap);
1098                         status &= ~0x00000020;
1099                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000020);
1100                 }
1101
1102                 /* DOUBLE_NOTIFY: You tried to set a NOTIFY on another NOTIFY. */
1103                 if (status & 0x00000040) {
1104                         nouveau_graph_trap_info(dev, &trap);
1105                         if (nouveau_ratelimit())
1106                                 nouveau_graph_dump_trap_info(dev,
1107                                                 "PGRAPH_DOUBLE_NOTIFY", &trap);
1108                         status &= ~0x00000040;
1109                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000040);
1110                 }
1111
1112                 /* CONTEXT_SWITCH: PGRAPH needs us to load a new context */
1113                 if (status & 0x00001000) {
1114                         nv_wr32(dev, 0x400500, 0x00000000);
1115                         nv_wr32(dev, NV03_PGRAPH_INTR,
1116                                 NV_PGRAPH_INTR_CONTEXT_SWITCH);
1117                         nv_wr32(dev, NV40_PGRAPH_INTR_EN, nv_rd32(dev,
1118                                 NV40_PGRAPH_INTR_EN) &
1119                                 ~NV_PGRAPH_INTR_CONTEXT_SWITCH);
1120                         nv_wr32(dev, 0x400500, 0x00010001);
1121
1122                         nv50_graph_context_switch(dev);
1123
1124                         status &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
1125                 }
1126
1127                 /* BUFFER_NOTIFY: Your m2mf transfer finished */
1128                 if (status & 0x00010000) {
1129                         nouveau_graph_trap_info(dev, &trap);
1130                         if (nouveau_ratelimit())
1131                                 nouveau_graph_dump_trap_info(dev,
1132                                                 "PGRAPH_BUFFER_NOTIFY", &trap);
1133                         status &= ~0x00010000;
1134                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00010000);
1135                 }
1136
1137                 /* DATA_ERROR: Invalid value for this method, or invalid
1138                  * state in current PGRAPH context for this operation */
1139                 if (status & 0x00100000) {
1140                         nouveau_graph_trap_info(dev, &trap);
1141                         if (nouveau_ratelimit()) {
1142                                 nouveau_graph_dump_trap_info(dev,
1143                                                 "PGRAPH_DATA_ERROR", &trap);
1144                                 NV_INFO (dev, "PGRAPH_DATA_ERROR - ");
1145                                 nouveau_print_enum_names(nv_rd32(dev, 0x400110),
1146                                                 nv50_data_error_names);
1147                                 printk("\n");
1148                         }
1149                         status &= ~0x00100000;
1150                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00100000);
1151                 }
1152
1153                 /* TRAP: Something bad happened in the middle of command
1154                  * execution.  Has a billion types, subtypes, and even
1155                  * subsubtypes. */
1156                 if (status & 0x00200000) {
1157                         nv50_pgraph_trap_handler(dev);
1158                         status &= ~0x00200000;
1159                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00200000);
1160                 }
1161
1162                 /* Unknown, never seen: 0x00400000 */
1163
1164                 /* SINGLE_STEP: Happens on every method if you turned on
1165                  * single stepping in 40008c */
1166                 if (status & 0x01000000) {
1167                         nouveau_graph_trap_info(dev, &trap);
1168                         if (nouveau_ratelimit())
1169                                 nouveau_graph_dump_trap_info(dev,
1170                                                 "PGRAPH_SINGLE_STEP", &trap);
1171                         status &= ~0x01000000;
1172                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x01000000);
1173                 }
1174
1175                 /* 0x02000000 happens when you pause a ctxprog...
1176                  * but the only way this can happen that I know is by
1177                  * poking the relevant MMIO register, and we don't
1178                  * do that. */
1179
1180                 if (status) {
1181                         NV_INFO(dev, "Unhandled PGRAPH_INTR - 0x%08x\n",
1182                                 status);
1183                         nv_wr32(dev, NV03_PGRAPH_INTR, status);
1184                 }
1185
1186                 {
1187                         const int isb = (1 << 16) | (1 << 0);
1188
1189                         if ((nv_rd32(dev, 0x400500) & isb) != isb)
1190                                 nv_wr32(dev, 0x400500,
1191                                         nv_rd32(dev, 0x400500) | isb);
1192                 }
1193         }
1194
1195         nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PGRAPH_PENDING);
1196         if (nv_rd32(dev, 0x400824) & (1 << 31))
1197                 nv_wr32(dev, 0x400824, nv_rd32(dev, 0x400824) & ~(1 << 31));
1198 }
1199
1200 static void
1201 nouveau_crtc_irq_handler(struct drm_device *dev, int crtc)
1202 {
1203         if (crtc & 1) {
1204                 nv_wr32(dev, NV_CRTC0_INTSTAT, NV_CRTC_INTR_VBLANK);
1205                 drm_handle_vblank(dev, 0);
1206         }
1207
1208         if (crtc & 2) {
1209                 nv_wr32(dev, NV_CRTC1_INTSTAT, NV_CRTC_INTR_VBLANK);
1210                 drm_handle_vblank(dev, 1);
1211         }
1212 }
1213
1214 irqreturn_t
1215 nouveau_irq_handler(DRM_IRQ_ARGS)
1216 {
1217         struct drm_device *dev = (struct drm_device *)arg;
1218         struct drm_nouveau_private *dev_priv = dev->dev_private;
1219         uint32_t status;
1220         unsigned long flags;
1221
1222         status = nv_rd32(dev, NV03_PMC_INTR_0);
1223         if (!status)
1224                 return IRQ_NONE;
1225
1226         spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
1227
1228         if (status & NV_PMC_INTR_0_PFIFO_PENDING) {
1229                 nouveau_fifo_irq_handler(dev);
1230                 status &= ~NV_PMC_INTR_0_PFIFO_PENDING;
1231         }
1232
1233         if (status & NV_PMC_INTR_0_PGRAPH_PENDING) {
1234                 if (dev_priv->card_type >= NV_50)
1235                         nv50_pgraph_irq_handler(dev);
1236                 else
1237                         nouveau_pgraph_irq_handler(dev);
1238
1239                 status &= ~NV_PMC_INTR_0_PGRAPH_PENDING;
1240         }
1241
1242         if (status & 0x00004000) {
1243                 u32 stat = nv_rd32(dev, 0x102130);
1244                 u32 mthd = nv_rd32(dev, 0x102190);
1245                 u32 data = nv_rd32(dev, 0x102194);
1246                 u32 inst = nv_rd32(dev, 0x102188) & 0x7fffffff;
1247
1248                 NV_INFO(dev, "PCRYPT_INTR: 0x%08x 0x%08x 0x%08x 0x%08x\n",
1249                         stat, mthd, data, inst);
1250                 nv_wr32(dev, 0x102130, stat);
1251                 nv_wr32(dev, 0x10200c, 0x10);
1252
1253                 nv50_fb_vm_trap(dev, nouveau_ratelimit(), "PCRYPT");
1254                 status &= ~0x00004000;
1255
1256         }
1257
1258         if (status & NV_PMC_INTR_0_CRTCn_PENDING) {
1259                 nouveau_crtc_irq_handler(dev, (status>>24)&3);
1260                 status &= ~NV_PMC_INTR_0_CRTCn_PENDING;
1261         }
1262
1263         if (status & (NV_PMC_INTR_0_NV50_DISPLAY_PENDING |
1264                       NV_PMC_INTR_0_NV50_I2C_PENDING)) {
1265                 nv50_display_irq_handler(dev);
1266                 status &= ~(NV_PMC_INTR_0_NV50_DISPLAY_PENDING |
1267                             NV_PMC_INTR_0_NV50_I2C_PENDING);
1268         }
1269
1270         if (status)
1271                 NV_ERROR(dev, "Unhandled PMC INTR status bits 0x%08x\n", status);
1272
1273         spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
1274
1275         if (dev_priv->msi_enabled)
1276                 nv_wr08(dev, 0x00088068, 0xff);
1277
1278         return IRQ_HANDLED;
1279 }
1280
1281 int
1282 nouveau_irq_init(struct drm_device *dev)
1283 {
1284         struct drm_nouveau_private *dev_priv = dev->dev_private;
1285         int ret;
1286
1287         if (nouveau_msi != 0 && dev_priv->card_type >= NV_50) {
1288                 ret = pci_enable_msi(dev->pdev);
1289                 if (ret == 0) {
1290                         NV_INFO(dev, "enabled MSI\n");
1291                         dev_priv->msi_enabled = true;
1292                 }
1293         }
1294
1295         return drm_irq_install(dev);
1296 }
1297
1298 void
1299 nouveau_irq_fini(struct drm_device *dev)
1300 {
1301         struct drm_nouveau_private *dev_priv = dev->dev_private;
1302
1303         drm_irq_uninstall(dev);
1304         if (dev_priv->msi_enabled)
1305                 pci_disable_msi(dev->pdev);
1306 }