drivers/mxc/gpu-viv/hal/kernel/gc_hal_kernel_event.c

   1 /****************************************************************************
   2 *
   3 *    Copyright (C) 2005 - 2013 by Vivante Corp.
   4 *
   5 *    This program is free software; you can redistribute it and/or modify
   6 *    it under the terms of the GNU General Public License as published by
   7 *    the Free Software Foundation; either version 2 of the license, or
   8 *    (at your option) any later version.
   9 *
  10 *    This program is distributed in the hope that it will be useful,
  11 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13 *    GNU General Public License for more details.
  14 *
  15 *    You should have received a copy of the GNU General Public License
  16 *    along with this program; if not write to the Free Software
  17 *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18 *
  19 *****************************************************************************/
  20
  21
  22 #include "gc_hal_kernel_precomp.h"
  23 #include "gc_hal_kernel_buffer.h"
  24
  25 #ifdef __QNXNTO__
  26 #include <atomic.h>
  27 #include "gc_hal_kernel_qnx.h"
  28 #endif
  29
  30 #define _GC_OBJ_ZONE                    gcvZONE_EVENT
  31
  32 #define gcdEVENT_ALLOCATION_COUNT       (4096 / gcmSIZEOF(gcsHAL_INTERFACE))
  33 #define gcdEVENT_MIN_THRESHOLD          4
  34
  35 /******************************************************************************\
  36 ********************************* Support Code *********************************
  37 \******************************************************************************/
  38
  39 static gceSTATUS
  40 gckEVENT_AllocateQueue(
  41     IN gckEVENT Event,
  42     OUT gcsEVENT_QUEUE_PTR * Queue
  43     )
  44 {
  45     gceSTATUS status;
  46
  47     gcmkHEADER_ARG("Event=0x%x", Event);
  48
  49     /* Verify the arguments. */
  50     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
  51     gcmkVERIFY_ARGUMENT(Queue != gcvNULL);
  52
  53     /* Do we have free queues? */
  54     if (Event->freeList == gcvNULL)
  55     {
  56         gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES);
  57     }
  58
  59     /* Move one free queue from the free list. */
  60     * Queue = Event->freeList;
  61     Event->freeList = Event->freeList->next;
  62
  63     /* Success. */
  64     gcmkFOOTER_ARG("*Queue=0x%x", gcmOPT_POINTER(Queue));
  65     return gcvSTATUS_OK;
  66
  67 OnError:
  68     /* Return the status. */
  69     gcmkFOOTER();
  70     return status;
  71 }
  72
  73 static gceSTATUS
  74 gckEVENT_FreeQueue(
  75     IN gckEVENT Event,
  76     OUT gcsEVENT_QUEUE_PTR Queue
  77     )
  78 {
  79     gceSTATUS status = gcvSTATUS_OK;
  80
  81     gcmkHEADER_ARG("Event=0x%x", Event);
  82
  83     /* Verify the arguments. */
  84     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
  85     gcmkVERIFY_ARGUMENT(Queue != gcvNULL);
  86
  87     /* Move one free queue from the free list. */
  88     Queue->next = Event->freeList;
  89     Event->freeList = Queue;
  90
  91     /* Success. */
  92     gcmkFOOTER();
  93     return status;
  94 }
  95
  96 static gceSTATUS
  97 gckEVENT_FreeRecord(
  98     IN gckEVENT Event,
  99     IN gcsEVENT_PTR Record
 100     )
 101 {
 102     gceSTATUS status;
 103     gctBOOL acquired = gcvFALSE;
 104
 105     gcmkHEADER_ARG("Event=0x%x Record=0x%x", Event, Record);
 106
 107     /* Verify the arguments. */
 108     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
 109     gcmkVERIFY_ARGUMENT(Record != gcvNULL);
 110
 111     /* Acquire the mutex. */
 112     gcmkONERROR(gckOS_AcquireMutex(Event->os,
 113                                    Event->freeEventMutex,
 114                                    gcvINFINITE));
 115     acquired = gcvTRUE;
 116
 117     /* Push the record on the free list. */
 118     Record->next           = Event->freeEventList;
 119     Event->freeEventList   = Record;
 120     Event->freeEventCount += 1;
 121
 122     /* Release the mutex. */
 123     gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->freeEventMutex));
 124
 125     /* Success. */
 126     gcmkFOOTER_NO();
 127     return gcvSTATUS_OK;
 128
 129 OnError:
 130     /* Roll back. */
 131     if (acquired)
 132     {
 133         gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->freeEventMutex));
 134     }
 135
 136     /* Return the status. */
 137     gcmkFOOTER();
 138     return gcvSTATUS_OK;
 139 }
 140
 141 static gceSTATUS
 142 gckEVENT_IsEmpty(
 143     IN gckEVENT Event,
 144     OUT gctBOOL_PTR IsEmpty
 145     )
 146 {
 147     gceSTATUS status;
 148     gctSIZE_T i;
 149
 150     gcmkHEADER_ARG("Event=0x%x", Event);
 151
 152     /* Verify the arguments. */
 153     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
 154     gcmkVERIFY_ARGUMENT(IsEmpty != gcvNULL);
 155
 156     /* Assume the event queue is empty. */
 157     *IsEmpty = gcvTRUE;
 158
 159     /* Walk the event queue. */
 160     for (i = 0; i < gcmCOUNTOF(Event->queues); ++i)
 161     {
 162         /* Check whether this event is in use. */
 163         if (Event->queues[i].head != gcvNULL)
 164         {
 165             /* The event is in use, hence the queue is not empty. */
 166             *IsEmpty = gcvFALSE;
 167             break;
 168         }
 169     }
 170
 171     /* Try acquiring the mutex. */
 172     status = gckOS_AcquireMutex(Event->os, Event->eventQueueMutex, 0);
 173     if (status == gcvSTATUS_TIMEOUT)
 174     {
 175         /* Timeout - queue is no longer empty. */
 176         *IsEmpty = gcvFALSE;
 177     }
 178     else
 179     {
 180         /* Bail out on error. */
 181         gcmkONERROR(status);
 182
 183         /* Release the mutex. */
 184         gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex));
 185     }
 186
 187     /* Success. */
 188     gcmkFOOTER_ARG("*IsEmpty=%d", gcmOPT_VALUE(IsEmpty));
 189     return gcvSTATUS_OK;
 190
 191 OnError:
 192     /* Return the status. */
 193     gcmkFOOTER();
 194     return status;
 195 }
 196
 197 static gceSTATUS
 198 _TryToIdleGPU(
 199     IN gckEVENT Event
 200 )
 201 {
 202     gceSTATUS status;
 203     gctBOOL empty = gcvFALSE, idle = gcvFALSE;
 204     gctBOOL powerLocked = gcvFALSE;
 205     gckHARDWARE hardware;
 206
 207     gcmkHEADER_ARG("Event=0x%x", Event);
 208
 209     /* Verify the arguments. */
 210     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
 211
 212     /* Grab gckHARDWARE object. */
 213     hardware = Event->kernel->hardware;
 214     gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE);
 215
 216     /* Check whether the event queue is empty. */
 217     gcmkONERROR(gckEVENT_IsEmpty(Event, &empty));
 218
 219     if (empty)
 220     {
 221         status = gckOS_AcquireMutex(hardware->os, hardware->powerMutex, 0);
 222         if (status == gcvSTATUS_TIMEOUT)
 223         {
 224             gcmkFOOTER_NO();
 225             return gcvSTATUS_OK;
 226         }
 227
 228         powerLocked = gcvTRUE;
 229
 230         /* Query whether the hardware is idle. */
 231         gcmkONERROR(gckHARDWARE_QueryIdle(Event->kernel->hardware, &idle));
 232
 233         gcmkONERROR(gckOS_ReleaseMutex(hardware->os, hardware->powerMutex));
 234         powerLocked = gcvFALSE;
 235
 236         if (idle)
 237         {
 238             /* Inform the system of idle GPU. */
 239             gcmkONERROR(gckOS_Broadcast(Event->os,
 240                                         Event->kernel->hardware,
 241                                         gcvBROADCAST_GPU_IDLE));
 242         }
 243     }
 244
 245     gcmkFOOTER_NO();
 246     return gcvSTATUS_OK;
 247
 248 OnError:
 249     if (powerLocked)
 250     {
 251         gcmkONERROR(gckOS_ReleaseMutex(hardware->os, hardware->powerMutex));
 252         powerLocked = gcvFALSE;
 253     }
 254
 255     gcmkFOOTER();
 256     return status;
 257 }
 258
 259 static gceSTATUS
 260 __RemoveRecordFromProcessDB(
 261     IN gckEVENT Event,
 262     IN gcsEVENT_PTR Record
 263     )
 264 {
 265     gcmkHEADER_ARG("Event=0x%x Record=0x%x", Event, Record);
 266     gcmkVERIFY_ARGUMENT(Record != gcvNULL);
 267
 268     while (Record != gcvNULL)
 269     {
 270         if (Record->info.command == gcvHAL_SIGNAL)
 271         {
 272             /* TODO: Find a better place to bind signal to hardware.*/
 273             gcmkVERIFY_OK(gckOS_SignalSetHardware(Event->os,
 274                         gcmUINT64_TO_PTR(Record->info.u.Signal.signal),
 275                         Event->kernel->hardware));
 276         }
 277
 278         if (Record->fromKernel)
 279         {
 280             /* No need to check db if event is from kernel. */
 281             Record = Record->next;
 282             continue;
 283         }
 284
 285         switch (Record->info.command)
 286         {
 287         case gcvHAL_FREE_NON_PAGED_MEMORY:
 288             gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB(
 289                 Event->kernel,
 290                 Record->processID,
 291                 gcvDB_NON_PAGED,
 292                 gcmUINT64_TO_PTR(Record->info.u.FreeNonPagedMemory.logical)));
 293             break;
 294
 295         case gcvHAL_FREE_CONTIGUOUS_MEMORY:
 296             gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB(
 297                 Event->kernel,
 298                 Record->processID,
 299                 gcvDB_CONTIGUOUS,
 300                 gcmUINT64_TO_PTR(Record->info.u.FreeContiguousMemory.logical)));
 301             break;
 302
 303         case gcvHAL_FREE_VIDEO_MEMORY:
 304             gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB(
 305                 Event->kernel,
 306                 Record->processID,
 307                 gcvDB_VIDEO_MEMORY,
 308                 gcmUINT64_TO_PTR(Record->info.u.FreeVideoMemory.node)));
 309
 310             {
 311                 gcuVIDMEM_NODE_PTR node = (gcuVIDMEM_NODE_PTR)(gcmUINT64_TO_PTR(Record->info.u.FreeVideoMemory.node));
 312
 313                 if (node->VidMem.memory->object.type == gcvOBJ_VIDMEM)
 314                 {
 315                      gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB(Event->kernel,
 316                                       Record->processID,
 317                                       gcvDB_VIDEO_MEMORY_RESERVED,
 318                                       node));
 319                 }
 320                 else if(node->Virtual.contiguous)
 321                 {
 322                     gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB(Event->kernel,
 323                                       Record->processID,
 324                                       gcvDB_VIDEO_MEMORY_CONTIGUOUS,
 325                                       node));
 326                 }
 327                 else
 328                 {
 329                     gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB(Event->kernel,
 330                                       Record->processID,
 331                                       gcvDB_VIDEO_MEMORY_VIRTUAL,
 332                                       node));
 333                 }
 334             }
 335
 336             break;
 337
 338         case gcvHAL_UNLOCK_VIDEO_MEMORY:
 339             gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB(
 340                 Event->kernel,
 341                 Record->processID,
 342                 gcvDB_VIDEO_MEMORY_LOCKED,
 343                 gcmUINT64_TO_PTR(Record->info.u.UnlockVideoMemory.node)));
 344             break;
 345
 346         case gcvHAL_UNMAP_USER_MEMORY:
 347             gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB(
 348                 Event->kernel,
 349                 Record->processID,
 350                 gcvDB_MAP_USER_MEMORY,
 351                 gcmINT2PTR(Record->info.u.UnmapUserMemory.info)));
 352             break;
 353
 354         case gcvHAL_FREE_VIRTUAL_COMMAND_BUFFER:
 355             gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB(
 356                 Event->kernel,
 357                 Record->processID,
 358                 gcvDB_COMMAND_BUFFER,
 359                 gcmUINT64_TO_PTR(Record->info.u.FreeVirtualCommandBuffer.logical)));
 360             break;
 361
 362         default:
 363             break;
 364         }
 365
 366         Record = Record->next;
 367     }
 368     gcmkFOOTER_NO();
 369     return gcvSTATUS_OK;
 370 }
 371
 372 void
 373 _SubmitTimerFunction(
 374     gctPOINTER Data
 375     )
 376 {
 377     gckEVENT event = (gckEVENT)Data;
 378     gcmkVERIFY_OK(gckEVENT_Submit(event, gcvTRUE, gcvFALSE));
 379 }
 380
 381 /******************************************************************************\
 382 ******************************* gckEVENT API Code *******************************
 383 \******************************************************************************/
 384
 385 /*******************************************************************************
 386 **
 387 **  gckEVENT_Construct
 388 **
 389 **  Construct a new gckEVENT object.
 390 **
 391 **  INPUT:
 392 **
 393 **      gckKERNEL Kernel
 394 **          Pointer to an gckKERNEL object.
 395 **
 396 **  OUTPUT:
 397 **
 398 **      gckEVENT * Event
 399 **          Pointer to a variable that receives the gckEVENT object pointer.
 400 */
 401 gceSTATUS
 402 gckEVENT_Construct(
 403     IN gckKERNEL Kernel,
 404     OUT gckEVENT * Event
 405     )
 406 {
 407     gckOS os;
 408     gceSTATUS status;
 409     gckEVENT eventObj = gcvNULL;
 410     int i;
 411     gcsEVENT_PTR record;
 412     gctPOINTER pointer = gcvNULL;
 413
 414     gcmkHEADER_ARG("Kernel=0x%x", Kernel);
 415
 416     /* Verify the arguments. */
 417     gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
 418     gcmkVERIFY_ARGUMENT(Event != gcvNULL);
 419
 420     /* Extract the pointer to the gckOS object. */
 421     os = Kernel->os;
 422     gcmkVERIFY_OBJECT(os, gcvOBJ_OS);
 423
 424     /* Allocate the gckEVENT object. */
 425     gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(struct _gckEVENT), &pointer));
 426
 427     eventObj = pointer;
 428
 429     /* Reset the object. */
 430     gcmkVERIFY_OK(gckOS_ZeroMemory(eventObj, gcmSIZEOF(struct _gckEVENT)));
 431
 432     /* Initialize the gckEVENT object. */
 433     eventObj->object.type = gcvOBJ_EVENT;
 434     eventObj->kernel      = Kernel;
 435     eventObj->os          = os;
 436
 437     /* Create the mutexes. */
 438     gcmkONERROR(gckOS_CreateMutex(os, &eventObj->eventQueueMutex));
 439     gcmkONERROR(gckOS_CreateMutex(os, &eventObj->freeEventMutex));
 440     gcmkONERROR(gckOS_CreateMutex(os, &eventObj->eventListMutex));
 441
 442     /* Create a bunch of event reccords. */
 443     for (i = 0; i < gcdEVENT_ALLOCATION_COUNT; i += 1)
 444     {
 445         /* Allocate an event record. */
 446         gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcsEVENT), &pointer));
 447
 448         record = pointer;
 449
 450         /* Push it on the free list. */
 451         record->next              = eventObj->freeEventList;
 452         eventObj->freeEventList   = record;
 453         eventObj->freeEventCount += 1;
 454     }
 455
 456     /* Initialize the free list of event queues. */
 457     for (i = 0; i < gcdREPO_LIST_COUNT; i += 1)
 458     {
 459         eventObj->repoList[i].next = eventObj->freeList;
 460         eventObj->freeList = &eventObj->repoList[i];
 461     }
 462
 463     /* Construct the atom. */
 464     gcmkONERROR(gckOS_AtomConstruct(os, &eventObj->freeAtom));
 465     gcmkONERROR(gckOS_AtomSet(os,
 466                               eventObj->freeAtom,
 467                               gcmCOUNTOF(eventObj->queues)));
 468
 469 #if gcdSMP
 470     gcmkONERROR(gckOS_AtomConstruct(os, &eventObj->pending));
 471 #endif
 472
 473     gcmkVERIFY_OK(gckOS_CreateTimer(os,
 474                                     _SubmitTimerFunction,
 475                                     (gctPOINTER)eventObj,
 476                                     &eventObj->submitTimer));
 477
 478     /* Return pointer to the gckEVENT object. */
 479     *Event = eventObj;
 480
 481     /* Success. */
 482     gcmkFOOTER_ARG("*Event=0x%x", *Event);
 483     return gcvSTATUS_OK;
 484
 485 OnError:
 486     /* Roll back. */
 487     if (eventObj != gcvNULL)
 488     {
 489         if (eventObj->eventQueueMutex != gcvNULL)
 490         {
 491             gcmkVERIFY_OK(gckOS_DeleteMutex(os, eventObj->eventQueueMutex));
 492         }
 493
 494         if (eventObj->freeEventMutex != gcvNULL)
 495         {
 496             gcmkVERIFY_OK(gckOS_DeleteMutex(os, eventObj->freeEventMutex));
 497         }
 498
 499         if (eventObj->eventListMutex != gcvNULL)
 500         {
 501             gcmkVERIFY_OK(gckOS_DeleteMutex(os, eventObj->eventListMutex));
 502         }
 503
 504         while (eventObj->freeEventList != gcvNULL)
 505         {
 506             record = eventObj->freeEventList;
 507             eventObj->freeEventList = record->next;
 508
 509             gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, record));
 510         }
 511
 512         if (eventObj->freeAtom != gcvNULL)
 513         {
 514             gcmkVERIFY_OK(gckOS_AtomDestroy(os, eventObj->freeAtom));
 515         }
 516
 517 #if gcdSMP
 518         if (eventObj->pending != gcvNULL)
 519         {
 520             gcmkVERIFY_OK(gckOS_AtomDestroy(os, eventObj->pending));
 521         }
 522 #endif
 523         gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, eventObj));
 524     }
 525
 526     /* Return the status. */
 527     gcmkFOOTER();
 528     return status;
 529 }
 530
 531 /*******************************************************************************
 532 **
 533 **  gckEVENT_Destroy
 534 **
 535 **  Destroy an gckEVENT object.
 536 **
 537 **  INPUT:
 538 **
 539 **      gckEVENT Event
 540 **          Pointer to an gckEVENT object.
 541 **
 542 **  OUTPUT:
 543 **
 544 **      Nothing.
 545 */
 546 gceSTATUS
 547 gckEVENT_Destroy(
 548     IN gckEVENT Event
 549     )
 550 {
 551     gcsEVENT_PTR record;
 552     gcsEVENT_QUEUE_PTR queue;
 553
 554     gcmkHEADER_ARG("Event=0x%x", Event);
 555
 556     /* Verify the arguments. */
 557     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
 558
 559     if (Event->submitTimer != gcvNULL)
 560     {
 561         gcmkVERIFY_OK(gckOS_StopTimer(Event->os, Event->submitTimer));
 562         gcmkVERIFY_OK(gckOS_DestroyTimer(Event->os, Event->submitTimer));
 563     }
 564
 565     /* Delete the queue mutex. */
 566     gcmkVERIFY_OK(gckOS_DeleteMutex(Event->os, Event->eventQueueMutex));
 567
 568     /* Free all free events. */
 569     while (Event->freeEventList != gcvNULL)
 570     {
 571         record = Event->freeEventList;
 572         Event->freeEventList = record->next;
 573
 574         gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Event->os, record));
 575     }
 576
 577     /* Delete the free mutex. */
 578     gcmkVERIFY_OK(gckOS_DeleteMutex(Event->os, Event->freeEventMutex));
 579
 580     /* Free all pending queues. */
 581     while (Event->queueHead != gcvNULL)
 582     {
 583         /* Get the current queue. */
 584         queue = Event->queueHead;
 585
 586         /* Free all pending events. */
 587         while (queue->head != gcvNULL)
 588         {
 589             record      = queue->head;
 590             queue->head = record->next;
 591
 592             gcmkTRACE_ZONE_N(
 593                 gcvLEVEL_WARNING, gcvZONE_EVENT,
 594                 gcmSIZEOF(record) + gcmSIZEOF(queue->source),
 595                 "Event record 0x%x is still pending for %d.",
 596                 record, queue->source
 597                 );
 598
 599             gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Event->os, record));
 600         }
 601
 602         /* Remove the top queue from the list. */
 603         if (Event->queueHead == Event->queueTail)
 604         {
 605             Event->queueHead =
 606             Event->queueTail = gcvNULL;
 607         }
 608         else
 609         {
 610             Event->queueHead = Event->queueHead->next;
 611         }
 612
 613         /* Free the queue. */
 614         gcmkVERIFY_OK(gckEVENT_FreeQueue(Event, queue));
 615     }
 616
 617     /* Delete the list mutex. */
 618     gcmkVERIFY_OK(gckOS_DeleteMutex(Event->os, Event->eventListMutex));
 619
 620     /* Delete the atom. */
 621     gcmkVERIFY_OK(gckOS_AtomDestroy(Event->os, Event->freeAtom));
 622
 623 #if gcdSMP
 624     gcmkVERIFY_OK(gckOS_AtomDestroy(Event->os, Event->pending));
 625 #endif
 626
 627     /* Mark the gckEVENT object as unknown. */
 628     Event->object.type = gcvOBJ_UNKNOWN;
 629
 630     /* Free the gckEVENT object. */
 631     gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Event->os, Event));
 632
 633     /* Success. */
 634     gcmkFOOTER_NO();
 635     return gcvSTATUS_OK;
 636 }
 637
 638 /*******************************************************************************
 639 **
 640 **  gckEVENT_GetEvent
 641 **
 642 **  Reserve the next available hardware event.
 643 **
 644 **  INPUT:
 645 **
 646 **      gckEVENT Event
 647 **          Pointer to an gckEVENT object.
 648 **
 649 **      gctBOOL Wait
 650 **          Set to gcvTRUE to force the function to wait if no events are
 651 **          immediately available.
 652 **
 653 **      gceKERNEL_WHERE Source
 654 **          Source of the event.
 655 **
 656 **  OUTPUT:
 657 **
 658 **      gctUINT8 * EventID
 659 **          Reserved event ID.
 660 */
 661 gceSTATUS
 662 gckEVENT_GetEvent(
 663     IN gckEVENT Event,
 664     IN gctBOOL Wait,
 665     OUT gctUINT8 * EventID,
 666     IN gceKERNEL_WHERE Source
 667     )
 668 {
 669     gctINT i, id;
 670     gceSTATUS status;
 671     gctBOOL acquired = gcvFALSE;
 672     gctINT32 free;
 673
 674 #if gcdGPU_TIMEOUT
 675     gctUINT32 timer = 0;
 676 #endif
 677
 678     gcmkHEADER_ARG("Event=0x%x Source=%d", Event, Source);
 679
 680     while (gcvTRUE)
 681     {
 682         /* Grab the queue mutex. */
 683         gcmkONERROR(gckOS_AcquireMutex(Event->os,
 684                                        Event->eventQueueMutex,
 685                                        gcvINFINITE));
 686         acquired = gcvTRUE;
 687
 688         /* Walk through all events. */
 689         id = Event->lastID;
 690         for (i = 0; i < gcmCOUNTOF(Event->queues); ++i)
 691         {
 692             gctINT nextID = gckMATH_ModuloInt((id + 1),
 693                                               gcmCOUNTOF(Event->queues));
 694
 695             if (Event->queues[id].head == gcvNULL)
 696             {
 697                 *EventID = (gctUINT8) id;
 698
 699                 Event->lastID = (gctUINT8) nextID;
 700
 701                 /* Save time stamp of event. */
 702                 Event->queues[id].stamp  = ++(Event->stamp);
 703                 Event->queues[id].source = Source;
 704
 705                 gcmkONERROR(gckOS_AtomDecrement(Event->os,
 706                                                 Event->freeAtom,
 707                                                 &free));
 708 #if gcdDYNAMIC_SPEED
 709                 if (free <= gcdDYNAMIC_EVENT_THRESHOLD)
 710                 {
 711                     gcmkONERROR(gckOS_BroadcastHurry(
 712                         Event->os,
 713                         Event->kernel->hardware,
 714                         gcdDYNAMIC_EVENT_THRESHOLD - free));
 715                 }
 716 #endif
 717
 718                 /* Release the queue mutex. */
 719                 gcmkONERROR(gckOS_ReleaseMutex(Event->os,
 720                                                Event->eventQueueMutex));
 721
 722                 /* Success. */
 723                 gcmkTRACE_ZONE_N(
 724                     gcvLEVEL_INFO, gcvZONE_EVENT,
 725                     gcmSIZEOF(id),
 726                     "Using id=%d",
 727                     id
 728                     );
 729
 730                 gcmkFOOTER_ARG("*EventID=%u", *EventID);
 731                 return gcvSTATUS_OK;
 732             }
 733
 734             id = nextID;
 735         }
 736
 737 #if gcdDYNAMIC_SPEED
 738         /* No free events, speed up the GPU right now! */
 739         gcmkONERROR(gckOS_BroadcastHurry(Event->os,
 740                                          Event->kernel->hardware,
 741                                          gcdDYNAMIC_EVENT_THRESHOLD));
 742 #endif
 743
 744         /* Release the queue mutex. */
 745         gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex));
 746         acquired = gcvFALSE;
 747
 748         /* Fail if wait is not requested. */
 749         if (!Wait)
 750         {
 751             /* Out of resources. */
 752             gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES);
 753         }
 754
 755         /* Delay a while. */
 756         gcmkONERROR(gckOS_Delay(Event->os, 1));
 757
 758 #if gcdGPU_TIMEOUT
 759         /* Increment the wait timer. */
 760         timer += 1;
 761
 762         if (timer == Event->kernel->timeOut)
 763         {
 764             /* Try to call any outstanding events. */
 765             gcmkONERROR(gckHARDWARE_Interrupt(Event->kernel->hardware,
 766                                               gcvTRUE));
 767         }
 768         else if (timer > Event->kernel->timeOut)
 769         {
 770             gcmkTRACE_N(
 771                 gcvLEVEL_ERROR,
 772                 gcmSIZEOF(gctCONST_STRING) + gcmSIZEOF(gctINT),
 773                 "%s(%d): no available events\n",
 774                 __FUNCTION__, __LINE__
 775                 );
 776
 777             /* Bail out. */
 778             gcmkONERROR(gcvSTATUS_GPU_NOT_RESPONDING);
 779         }
 780 #endif
 781     }
 782
 783 OnError:
 784     if (acquired)
 785     {
 786         /* Release the queue mutex. */
 787         gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex));
 788     }
 789
 790     /* Return the status. */
 791     gcmkFOOTER();
 792     return status;
 793 }
 794
 795 /*******************************************************************************
 796 **
 797 **  gckEVENT_AllocateRecord
 798 **
 799 **  Allocate a record for the new event.
 800 **
 801 **  INPUT:
 802 **
 803 **      gckEVENT Event
 804 **          Pointer to an gckEVENT object.
 805 **
 806 **      gctBOOL AllocateAllowed
 807 **          State for allocation if out of free events.
 808 **
 809 **  OUTPUT:
 810 **
 811 **      gcsEVENT_PTR * Record
 812 **          Allocated event record.
 813 */
 814 gceSTATUS
 815 gckEVENT_AllocateRecord(
 816     IN gckEVENT Event,
 817     IN gctBOOL AllocateAllowed,
 818     OUT gcsEVENT_PTR * Record
 819     )
 820 {
 821     gceSTATUS status;
 822     gctBOOL acquired = gcvFALSE;
 823     gctINT i;
 824     gcsEVENT_PTR record;
 825     gctPOINTER pointer = gcvNULL;
 826
 827     gcmkHEADER_ARG("Event=0x%x AllocateAllowed=%d", Event, AllocateAllowed);
 828
 829     /* Verify the arguments. */
 830     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
 831     gcmkVERIFY_ARGUMENT(Record != gcvNULL);
 832
 833     /* Acquire the mutex. */
 834     gcmkONERROR(gckOS_AcquireMutex(Event->os, Event->freeEventMutex, gcvINFINITE));
 835     acquired = gcvTRUE;
 836
 837     /* Test if we are below the allocation threshold. */
 838     if ( (AllocateAllowed && (Event->freeEventCount < gcdEVENT_MIN_THRESHOLD)) ||
 839          (Event->freeEventCount == 0) )
 840     {
 841         /* Allocate a bunch of records. */
 842         for (i = 0; i < gcdEVENT_ALLOCATION_COUNT; i += 1)
 843         {
 844             /* Allocate an event record. */
 845             gcmkONERROR(gckOS_Allocate(Event->os,
 846                                        gcmSIZEOF(gcsEVENT),
 847                                        &pointer));
 848
 849             record = pointer;
 850
 851             /* Push it on the free list. */
 852             record->next           = Event->freeEventList;
 853             Event->freeEventList   = record;
 854             Event->freeEventCount += 1;
 855         }
 856     }
 857
 858     *Record                = Event->freeEventList;
 859     Event->freeEventList   = Event->freeEventList->next;
 860     Event->freeEventCount -= 1;
 861
 862     /* Release the mutex. */
 863     gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->freeEventMutex));
 864     acquired = gcvFALSE;
 865
 866     /* Success. */
 867     gcmkFOOTER_ARG("*Record=0x%x", gcmOPT_POINTER(Record));
 868     return gcvSTATUS_OK;
 869
 870 OnError:
 871     /* Roll back. */
 872     if (acquired)
 873     {
 874         gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->freeEventMutex));
 875     }
 876
 877     /* Return the status. */
 878     gcmkFOOTER();
 879     return status;
 880 }
 881
 882 /*******************************************************************************
 883 **
 884 **  gckEVENT_AddList
 885 **
 886 **  Add a new event to the list of events.
 887 **
 888 **  INPUT:
 889 **
 890 **      gckEVENT Event
 891 **          Pointer to an gckEVENT object.
 892 **
 893 **      gcsHAL_INTERFACE_PTR Interface
 894 **          Pointer to the interface for the event to be added.
 895 **
 896 **      gceKERNEL_WHERE FromWhere
 897 **          Place in the pipe where the event needs to be generated.
 898 **
 899 **      gctBOOL AllocateAllowed
 900 **          State for allocation if out of free events.
 901 **
 902 **  OUTPUT:
 903 **
 904 **      Nothing.
 905 */
 906 gceSTATUS
 907 gckEVENT_AddList(
 908     IN gckEVENT Event,
 909     IN gcsHAL_INTERFACE_PTR Interface,
 910     IN gceKERNEL_WHERE FromWhere,
 911     IN gctBOOL AllocateAllowed,
 912     IN gctBOOL FromKernel
 913     )
 914 {
 915     gceSTATUS status;
 916     gctBOOL acquired = gcvFALSE;
 917     gcsEVENT_PTR record = gcvNULL;
 918     gcsEVENT_QUEUE_PTR queue;
 919     gckKERNEL kernel = Event->kernel;
 920
 921     gcmkHEADER_ARG("Event=0x%x Interface=0x%x",
 922                    Event, Interface);
 923
 924     gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, _GC_OBJ_ZONE,
 925                     "FromWhere=%d AllocateAllowed=%d",
 926                     FromWhere, AllocateAllowed);
 927
 928     /* Verify the arguments. */
 929     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
 930     gcmkVERIFY_ARGUMENT(Interface != gcvNULL);
 931
 932     /* Verify the event command. */
 933     gcmkASSERT
 934         (  (Interface->command == gcvHAL_FREE_NON_PAGED_MEMORY)
 935         || (Interface->command == gcvHAL_FREE_CONTIGUOUS_MEMORY)
 936         || (Interface->command == gcvHAL_FREE_VIDEO_MEMORY)
 937         || (Interface->command == gcvHAL_WRITE_DATA)
 938         || (Interface->command == gcvHAL_UNLOCK_VIDEO_MEMORY)
 939         || (Interface->command == gcvHAL_SIGNAL)
 940         || (Interface->command == gcvHAL_UNMAP_USER_MEMORY)
 941         || (Interface->command == gcvHAL_TIMESTAMP)
 942         || (Interface->command == gcvHAL_COMMIT_DONE)
 943         || (Interface->command == gcvHAL_FREE_VIRTUAL_COMMAND_BUFFER)
 944         || (Interface->command == gcvHAL_SYNC_POINT)
 945         );
 946
 947     /* Validate the source. */
 948     if ((FromWhere != gcvKERNEL_COMMAND) && (FromWhere != gcvKERNEL_PIXEL))
 949     {
 950         /* Invalid argument. */
 951         gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
 952     }
 953
 954     /* Allocate a free record. */
 955     gcmkONERROR(gckEVENT_AllocateRecord(Event, AllocateAllowed, &record));
 956
 957     /* Termninate the record. */
 958     record->next = gcvNULL;
 959
 960     /* Record the committer. */
 961     record->fromKernel = FromKernel;
 962
 963     /* Copy the event interface into the record. */
 964     gckOS_MemCopy(&record->info, Interface, gcmSIZEOF(record->info));
 965
 966     /* Get process ID. */
 967     gcmkONERROR(gckOS_GetProcessID(&record->processID));
 968
 969 #ifdef __QNXNTO__
 970     record->kernel = Event->kernel;
 971 #endif
 972
 973     gcmkONERROR(__RemoveRecordFromProcessDB(Event, record));
 974
 975     /* Acquire the mutex. */
 976     gcmkONERROR(gckOS_AcquireMutex(Event->os, Event->eventListMutex, gcvINFINITE));
 977     acquired = gcvTRUE;
 978
 979     /* Do we need to allocate a new queue? */
 980     if ((Event->queueTail == gcvNULL) || (Event->queueTail->source < FromWhere))
 981     {
 982         /* Allocate a new queue. */
 983         gcmkONERROR(gckEVENT_AllocateQueue(Event, &queue));
 984
 985         /* Initialize the queue. */
 986         queue->source = FromWhere;
 987         queue->head   = gcvNULL;
 988         queue->next   = gcvNULL;
 989
 990         /* Attach it to the list of allocated queues. */
 991         if (Event->queueTail == gcvNULL)
 992         {
 993             Event->queueHead =
 994             Event->queueTail = queue;
 995         }
 996         else
 997         {
 998             Event->queueTail->next = queue;
 999             Event->queueTail       = queue;
1000         }
1001     }
1002     else
1003     {
1004         queue = Event->queueTail;
1005     }
1006
1007     /* Attach the record to the queue. */
1008     if (queue->head == gcvNULL)
1009     {
1010         queue->head = record;
1011         queue->tail = record;
1012     }
1013     else
1014     {
1015         queue->tail->next = record;
1016         queue->tail       = record;
1017     }
1018
1019     /* Unmap user space logical address.
1020      * Linux kernel does not support unmap the memory of other process any more since 3.5.
1021      * Let's unmap memory of self process before submit the event to gpu.
1022      * */
1023     switch(Interface->command)
1024     {
1025     case gcvHAL_FREE_NON_PAGED_MEMORY:
1026         gcmkONERROR(gckOS_UnmapUserLogical(
1027                         Event->os,
1028                         gcmNAME_TO_PTR(Interface->u.FreeNonPagedMemory.physical),
1029                         (gctSIZE_T) Interface->u.FreeNonPagedMemory.bytes,
1030                         gcmUINT64_TO_PTR(Interface->u.FreeNonPagedMemory.logical)));
1031         break;
1032     case gcvHAL_FREE_CONTIGUOUS_MEMORY:
1033         gcmkONERROR(gckOS_UnmapUserLogical(
1034                         Event->os,
1035                         gcmNAME_TO_PTR(Interface->u.FreeContiguousMemory.physical),
1036                         (gctSIZE_T) Interface->u.FreeContiguousMemory.bytes,
1037                         gcmUINT64_TO_PTR(Interface->u.FreeContiguousMemory.logical)));
1038         break;
1039     default:
1040         break;
1041     }
1042
1043
1044     /* Release the mutex. */
1045     gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventListMutex));
1046
1047     /* Success. */
1048     gcmkFOOTER_NO();
1049     return gcvSTATUS_OK;
1050
1051 OnError:
1052     /* Roll back. */
1053     if (acquired)
1054     {
1055         gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventListMutex));
1056     }
1057
1058     if (record != gcvNULL)
1059     {
1060         gcmkVERIFY_OK(gckEVENT_FreeRecord(Event, record));
1061     }
1062
1063     /* Return the status. */
1064     gcmkFOOTER();
1065     return status;
1066 }
1067
1068 /*******************************************************************************
1069 **
1070 **  gckEVENT_Unlock
1071 **
1072 **  Schedule an event to unlock virtual memory.
1073 **
1074 **  INPUT:
1075 **
1076 **      gckEVENT Event
1077 **          Pointer to an gckEVENT object.
1078 **
1079 **      gceKERNEL_WHERE FromWhere
1080 **          Place in the pipe where the event needs to be generated.
1081 **
1082 **      gcuVIDMEM_NODE_PTR Node
1083 **          Pointer to a gcuVIDMEM_NODE union that specifies the virtual memory
1084 **          to unlock.
1085 **
1086 **      gceSURF_TYPE Type
1087 **          Type of surface to unlock.
1088 **
1089 **  OUTPUT:
1090 **
1091 **      Nothing.
1092 */
1093 gceSTATUS
1094 gckEVENT_Unlock(
1095     IN gckEVENT Event,
1096     IN gceKERNEL_WHERE FromWhere,
1097     IN gcuVIDMEM_NODE_PTR Node,
1098     IN gceSURF_TYPE Type
1099     )
1100 {
1101     gceSTATUS status;
1102     gcsHAL_INTERFACE iface;
1103
1104     gcmkHEADER_ARG("Event=0x%x FromWhere=%d Node=0x%x Type=%d",
1105                    Event, FromWhere, Node, Type);
1106
1107     /* Verify the arguments. */
1108     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
1109     gcmkVERIFY_ARGUMENT(Node != gcvNULL);
1110
1111     /* Mark the event as an unlock. */
1112     iface.command                           = gcvHAL_UNLOCK_VIDEO_MEMORY;
1113     iface.u.UnlockVideoMemory.node          = gcmPTR_TO_UINT64(Node);
1114     iface.u.UnlockVideoMemory.type          = Type;
1115     iface.u.UnlockVideoMemory.asynchroneous = 0;
1116
1117     /* Append it to the queue. */
1118     gcmkONERROR(gckEVENT_AddList(Event, &iface, FromWhere, gcvFALSE, gcvTRUE));
1119
1120     /* Success. */
1121     gcmkFOOTER_NO();
1122     return gcvSTATUS_OK;
1123
1124 OnError:
1125     /* Return the status. */
1126     gcmkFOOTER();
1127     return status;
1128 }
1129
1130 /*******************************************************************************
1131 **
1132 **  gckEVENT_FreeVideoMemory
1133 **
1134 **  Schedule an event to free video memory.
1135 **
1136 **  INPUT:
1137 **
1138 **      gckEVENT Event
1139 **          Pointer to an gckEVENT object.
1140 **
1141 **      gcuVIDMEM_NODE_PTR VideoMemory
1142 **          Pointer to a gcuVIDMEM_NODE object to free.
1143 **
1144 **      gceKERNEL_WHERE FromWhere
1145 **          Place in the pipe where the event needs to be generated.
1146 **
1147 **  OUTPUT:
1148 **
1149 **      Nothing.
1150 */
1151 gceSTATUS
1152 gckEVENT_FreeVideoMemory(
1153     IN gckEVENT Event,
1154     IN gcuVIDMEM_NODE_PTR VideoMemory,
1155     IN gceKERNEL_WHERE FromWhere
1156     )
1157 {
1158     gceSTATUS status;
1159     gcsHAL_INTERFACE iface;
1160
1161     gcmkHEADER_ARG("Event=0x%x VideoMemory=0x%x FromWhere=%d",
1162                    Event, VideoMemory, FromWhere);
1163
1164     /* Verify the arguments. */
1165     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
1166     gcmkVERIFY_ARGUMENT(VideoMemory != gcvNULL);
1167
1168     /* Create an event. */
1169     iface.command = gcvHAL_FREE_VIDEO_MEMORY;
1170     iface.u.FreeVideoMemory.node = gcmPTR_TO_UINT64(VideoMemory);
1171
1172     /* Append it to the queue. */
1173     gcmkONERROR(gckEVENT_AddList(Event, &iface, FromWhere, gcvFALSE, gcvTRUE));
1174
1175     /* Success. */
1176     gcmkFOOTER_NO();
1177     return gcvSTATUS_OK;
1178
1179 OnError:
1180     /* Return the status. */
1181     gcmkFOOTER();
1182     return status;
1183 }
1184
1185 /*******************************************************************************
1186 **
1187 **  gckEVENT_FreeNonPagedMemory
1188 **
1189 **  Schedule an event to free non-paged memory.
1190 **
1191 **  INPUT:
1192 **
1193 **      gckEVENT Event
1194 **          Pointer to an gckEVENT object.
1195 **
1196 **      gctSIZE_T Bytes
1197 **          Number of bytes of non-paged memory to free.
1198 **
1199 **      gctPHYS_ADDR Physical
1200 **          Physical address of non-paged memory to free.
1201 **
1202 **      gctPOINTER Logical
1203 **          Logical address of non-paged memory to free.
1204 **
1205 **      gceKERNEL_WHERE FromWhere
1206 **          Place in the pipe where the event needs to be generated.
1207 */
1208 gceSTATUS
1209 gckEVENT_FreeNonPagedMemory(
1210     IN gckEVENT Event,
1211     IN gctSIZE_T Bytes,
1212     IN gctPHYS_ADDR Physical,
1213     IN gctPOINTER Logical,
1214     IN gceKERNEL_WHERE FromWhere
1215     )
1216 {
1217     gceSTATUS status;
1218     gcsHAL_INTERFACE iface;
1219     gckKERNEL kernel = Event->kernel;
1220
1221     gcmkHEADER_ARG("Event=0x%x Bytes=%lu Physical=0x%x Logical=0x%x "
1222                    "FromWhere=%d",
1223                    Event, Bytes, Physical, Logical, FromWhere);
1224
1225     /* Verify the arguments. */
1226     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
1227     gcmkVERIFY_ARGUMENT(Physical != gcvNULL);
1228     gcmkVERIFY_ARGUMENT(Logical != gcvNULL);
1229     gcmkVERIFY_ARGUMENT(Bytes > 0);
1230
1231     /* Create an event. */
1232     iface.command = gcvHAL_FREE_NON_PAGED_MEMORY;
1233     iface.u.FreeNonPagedMemory.bytes    = Bytes;
1234     iface.u.FreeNonPagedMemory.physical = gcmPTR_TO_NAME(Physical);
1235     iface.u.FreeNonPagedMemory.logical  = gcmPTR_TO_UINT64(Logical);
1236
1237     /* Append it to the queue. */
1238     gcmkONERROR(gckEVENT_AddList(Event, &iface, FromWhere, gcvFALSE, gcvTRUE));
1239
1240     /* Success. */
1241     gcmkFOOTER_NO();
1242     return gcvSTATUS_OK;
1243
1244 OnError:
1245     /* Return the status. */
1246     gcmkFOOTER();
1247     return status;
1248 }
1249
1250 gceSTATUS
1251 gckEVENT_DestroyVirtualCommandBuffer(
1252     IN gckEVENT Event,
1253     IN gctSIZE_T Bytes,
1254     IN gctPHYS_ADDR Physical,
1255     IN gctPOINTER Logical,
1256     IN gceKERNEL_WHERE FromWhere
1257     )
1258 {
1259     gceSTATUS status;
1260     gcsHAL_INTERFACE iface;
1261     gckKERNEL kernel = Event->kernel;
1262
1263     gcmkHEADER_ARG("Event=0x%x Bytes=%lu Physical=0x%x Logical=0x%x "
1264                    "FromWhere=%d",
1265                    Event, Bytes, Physical, Logical, FromWhere);
1266
1267     /* Verify the arguments. */
1268     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
1269     gcmkVERIFY_ARGUMENT(Physical != gcvNULL);
1270     gcmkVERIFY_ARGUMENT(Logical != gcvNULL);
1271     gcmkVERIFY_ARGUMENT(Bytes > 0);
1272
1273     /* Create an event. */
1274     iface.command = gcvHAL_FREE_VIRTUAL_COMMAND_BUFFER;
1275     iface.u.FreeVirtualCommandBuffer.bytes    = Bytes;
1276     iface.u.FreeVirtualCommandBuffer.physical = gcmPTR_TO_NAME(Physical);
1277     iface.u.FreeVirtualCommandBuffer.logical  = gcmPTR_TO_UINT64(Logical);
1278
1279     /* Append it to the queue. */
1280     gcmkONERROR(gckEVENT_AddList(Event, &iface, FromWhere, gcvFALSE, gcvTRUE));
1281
1282     /* Success. */
1283     gcmkFOOTER_NO();
1284     return gcvSTATUS_OK;
1285
1286 OnError:
1287     /* Return the status. */
1288     gcmkFOOTER();
1289     return status;
1290 }
1291
1292 /*******************************************************************************
1293 **
1294 **  gckEVENT_FreeContigiuousMemory
1295 **
1296 **  Schedule an event to free contiguous memory.
1297 **
1298 **  INPUT:
1299 **
1300 **      gckEVENT Event
1301 **          Pointer to an gckEVENT object.
1302 **
1303 **      gctSIZE_T Bytes
1304 **          Number of bytes of contiguous memory to free.
1305 **
1306 **      gctPHYS_ADDR Physical
1307 **          Physical address of contiguous memory to free.
1308 **
1309 **      gctPOINTER Logical
1310 **          Logical address of contiguous memory to free.
1311 **
1312 **      gceKERNEL_WHERE FromWhere
1313 **          Place in the pipe where the event needs to be generated.
1314 */
1315 gceSTATUS
1316 gckEVENT_FreeContiguousMemory(
1317     IN gckEVENT Event,
1318     IN gctSIZE_T Bytes,
1319     IN gctPHYS_ADDR Physical,
1320     IN gctPOINTER Logical,
1321     IN gceKERNEL_WHERE FromWhere
1322     )
1323 {
1324     gceSTATUS status;
1325     gcsHAL_INTERFACE iface;
1326     gckKERNEL kernel = Event->kernel;
1327
1328     gcmkHEADER_ARG("Event=0x%x Bytes=%lu Physical=0x%x Logical=0x%x "
1329                    "FromWhere=%d",
1330                    Event, Bytes, Physical, Logical, FromWhere);
1331
1332     /* Verify the arguments. */
1333     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
1334     gcmkVERIFY_ARGUMENT(Physical != gcvNULL);
1335     gcmkVERIFY_ARGUMENT(Logical != gcvNULL);
1336     gcmkVERIFY_ARGUMENT(Bytes > 0);
1337
1338     /* Create an event. */
1339     iface.command = gcvHAL_FREE_CONTIGUOUS_MEMORY;
1340     iface.u.FreeContiguousMemory.bytes    = Bytes;
1341     iface.u.FreeContiguousMemory.physical = gcmPTR_TO_NAME(Physical);
1342     iface.u.FreeContiguousMemory.logical  = gcmPTR_TO_UINT64(Logical);
1343
1344     /* Append it to the queue. */
1345     gcmkONERROR(gckEVENT_AddList(Event, &iface, FromWhere, gcvFALSE, gcvTRUE));
1346
1347     /* Success. */
1348     gcmkFOOTER_NO();
1349     return gcvSTATUS_OK;
1350
1351 OnError:
1352     /* Return the status. */
1353     gcmkFOOTER();
1354     return status;
1355 }
1356
1357 /*******************************************************************************
1358 **
1359 **  gckEVENT_Signal
1360 **
1361 **  Schedule an event to trigger a signal.
1362 **
1363 **  INPUT:
1364 **
1365 **      gckEVENT Event
1366 **          Pointer to an gckEVENT object.
1367 **
1368 **      gctSIGNAL Signal
1369 **          Pointer to the signal to trigger.
1370 **
1371 **      gceKERNEL_WHERE FromWhere
1372 **          Place in the pipe where the event needs to be generated.
1373 **
1374 **  OUTPUT:
1375 **
1376 **      Nothing.
1377 */
1378 gceSTATUS
1379 gckEVENT_Signal(
1380     IN gckEVENT Event,
1381     IN gctSIGNAL Signal,
1382     IN gceKERNEL_WHERE FromWhere
1383     )
1384 {
1385     gceSTATUS status;
1386     gcsHAL_INTERFACE iface;
1387
1388     gcmkHEADER_ARG("Event=0x%x Signal=0x%x FromWhere=%d",
1389                    Event, Signal, FromWhere);
1390
1391     /* Verify the arguments. */
1392     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
1393     gcmkVERIFY_ARGUMENT(Signal != gcvNULL);
1394
1395     /* Mark the event as a signal. */
1396     iface.command            = gcvHAL_SIGNAL;
1397     iface.u.Signal.signal    = gcmPTR_TO_UINT64(Signal);
1398 #ifdef __QNXNTO__
1399     iface.u.Signal.coid      = 0;
1400     iface.u.Signal.rcvid     = 0;
1401 #endif
1402     iface.u.Signal.auxSignal = 0;
1403     iface.u.Signal.process   = 0;
1404
1405     /* Append it to the queue. */
1406     gcmkONERROR(gckEVENT_AddList(Event, &iface, FromWhere, gcvFALSE, gcvTRUE));
1407
1408     /* Success. */
1409     gcmkFOOTER_NO();
1410     return gcvSTATUS_OK;
1411
1412 OnError:
1413     /* Return the status. */
1414     gcmkFOOTER();
1415     return status;
1416 }
1417
1418 /*******************************************************************************
1419 **
1420 **  gckEVENT_CommitDone
1421 **
1422 **  Schedule an event to wake up work thread when commit is done by GPU.
1423 **
1424 **  INPUT:
1425 **
1426 **      gckEVENT Event
1427 **          Pointer to an gckEVENT object.
1428 **
1429 **      gceKERNEL_WHERE FromWhere
1430 **          Place in the pipe where the event needs to be generated.
1431 **
1432 **  OUTPUT:
1433 **
1434 **      Nothing.
1435 */
1436 gceSTATUS
1437 gckEVENT_CommitDone(
1438     IN gckEVENT Event,
1439     IN gceKERNEL_WHERE FromWhere
1440     )
1441 {
1442     gceSTATUS status;
1443     gcsHAL_INTERFACE iface;
1444
1445     gcmkHEADER_ARG("Event=0x%x FromWhere=%d", Event, FromWhere);
1446
1447     /* Verify the arguments. */
1448     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
1449
1450     iface.command = gcvHAL_COMMIT_DONE;
1451
1452     /* Append it to the queue. */
1453     gcmkONERROR(gckEVENT_AddList(Event, &iface, FromWhere, gcvFALSE, gcvTRUE));
1454
1455     /* Success. */
1456     gcmkFOOTER_NO();
1457     return gcvSTATUS_OK;
1458
1459 OnError:
1460     /* Return the status. */
1461     gcmkFOOTER();
1462     return status;
1463 }
1464 /*******************************************************************************
1465 **
1466 **  gckEVENT_Submit
1467 **
1468 **  Submit the current event queue to the GPU.
1469 **
1470 **  INPUT:
1471 **
1472 **      gckEVENT Event
1473 **          Pointer to an gckEVENT object.
1474 **
1475 **      gctBOOL Wait
1476 **          Submit requires one vacant event; if Wait is set to not zero,
1477 **          and there are no vacant events at this time, the function will
1478 **          wait until an event becomes vacant so that submission of the
1479 **          queue is successful.
1480 **
1481 **      gctBOOL FromPower
1482 **          Determines whether the call originates from inside the power
1483 **          management or not.
1484 **
1485 **  OUTPUT:
1486 **
1487 **      Nothing.
1488 */
1489 gceSTATUS
1490 gckEVENT_Submit(
1491     IN gckEVENT Event,
1492     IN gctBOOL Wait,
1493     IN gctBOOL FromPower
1494     )
1495 {
1496     gceSTATUS status;
1497     gctUINT8 id = 0xFF;
1498     gcsEVENT_QUEUE_PTR queue;
1499     gctBOOL acquired = gcvFALSE;
1500     gckCOMMAND command = gcvNULL;
1501     gctBOOL commitEntered = gcvFALSE;
1502 #if !gcdNULL_DRIVER
1503     gctSIZE_T bytes;
1504     gctPOINTER buffer;
1505 #endif
1506
1507     gcmkHEADER_ARG("Event=0x%x Wait=%d", Event, Wait);
1508
1509     /* Get gckCOMMAND object. */
1510     command = Event->kernel->command;
1511
1512     /* Are there event queues? */
1513     if (Event->queueHead != gcvNULL)
1514     {
1515         /* Acquire the command queue. */
1516         gcmkONERROR(gckCOMMAND_EnterCommit(command, FromPower));
1517         commitEntered = gcvTRUE;
1518
1519         /* Process all queues. */
1520         while (Event->queueHead != gcvNULL)
1521         {
1522             /* Acquire the list mutex. */
1523             gcmkONERROR(gckOS_AcquireMutex(Event->os,
1524                                            Event->eventListMutex,
1525                                            gcvINFINITE));
1526             acquired = gcvTRUE;
1527
1528             /* Get the current queue. */
1529             queue = Event->queueHead;
1530
1531             /* Allocate an event ID. */
1532             gcmkONERROR(gckEVENT_GetEvent(Event, Wait, &id, queue->source));
1533
1534             /* Copy event list to event ID queue. */
1535             Event->queues[id].head   = queue->head;
1536
1537             /* Remove the top queue from the list. */
1538             if (Event->queueHead == Event->queueTail)
1539             {
1540                 Event->queueHead = gcvNULL;
1541                 Event->queueTail = gcvNULL;
1542             }
1543             else
1544             {
1545                 Event->queueHead = Event->queueHead->next;
1546             }
1547
1548             /* Free the queue. */
1549             gcmkONERROR(gckEVENT_FreeQueue(Event, queue));
1550
1551             /* Release the list mutex. */
1552             gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventListMutex));
1553             acquired = gcvFALSE;
1554
1555 #if gcdNULL_DRIVER
1556             /* Notify immediately on infinite hardware. */
1557             gcmkONERROR(gckEVENT_Interrupt(Event, 1 << id));
1558
1559             gcmkONERROR(gckEVENT_Notify(Event, 0));
1560 #else
1561             /* Get the size of the hardware event. */
1562             gcmkONERROR(gckHARDWARE_Event(Event->kernel->hardware,
1563                                           gcvNULL,
1564                                           id,
1565                                           Event->queues[id].source,
1566                                           &bytes));
1567
1568             /* Reserve space in the command queue. */
1569             gcmkONERROR(gckCOMMAND_Reserve(command,
1570                                            bytes,
1571                                            &buffer,
1572                                            &bytes));
1573
1574             /* Set the hardware event in the command queue. */
1575             gcmkONERROR(gckHARDWARE_Event(Event->kernel->hardware,
1576                                           buffer,
1577                                           id,
1578                                           Event->queues[id].source,
1579                                           &bytes));
1580
1581             /* Execute the hardware event. */
1582             gcmkONERROR(gckCOMMAND_Execute(command, bytes));
1583 #endif
1584         }
1585
1586         /* Release the command queue. */
1587         gcmkONERROR(gckCOMMAND_ExitCommit(command, FromPower));
1588         commitEntered = gcvFALSE;
1589
1590 #if !gcdNULL_DRIVER
1591         gcmkVERIFY_OK(_TryToIdleGPU(Event));
1592 #endif
1593     }
1594
1595     /* Success. */
1596     gcmkFOOTER_NO();
1597     return gcvSTATUS_OK;
1598
1599 OnError:
1600     if (commitEntered)
1601     {
1602         /* Release the command queue mutex. */
1603         gcmkVERIFY_OK(gckCOMMAND_ExitCommit(command, FromPower));
1604     }
1605
1606     if (acquired)
1607     {
1608         /* Need to unroll the mutex acquire. */
1609         gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventListMutex));
1610     }
1611
1612     if (id != 0xFF)
1613     {
1614         /* Need to unroll the event allocation. */
1615         Event->queues[id].head = gcvNULL;
1616     }
1617
1618     if (status == gcvSTATUS_GPU_NOT_RESPONDING)
1619     {
1620         /* Broadcast GPU stuck. */
1621         status = gckOS_Broadcast(Event->os,
1622                                  Event->kernel->hardware,
1623                                  gcvBROADCAST_GPU_STUCK);
1624     }
1625
1626     /* Return the status. */
1627     gcmkFOOTER();
1628     return status;
1629 }
1630
1631 /*******************************************************************************
1632 **
1633 **  gckEVENT_Commit
1634 **
1635 **  Commit an event queue from the user.
1636 **
1637 **  INPUT:
1638 **
1639 **      gckEVENT Event
1640 **          Pointer to an gckEVENT object.
1641 **
1642 **      gcsQUEUE_PTR Queue
1643 **          User event queue.
1644 **
1645 **  OUTPUT:
1646 **
1647 **      Nothing.
1648 */
1649 gceSTATUS
1650 gckEVENT_Commit(
1651     IN gckEVENT Event,
1652     IN gcsQUEUE_PTR Queue
1653     )
1654 {
1655     gceSTATUS status;
1656     gcsQUEUE_PTR record = gcvNULL, next;
1657     gctUINT32 processID;
1658     gctBOOL needCopy = gcvFALSE;
1659
1660     gcmkHEADER_ARG("Event=0x%x Queue=0x%x", Event, Queue);
1661
1662     /* Verify the arguments. */
1663     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
1664
1665     /* Get the current process ID. */
1666     gcmkONERROR(gckOS_GetProcessID(&processID));
1667
1668     /* Query if we need to copy the client data. */
1669     gcmkONERROR(gckOS_QueryNeedCopy(Event->os, processID, &needCopy));
1670
1671     /* Loop while there are records in the queue. */
1672     while (Queue != gcvNULL)
1673     {
1674         gcsQUEUE queue;
1675
1676         if (needCopy)
1677         {
1678             /* Point to stack record. */
1679             record = &queue;
1680
1681             /* Copy the data from the client. */
1682             gcmkONERROR(gckOS_CopyFromUserData(Event->os,
1683                                                record,
1684                                                Queue,
1685                                                gcmSIZEOF(gcsQUEUE)));
1686         }
1687         else
1688         {
1689             gctPOINTER pointer = gcvNULL;
1690
1691             /* Map record into kernel memory. */
1692             gcmkONERROR(gckOS_MapUserPointer(Event->os,
1693                                              Queue,
1694                                              gcmSIZEOF(gcsQUEUE),
1695                                              &pointer));
1696
1697             record = pointer;
1698         }
1699
1700         /* Append event record to event queue. */
1701         gcmkONERROR(
1702             gckEVENT_AddList(Event, &record->iface, gcvKERNEL_PIXEL, gcvTRUE, gcvFALSE));
1703
1704         /* Next record in the queue. */
1705         next = gcmUINT64_TO_PTR(record->next);
1706
1707         if (!needCopy)
1708         {
1709             /* Unmap record from kernel memory. */
1710             gcmkONERROR(
1711                 gckOS_UnmapUserPointer(Event->os,
1712                                        Queue,
1713                                        gcmSIZEOF(gcsQUEUE),
1714                                        (gctPOINTER *) record));
1715             record = gcvNULL;
1716         }
1717
1718         Queue = next;
1719     }
1720
1721     /* Submit the event list. */
1722     gcmkONERROR(gckEVENT_Submit(Event, gcvTRUE, gcvFALSE));
1723
1724     /* Success */
1725     gcmkFOOTER_NO();
1726     return gcvSTATUS_OK;
1727
1728 OnError:
1729     if ((record != gcvNULL) && !needCopy)
1730     {
1731         /* Roll back. */
1732         gcmkVERIFY_OK(gckOS_UnmapUserPointer(Event->os,
1733                                              Queue,
1734                                              gcmSIZEOF(gcsQUEUE),
1735                                              (gctPOINTER *) record));
1736     }
1737
1738     /* Return the status. */
1739     gcmkFOOTER();
1740     return status;
1741 }
1742
1743 /*******************************************************************************
1744 **
1745 **  gckEVENT_Compose
1746 **
1747 **  Schedule a composition event and start a composition.
1748 **
1749 **  INPUT:
1750 **
1751 **      gckEVENT Event
1752 **          Pointer to an gckEVENT object.
1753 **
1754 **      gcsHAL_COMPOSE_PTR Info
1755 **          Pointer to the composition structure.
1756 **
1757 **  OUTPUT:
1758 **
1759 **      Nothing.
1760 */
1761 gceSTATUS
1762 gckEVENT_Compose(
1763     IN gckEVENT Event,
1764     IN gcsHAL_COMPOSE_PTR Info
1765     )
1766 {
1767     gceSTATUS status;
1768     gcsEVENT_PTR headRecord;
1769     gcsEVENT_PTR tailRecord;
1770     gcsEVENT_PTR tempRecord;
1771     gctUINT8 id = 0xFF;
1772     gctUINT32 processID;
1773
1774     gcmkHEADER_ARG("Event=0x%x Info=0x%x", Event, Info);
1775
1776     /* Verify the arguments. */
1777     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
1778     gcmkVERIFY_ARGUMENT(Info != gcvNULL);
1779
1780     /* Allocate an event ID. */
1781     gcmkONERROR(gckEVENT_GetEvent(Event, gcvTRUE, &id, gcvKERNEL_PIXEL));
1782
1783     /* Get process ID. */
1784     gcmkONERROR(gckOS_GetProcessID(&processID));
1785
1786     /* Allocate a record. */
1787     gcmkONERROR(gckEVENT_AllocateRecord(Event, gcvTRUE, &tempRecord));
1788     headRecord = tailRecord = tempRecord;
1789
1790     /* Initialize the record. */
1791     tempRecord->info.command            = gcvHAL_SIGNAL;
1792     tempRecord->info.u.Signal.process   = Info->process;
1793 #ifdef __QNXNTO__
1794     tempRecord->info.u.Signal.coid      = Info->coid;
1795     tempRecord->info.u.Signal.rcvid     = Info->rcvid;
1796 #endif
1797     tempRecord->info.u.Signal.signal    = Info->signal;
1798     tempRecord->info.u.Signal.auxSignal = 0;
1799     tempRecord->next = gcvNULL;
1800     tempRecord->processID = processID;
1801
1802     /* Allocate another record for user signal #1. */
1803     if (gcmUINT64_TO_PTR(Info->userSignal1) != gcvNULL)
1804     {
1805         /* Allocate a record. */
1806         gcmkONERROR(gckEVENT_AllocateRecord(Event, gcvTRUE, &tempRecord));
1807         tailRecord->next = tempRecord;
1808         tailRecord = tempRecord;
1809
1810         /* Initialize the record. */
1811         tempRecord->info.command            = gcvHAL_SIGNAL;
1812         tempRecord->info.u.Signal.process   = Info->userProcess;
1813 #ifdef __QNXNTO__
1814         tempRecord->info.u.Signal.coid      = Info->coid;
1815         tempRecord->info.u.Signal.rcvid     = Info->rcvid;
1816 #endif
1817         tempRecord->info.u.Signal.signal    = Info->userSignal1;
1818         tempRecord->info.u.Signal.auxSignal = 0;
1819         tempRecord->next = gcvNULL;
1820         tempRecord->processID = processID;
1821     }
1822
1823     /* Allocate another record for user signal #2. */
1824     if (gcmUINT64_TO_PTR(Info->userSignal2) != gcvNULL)
1825     {
1826         /* Allocate a record. */
1827         gcmkONERROR(gckEVENT_AllocateRecord(Event, gcvTRUE, &tempRecord));
1828         tailRecord->next = tempRecord;
1829         tailRecord = tempRecord;
1830
1831         /* Initialize the record. */
1832         tempRecord->info.command            = gcvHAL_SIGNAL;
1833         tempRecord->info.u.Signal.process   = Info->userProcess;
1834 #ifdef __QNXNTO__
1835         tempRecord->info.u.Signal.coid      = Info->coid;
1836         tempRecord->info.u.Signal.rcvid     = Info->rcvid;
1837 #endif
1838         tempRecord->info.u.Signal.signal    = Info->userSignal2;
1839         tempRecord->info.u.Signal.auxSignal = 0;
1840         tempRecord->next = gcvNULL;
1841         tempRecord->processID = processID;
1842     }
1843
1844         /* Set the event list. */
1845     Event->queues[id].head = headRecord;
1846
1847     /* Start composition. */
1848     gcmkONERROR(gckHARDWARE_Compose(
1849         Event->kernel->hardware, processID,
1850         gcmUINT64_TO_PTR(Info->physical), gcmUINT64_TO_PTR(Info->logical), Info->offset, Info->size, id
1851         ));
1852
1853     /* Success. */
1854     gcmkFOOTER_NO();
1855     return gcvSTATUS_OK;
1856
1857 OnError:
1858     /* Return the status. */
1859     gcmkFOOTER();
1860     return status;
1861 }
1862
1863 /*******************************************************************************
1864 **
1865 **  gckEVENT_Interrupt
1866 **
1867 **  Called by the interrupt service routine to store the triggered interrupt
1868 **  mask to be later processed by gckEVENT_Notify.
1869 **
1870 **  INPUT:
1871 **
1872 **      gckEVENT Event
1873 **          Pointer to an gckEVENT object.
1874 **
1875 **      gctUINT32 Data
1876 **          Mask for the 32 interrupts.
1877 **
1878 **  OUTPUT:
1879 **
1880 **      Nothing.
1881 */
1882 gceSTATUS
1883 gckEVENT_Interrupt(
1884     IN gckEVENT Event,
1885     IN gctUINT32 Data
1886     )
1887 {
1888     gcmkHEADER_ARG("Event=0x%x Data=0x%x", Event, Data);
1889
1890     /* Verify the arguments. */
1891     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
1892
1893     /* Combine current interrupt status with pending flags. */
1894 #if gcdSMP
1895     gckOS_AtomSetMask(Event->pending, Data);
1896 #elif defined(__QNXNTO__)
1897     atomic_set(&Event->pending, Data);
1898 #else
1899     Event->pending |= Data;
1900 #endif
1901
1902     /* Success. */
1903     gcmkFOOTER_NO();
1904     return gcvSTATUS_OK;
1905 }
1906
1907 /*******************************************************************************
1908 **
1909 **  gckEVENT_Notify
1910 **
1911 **  Process all triggered interrupts.
1912 **
1913 **  INPUT:
1914 **
1915 **      gckEVENT Event
1916 **          Pointer to an gckEVENT object.
1917 **
1918 **  OUTPUT:
1919 **
1920 **      Nothing.
1921 */
1922 gceSTATUS
1923 gckEVENT_Notify(
1924     IN gckEVENT Event,
1925     IN gctUINT32 IDs
1926     )
1927 {
1928     gceSTATUS status = gcvSTATUS_OK;
1929     gctINT i;
1930     gcsEVENT_QUEUE * queue;
1931     gctUINT mask = 0;
1932     gctBOOL acquired = gcvFALSE;
1933     gcuVIDMEM_NODE_PTR node;
1934     gctPOINTER info;
1935     gctSIGNAL signal;
1936     gctUINT pending;
1937     gckKERNEL kernel = Event->kernel;
1938 #if !gcdSMP
1939     gctBOOL suspended = gcvFALSE;
1940 #endif
1941 #if gcmIS_DEBUG(gcdDEBUG_TRACE)
1942     gctINT eventNumber = 0;
1943 #endif
1944     gctINT32 free;
1945 #if gcdSECURE_USER
1946     gcskSECURE_CACHE_PTR cache;
1947 #endif
1948
1949     gcmkHEADER_ARG("Event=0x%x IDs=0x%x", Event, IDs);
1950
1951     /* Verify the arguments. */
1952     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
1953
1954     gcmDEBUG_ONLY(
1955         if (IDs != 0)
1956         {
1957             for (i = 0; i < gcmCOUNTOF(Event->queues); ++i)
1958             {
1959                 if (Event->queues[i].head != gcvNULL)
1960                 {
1961                     gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT,
1962                                    "Queue(%d): stamp=%llu source=%d",
1963                                    i,
1964                                    Event->queues[i].stamp,
1965                                    Event->queues[i].source);
1966                 }
1967             }
1968         }
1969     );
1970
1971     for (;;)
1972     {
1973         gcsEVENT_PTR record;
1974
1975 #if gcdSMP
1976         /* Get current interrupts. */
1977         gckOS_AtomGet(Event->os, Event->pending, (gctINT32_PTR)&pending);
1978 #else
1979         /* Suspend interrupts. */
1980         gcmkONERROR(gckOS_SuspendInterruptEx(Event->os, Event->kernel->core));
1981         suspended = gcvTRUE;
1982
1983         /* Get current interrupts. */
1984         pending = Event->pending;
1985
1986         /* Resume interrupts. */
1987         gcmkONERROR(gckOS_ResumeInterruptEx(Event->os, Event->kernel->core));
1988         suspended = gcvFALSE;
1989 #endif
1990
1991         if (pending == 0)
1992         {
1993             /* No more pending interrupts - done. */
1994             break;
1995         }
1996
1997         if (pending & 0x80000000)
1998         {
1999             gckOS_Print("!!!!!!!!!!!!! AXI BUS ERROR !!!!!!!!!!!!!\n");
2000             gcmkTRACE_ZONE(gcvLEVEL_ERROR, gcvZONE_EVENT, "AXI BUS ERROR");
2001             pending &= 0x7FFFFFFF;
2002         }
2003
2004         if (pending & 0x40000000)
2005         {
2006             gckHARDWARE_DumpMMUException(Event->kernel->hardware);
2007
2008             pending &= 0xBFFFFFFF;
2009         }
2010
2011         gcmkTRACE_ZONE_N(
2012             gcvLEVEL_INFO, gcvZONE_EVENT,
2013             gcmSIZEOF(pending),
2014             "Pending interrupts 0x%x",
2015             pending
2016             );
2017
2018         queue = gcvNULL;
2019
2020         gcmDEBUG_ONLY(
2021             if (IDs == 0)
2022             {
2023                 for (i = 0; i < gcmCOUNTOF(Event->queues); ++i)
2024                 {
2025                     if (Event->queues[i].head != gcvNULL)
2026                     {
2027                         gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT,
2028                                        "Queue(%d): stamp=%llu source=%d",
2029                                        i,
2030                                        Event->queues[i].stamp,
2031                                        Event->queues[i].source);
2032                     }
2033                 }
2034             }
2035         );
2036
2037         /* Find the oldest pending interrupt. */
2038         for (i = 0; i < gcmCOUNTOF(Event->queues); ++i)
2039         {
2040             if ((Event->queues[i].head != gcvNULL)
2041             &&  (pending & (1 << i))
2042             )
2043             {
2044                 if ((queue == gcvNULL)
2045                 ||  (Event->queues[i].stamp < queue->stamp)
2046                 )
2047                 {
2048                     queue = &Event->queues[i];
2049                     mask  = 1 << i;
2050 #if gcmIS_DEBUG(gcdDEBUG_TRACE)
2051                     eventNumber = i;
2052 #endif
2053                 }
2054             }
2055         }
2056
2057         if (queue == gcvNULL)
2058         {
2059             gcmkTRACE_ZONE_N(
2060                 gcvLEVEL_ERROR, gcvZONE_EVENT,
2061                 gcmSIZEOF(pending),
2062                 "Interrupts 0x%x are not pending.",
2063                 pending
2064                 );
2065
2066 #if gcdSMP
2067             /* Mark pending interrupts as handled. */
2068             gckOS_AtomClearMask(Event->pending, pending);
2069 #elif defined(__QNXNTO__)
2070             /* Mark pending interrupts as handled. */
2071             atomic_clr((gctUINT32_PTR)&Event->pending, pending);
2072 #else
2073             /* Suspend interrupts. */
2074             gcmkONERROR(gckOS_SuspendInterruptEx(Event->os, Event->kernel->core));
2075             suspended = gcvTRUE;
2076
2077             /* Mark pending interrupts as handled. */
2078             Event->pending &= ~pending;
2079
2080             /* Resume interrupts. */
2081             gcmkONERROR(gckOS_ResumeInterruptEx(Event->os, Event->kernel->core));
2082             suspended = gcvFALSE;
2083 #endif
2084             break;
2085         }
2086
2087         /* Check whether there is a missed interrupt. */
2088         for (i = 0; i < gcmCOUNTOF(Event->queues); ++i)
2089         {
2090             if ((Event->queues[i].head != gcvNULL)
2091             &&  (Event->queues[i].stamp < queue->stamp)
2092             &&  (Event->queues[i].source <= queue->source)
2093             )
2094             {
2095                 gcmkTRACE_N(
2096                     gcvLEVEL_ERROR,
2097                     gcmSIZEOF(i) + gcmSIZEOF(Event->queues[i].stamp),
2098                     "Event %d lost (stamp %llu)",
2099                     i, Event->queues[i].stamp
2100                     );
2101
2102                 /* Use this event instead. */
2103                 queue = &Event->queues[i];
2104                 mask  = 0;
2105             }
2106         }
2107
2108         if (mask != 0)
2109         {
2110 #if gcmIS_DEBUG(gcdDEBUG_TRACE)
2111             gcmkTRACE_ZONE_N(
2112                 gcvLEVEL_INFO, gcvZONE_EVENT,
2113                 gcmSIZEOF(eventNumber),
2114                 "Processing interrupt %d",
2115                 eventNumber
2116                 );
2117 #endif
2118         }
2119
2120 #if gcdSMP
2121         /* Mark pending interrupt as handled. */
2122         gckOS_AtomClearMask(Event->pending, mask);
2123 #elif defined(__QNXNTO__)
2124         /* Mark pending interrupt as handled. */
2125         atomic_clr(&Event->pending, mask);
2126 #else
2127         /* Suspend interrupts. */
2128         gcmkONERROR(gckOS_SuspendInterruptEx(Event->os, Event->kernel->core));
2129         suspended = gcvTRUE;
2130
2131         /* Mark pending interrupt as handled. */
2132         Event->pending &= ~mask;
2133
2134         /* Resume interrupts. */
2135         gcmkONERROR(gckOS_ResumeInterruptEx(Event->os, Event->kernel->core));
2136         suspended = gcvFALSE;
2137 #endif
2138
2139         /* Grab the mutex queue. */
2140         gcmkONERROR(gckOS_AcquireMutex(Event->os,
2141                                        Event->eventQueueMutex,
2142                                        gcvINFINITE));
2143         acquired = gcvTRUE;
2144
2145         /* We are in the notify loop. */
2146         Event->inNotify = gcvTRUE;
2147
2148         /* Grab the event head. */
2149         record = queue->head;
2150
2151         /* Now quickly clear its event list. */
2152         queue->head = gcvNULL;
2153
2154         /* Release the mutex queue. */
2155         gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex));
2156         acquired = gcvFALSE;
2157
2158         /* Increase the number of free events. */
2159         gcmkONERROR(gckOS_AtomIncrement(Event->os, Event->freeAtom, &free));
2160
2161         /* Walk all events for this interrupt. */
2162         while (record != gcvNULL)
2163         {
2164             gcsEVENT_PTR recordNext;
2165 #ifndef __QNXNTO__
2166             gctPOINTER logical;
2167 #endif
2168 #if gcdSECURE_USER
2169             gctSIZE_T bytes;
2170 #endif
2171
2172             /* Grab next record. */
2173             recordNext = record->next;
2174
2175 #ifdef __QNXNTO__
2176             /* Assign record->processID as the pid for this galcore thread.
2177              * Used in OS calls like gckOS_UnlockMemory() which do not take a pid.
2178              */
2179             drv_thread_specific_key_assign(record->processID, 0, Event->kernel->core);
2180 #endif
2181
2182 #if gcdSECURE_USER
2183             /* Get the cache that belongs to this process. */
2184             gcmkONERROR(gckKERNEL_GetProcessDBCache(Event->kernel,
2185                         record->processID,
2186                         &cache));
2187 #endif
2188
2189             gcmkTRACE_ZONE_N(
2190                 gcvLEVEL_INFO, gcvZONE_EVENT,
2191                 gcmSIZEOF(record->info.command),
2192                 "Processing event type: %d",
2193                 record->info.command
2194                 );
2195
2196             switch (record->info.command)
2197             {
2198             case gcvHAL_FREE_NON_PAGED_MEMORY:
2199                 gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT,
2200                                "gcvHAL_FREE_NON_PAGED_MEMORY: 0x%x",
2201                                gcmNAME_TO_PTR(record->info.u.FreeNonPagedMemory.physical));
2202
2203                 /* Free non-paged memory. */
2204                 status = gckOS_FreeNonPagedMemory(
2205                             Event->os,
2206                             (gctSIZE_T) record->info.u.FreeNonPagedMemory.bytes,
2207                             gcmNAME_TO_PTR(record->info.u.FreeNonPagedMemory.physical),
2208                             gcmUINT64_TO_PTR(record->info.u.FreeNonPagedMemory.logical));
2209
2210                 if (gcmIS_SUCCESS(status))
2211                 {
2212 #if gcdSECURE_USER
2213                     gcmkVERIFY_OK(gckKERNEL_FlushTranslationCache(
2214                         Event->kernel,
2215                         cache,
2216                         gcmUINT64_TO_PTR(record->record.u.FreeNonPagedMemory.logical),
2217                         (gctSIZE_T) record->record.u.FreeNonPagedMemory.bytes));
2218 #endif
2219                 }
2220                 gcmRELEASE_NAME(record->info.u.FreeNonPagedMemory.physical);
2221                 break;
2222
2223             case gcvHAL_FREE_CONTIGUOUS_MEMORY:
2224                 gcmkTRACE_ZONE(
2225                     gcvLEVEL_VERBOSE, gcvZONE_EVENT,
2226                     "gcvHAL_FREE_CONTIGUOUS_MEMORY: 0x%x",
2227                     gcmNAME_TO_PTR(record->info.u.FreeContiguousMemory.physical));
2228
2229                 /* Unmap the user memory. */
2230                 status = gckOS_FreeContiguous(
2231                             Event->os,
2232                             gcmNAME_TO_PTR(record->info.u.FreeContiguousMemory.physical),
2233                             gcmUINT64_TO_PTR(record->info.u.FreeContiguousMemory.logical),
2234                             (gctSIZE_T) record->info.u.FreeContiguousMemory.bytes);
2235
2236                 if (gcmIS_SUCCESS(status))
2237                 {
2238 #if gcdSECURE_USER
2239                     gcmkVERIFY_OK(gckKERNEL_FlushTranslationCache(
2240                         Event->kernel,
2241                         cache,
2242                         gcmUINT64_TO_PTR(record->record.u.FreeContiguousMemory.logical),
2243                         (gctSIZE_T) record->record.u.FreeContiguousMemory.bytes));
2244 #endif
2245                 }
2246                 gcmRELEASE_NAME(record->info.u.FreeContiguousMemory.physical);
2247                 break;
2248
2249             case gcvHAL_FREE_VIDEO_MEMORY:
2250                 node = gcmUINT64_TO_PTR(record->info.u.FreeVideoMemory.node);
2251                 gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT,
2252                                "gcvHAL_FREE_VIDEO_MEMORY: 0x%x",
2253                                node);
2254 #ifdef __QNXNTO__
2255 #if gcdUSE_VIDMEM_PER_PID
2256                 /* Check if the VidMem object still exists. */
2257                 if (gckKERNEL_GetVideoMemoryPoolPid(record->kernel,
2258                                                     gcvPOOL_SYSTEM,
2259                                                     record->processID,
2260                                                     gcvNULL) == gcvSTATUS_NOT_FOUND)
2261                 {
2262                     /*printf("Vidmem not found for process:%d\n", queue->processID);*/
2263                     status = gcvSTATUS_OK;
2264                     break;
2265                 }
2266 #else
2267                 if ((node->VidMem.memory->object.type == gcvOBJ_VIDMEM)
2268                 &&  (node->VidMem.logical != gcvNULL)
2269                 )
2270                 {
2271                     gcmkERR_BREAK(
2272                         gckKERNEL_UnmapVideoMemory(record->kernel,
2273                                                    node->VidMem.logical,
2274                                                    record->processID,
2275                                                    node->VidMem.bytes));
2276                     node->VidMem.logical = gcvNULL;
2277                 }
2278 #endif
2279 #endif
2280
2281                 /* Free video memory. */
2282                 status =
2283                     gckVIDMEM_Free(node);
2284
2285                 break;
2286
2287             case gcvHAL_WRITE_DATA:
2288 #ifndef __QNXNTO__
2289                 /* Convert physical into logical address. */
2290                 gcmkERR_BREAK(
2291                     gckOS_MapPhysical(Event->os,
2292                                       record->info.u.WriteData.address,
2293                                       gcmSIZEOF(gctUINT32),
2294                                       &logical));
2295
2296                 /* Write data. */
2297                 gcmkERR_BREAK(
2298                     gckOS_WriteMemory(Event->os,
2299                                       logical,
2300                                       record->info.u.WriteData.data));
2301
2302                 /* Unmap the physical memory. */
2303                 gcmkERR_BREAK(
2304                     gckOS_UnmapPhysical(Event->os,
2305                                         logical,
2306                                         gcmSIZEOF(gctUINT32)));
2307 #else
2308                 /* Write data. */
2309                 gcmkERR_BREAK(
2310                     gckOS_WriteMemory(Event->os,
2311                                       (gctPOINTER)
2312                                           record->info.u.WriteData.address,
2313                                       record->info.u.WriteData.data));
2314 #endif
2315                 break;
2316
2317             case gcvHAL_UNLOCK_VIDEO_MEMORY:
2318                 node = gcmUINT64_TO_PTR(record->info.u.UnlockVideoMemory.node);
2319
2320                 gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT,
2321                                "gcvHAL_UNLOCK_VIDEO_MEMORY: 0x%x",
2322                                node);
2323
2324                 /* Save node information before it disappears. */
2325 #if gcdSECURE_USER
2326                 if (node->VidMem.memory->object.type == gcvOBJ_VIDMEM)
2327                 {
2328                     logical = gcvNULL;
2329                     bytes   = 0;
2330                 }
2331                 else
2332                 {
2333                     logical = node->Virtual.logical;
2334                     bytes   = node->Virtual.bytes;
2335                 }
2336 #endif
2337
2338                 /* Unlock. */
2339                 status = gckVIDMEM_Unlock(
2340                     Event->kernel,
2341                     node,
2342                     record->info.u.UnlockVideoMemory.type,
2343                     gcvNULL);
2344
2345 #if gcdSECURE_USER
2346                 if (gcmIS_SUCCESS(status) && (logical != gcvNULL))
2347                 {
2348                     gcmkVERIFY_OK(gckKERNEL_FlushTranslationCache(
2349                         Event->kernel,
2350                         cache,
2351                         logical,
2352                         bytes));
2353                 }
2354 #endif
2355                 break;
2356
2357             case gcvHAL_SIGNAL:
2358                 signal = gcmUINT64_TO_PTR(record->info.u.Signal.signal);
2359                 gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT,
2360                                "gcvHAL_SIGNAL: 0x%x",
2361                                signal);
2362
2363 #ifdef __QNXNTO__
2364                 if ((record->info.u.Signal.coid == 0)
2365                 &&  (record->info.u.Signal.rcvid == 0)
2366                 )
2367                 {
2368                     /* Kernel signal. */
2369                     gcmkERR_BREAK(
2370                         gckOS_Signal(Event->os,
2371                                      signal,
2372                                      gcvTRUE));
2373                 }
2374                 else
2375                 {
2376                     /* User signal. */
2377                     gcmkERR_BREAK(
2378                         gckOS_UserSignal(Event->os,
2379                                          signal,
2380                                          record->info.u.Signal.rcvid,
2381                                          record->info.u.Signal.coid));
2382                 }
2383 #else
2384                 /* Set signal. */
2385                 if (gcmUINT64_TO_PTR(record->info.u.Signal.process) == gcvNULL)
2386                 {
2387                     /* Kernel signal. */
2388                     gcmkERR_BREAK(
2389                         gckOS_Signal(Event->os,
2390                                      signal,
2391                                      gcvTRUE));
2392                 }
2393                 else
2394                 {
2395                     /* User signal. */
2396                     gcmkERR_BREAK(
2397                         gckOS_UserSignal(Event->os,
2398                                          signal,
2399                                          gcmUINT64_TO_PTR(record->info.u.Signal.process)));
2400                 }
2401
2402                 gcmkASSERT(record->info.u.Signal.auxSignal == 0);
2403 #endif
2404                 break;
2405
2406             case gcvHAL_UNMAP_USER_MEMORY:
2407                 info = gcmNAME_TO_PTR(record->info.u.UnmapUserMemory.info);
2408                 gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT,
2409                                "gcvHAL_UNMAP_USER_MEMORY: 0x%x",
2410                                info);
2411
2412                 /* Unmap the user memory. */
2413                 status = gckOS_UnmapUserMemory(
2414                     Event->os,
2415                     Event->kernel->core,
2416                     gcmUINT64_TO_PTR(record->info.u.UnmapUserMemory.memory),
2417                     (gctSIZE_T) record->info.u.UnmapUserMemory.size,
2418                     info,
2419                     record->info.u.UnmapUserMemory.address);
2420
2421 #if gcdSECURE_USER
2422                 if (gcmIS_SUCCESS(status))
2423                 {
2424                     gcmkVERIFY_OK(gckKERNEL_FlushTranslationCache(
2425                         Event->kernel,
2426                         cache,
2427                         gcmUINT64_TO_PTR(record->info.u.UnmapUserMemory.memory),
2428                         (gctSIZE_T) record->info.u.UnmapUserMemory.size));
2429                 }
2430 #endif
2431                 gcmRELEASE_NAME(record->info.u.UnmapUserMemory.info);
2432                 break;
2433
2434             case gcvHAL_TIMESTAMP:
2435                 gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT,
2436                                "gcvHAL_TIMESTAMP: %d %d",
2437                                record->info.u.TimeStamp.timer,
2438                                record->info.u.TimeStamp.request);
2439
2440                 /* Process the timestamp. */
2441                 switch (record->info.u.TimeStamp.request)
2442                 {
2443                 case 0:
2444                     status = gckOS_GetTime(&Event->kernel->timers[
2445                                            record->info.u.TimeStamp.timer].
2446                                            stopTime);
2447                     break;
2448
2449                 case 1:
2450                     status = gckOS_GetTime(&Event->kernel->timers[
2451                                            record->info.u.TimeStamp.timer].
2452                                            startTime);
2453                     break;
2454
2455                 default:
2456                     gcmkTRACE_ZONE_N(
2457                         gcvLEVEL_ERROR, gcvZONE_EVENT,
2458                         gcmSIZEOF(record->info.u.TimeStamp.request),
2459                         "Invalid timestamp request: %d",
2460                         record->info.u.TimeStamp.request
2461                         );
2462
2463                     status = gcvSTATUS_INVALID_ARGUMENT;
2464                     break;
2465                 }
2466                 break;
2467
2468 #if gcdVIRTUAL_COMMAND_BUFFER
2469              case gcvHAL_FREE_VIRTUAL_COMMAND_BUFFER:
2470                  gcmkVERIFY_OK(
2471                      gckKERNEL_DestroyVirtualCommandBuffer(Event->kernel,
2472                          (gctSIZE_T) record->info.u.FreeVirtualCommandBuffer.bytes,
2473                          gcmNAME_TO_PTR(record->info.u.FreeVirtualCommandBuffer.physical),
2474                          gcmUINT64_TO_PTR(record->info.u.FreeVirtualCommandBuffer.logical)
2475                          ));
2476                  gcmRELEASE_NAME(record->info.u.FreeVirtualCommandBuffer.physical);
2477                  break;
2478 #endif
2479
2480 #if gcdANDROID_NATIVE_FENCE_SYNC
2481             case gcvHAL_SYNC_POINT:
2482                 {
2483                     gctSYNC_POINT syncPoint;
2484
2485                     syncPoint = gcmUINT64_TO_PTR(record->info.u.SyncPoint.syncPoint);
2486                     status = gckOS_SignalSyncPoint(Event->os, syncPoint);
2487                 }
2488                 break;
2489 #endif
2490
2491             case gcvHAL_COMMIT_DONE:
2492                 break;
2493
2494             default:
2495                 /* Invalid argument. */
2496                 gcmkTRACE_ZONE_N(
2497                     gcvLEVEL_ERROR, gcvZONE_EVENT,
2498                     gcmSIZEOF(record->info.command),
2499                     "Unknown event type: %d",
2500                     record->info.command
2501                     );
2502
2503                 status = gcvSTATUS_INVALID_ARGUMENT;
2504                 break;
2505             }
2506
2507             /* Make sure there are no errors generated. */
2508             if (gcmIS_ERROR(status))
2509             {
2510                 gcmkTRACE_ZONE_N(
2511                     gcvLEVEL_WARNING, gcvZONE_EVENT,
2512                     gcmSIZEOF(status),
2513                     "Event produced status: %d(%s)",
2514                     status, gckOS_DebugStatus2Name(status));
2515             }
2516
2517             /* Free the event. */
2518             gcmkVERIFY_OK(gckEVENT_FreeRecord(Event, record));
2519
2520             /* Advance to next record. */
2521             record = recordNext;
2522         }
2523
2524         gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT,
2525                        "Handled interrupt 0x%x", mask);
2526     }
2527
2528     if (IDs == 0)
2529     {
2530         gcmkONERROR(_TryToIdleGPU(Event));
2531     }
2532
2533     /* We are out the notify loop. */
2534     Event->inNotify = gcvFALSE;
2535
2536     /* Success. */
2537     gcmkFOOTER_NO();
2538     return gcvSTATUS_OK;
2539
2540 OnError:
2541     if (acquired)
2542     {
2543         /* Release mutex. */
2544         gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex));
2545     }
2546
2547 #if !gcdSMP
2548     if (suspended)
2549     {
2550         /* Resume interrupts. */
2551         gcmkVERIFY_OK(gckOS_ResumeInterruptEx(Event->os, Event->kernel->core));
2552     }
2553 #endif
2554
2555     /* We are out the notify loop. */
2556     Event->inNotify = gcvFALSE;
2557
2558     /* Return the status. */
2559     gcmkFOOTER();
2560     return status;
2561 }
2562
2563 /*******************************************************************************
2564 **  gckEVENT_FreeProcess
2565 **
2566 **  Free all events owned by a particular process ID.
2567 **
2568 **  INPUT:
2569 **
2570 **      gckEVENT Event
2571 **          Pointer to an gckEVENT object.
2572 **
2573 **      gctUINT32 ProcessID
2574 **          Process ID of the process to be freed up.
2575 **
2576 **  OUTPUT:
2577 **
2578 **      Nothing.
2579 */
2580 gceSTATUS
2581 gckEVENT_FreeProcess(
2582     IN gckEVENT Event,
2583     IN gctUINT32 ProcessID
2584     )
2585 {
2586     gctSIZE_T i;
2587     gctBOOL acquired = gcvFALSE;
2588     gcsEVENT_PTR record, next;
2589     gceSTATUS status;
2590     gcsEVENT_PTR deleteHead, deleteTail;
2591
2592     gcmkHEADER_ARG("Event=0x%x ProcessID=%d", Event, ProcessID);
2593
2594     /* Verify the arguments. */
2595     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
2596
2597     /* Walk through all queues. */
2598     for (i = 0; i < gcmCOUNTOF(Event->queues); ++i)
2599     {
2600         if (Event->queues[i].head != gcvNULL)
2601         {
2602             /* Grab the event queue mutex. */
2603             gcmkONERROR(gckOS_AcquireMutex(Event->os,
2604                                            Event->eventQueueMutex,
2605                                            gcvINFINITE));
2606             acquired = gcvTRUE;
2607
2608             /* Grab the mutex head. */
2609             record                = Event->queues[i].head;
2610             Event->queues[i].head = gcvNULL;
2611             Event->queues[i].tail = gcvNULL;
2612             deleteHead            = gcvNULL;
2613             deleteTail            = gcvNULL;
2614
2615             while (record != gcvNULL)
2616             {
2617                 next = record->next;
2618                 if (record->processID == ProcessID)
2619                 {
2620                     if (deleteHead == gcvNULL)
2621                     {
2622                         deleteHead = record;
2623                     }
2624                     else
2625                     {
2626                         deleteTail->next = record;
2627                     }
2628
2629                     deleteTail = record;
2630                 }
2631                 else
2632                 {
2633                     if (Event->queues[i].head == gcvNULL)
2634                     {
2635                         Event->queues[i].head = record;
2636                     }
2637                     else
2638                     {
2639                         Event->queues[i].tail->next = record;
2640                     }
2641
2642                     Event->queues[i].tail = record;
2643                 }
2644
2645                 record->next = gcvNULL;
2646                 record = next;
2647             }
2648
2649             /* Release the mutex queue. */
2650             gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex));
2651             acquired = gcvFALSE;
2652
2653             /* Loop through the entire list of events. */
2654             for (record = deleteHead; record != gcvNULL; record = next)
2655             {
2656                 /* Get the next event record. */
2657                 next = record->next;
2658
2659                 /* Free the event record. */
2660                 gcmkONERROR(gckEVENT_FreeRecord(Event, record));
2661             }
2662         }
2663     }
2664
2665     gcmkONERROR(_TryToIdleGPU(Event));
2666
2667     /* Success. */
2668     gcmkFOOTER_NO();
2669     return gcvSTATUS_OK;
2670
2671 OnError:
2672     /* Release the event queue mutex. */
2673     if (acquired)
2674     {
2675         gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex));
2676     }
2677
2678     /* Return the status. */
2679     gcmkFOOTER();
2680     return status;
2681 }
2682
2683 /*******************************************************************************
2684 **  gckEVENT_Stop
2685 **
2686 **  Stop the hardware using the End event mechanism.
2687 **
2688 **  INPUT:
2689 **
2690 **      gckEVENT Event
2691 **          Pointer to an gckEVENT object.
2692 **
2693 **      gctUINT32 ProcessID
2694 **          Process ID Logical belongs.
2695 **
2696 **      gctPHYS_ADDR Handle
2697 **          Physical address handle.  If gcvNULL it is video memory.
2698 **
2699 **      gctPOINTER Logical
2700 **          Logical address to flush.
2701 **
2702 **      gctSIGNAL Signal
2703 **          Pointer to the signal to trigger.
2704 **
2705 **  OUTPUT:
2706 **
2707 **      Nothing.
2708 */
2709 gceSTATUS
2710 gckEVENT_Stop(
2711     IN gckEVENT Event,
2712     IN gctUINT32 ProcessID,
2713     IN gctPHYS_ADDR Handle,
2714     IN gctPOINTER Logical,
2715     IN gctSIGNAL Signal,
2716         IN OUT gctSIZE_T * waitSize
2717     )
2718 {
2719     gceSTATUS status;
2720    /* gctSIZE_T waitSize;*/
2721     gcsEVENT_PTR record;
2722     gctUINT8 id = 0xFF;
2723
2724     gcmkHEADER_ARG("Event=0x%x ProcessID=%u Handle=0x%x Logical=0x%x "
2725                    "Signal=0x%x",
2726                    Event, ProcessID, Handle, Logical, Signal);
2727
2728     /* Verify the arguments. */
2729     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
2730
2731     /* Submit the current event queue. */
2732     gcmkONERROR(gckEVENT_Submit(Event, gcvTRUE, gcvFALSE));
2733
2734     gcmkONERROR(gckEVENT_GetEvent(Event, gcvTRUE, &id, gcvKERNEL_PIXEL));
2735
2736     /* Allocate a record. */
2737     gcmkONERROR(gckEVENT_AllocateRecord(Event, gcvTRUE, &record));
2738
2739     /* Initialize the record. */
2740     record->next = gcvNULL;
2741     record->processID               = ProcessID;
2742     record->info.command            = gcvHAL_SIGNAL;
2743     record->info.u.Signal.signal    = gcmPTR_TO_UINT64(Signal);
2744 #ifdef __QNXNTO__
2745     record->info.u.Signal.coid      = 0;
2746     record->info.u.Signal.rcvid     = 0;
2747 #endif
2748     record->info.u.Signal.auxSignal = 0;
2749     record->info.u.Signal.process   = 0;
2750
2751     /* Append the record. */
2752     Event->queues[id].head      = record;
2753
2754     /* Replace last WAIT with END. */
2755     gcmkONERROR(gckHARDWARE_End(
2756         Event->kernel->hardware, Logical, waitSize
2757         ));
2758
2759 #if gcdNONPAGED_MEMORY_CACHEABLE
2760     /* Flush the cache for the END. */
2761     gcmkONERROR(gckOS_CacheClean(
2762         Event->os,
2763         ProcessID,
2764         gcvNULL,
2765         Handle,
2766         Logical,
2767         *waitSize
2768         ));
2769 #endif
2770
2771     /* Wait for the signal. */
2772     gcmkONERROR(gckOS_WaitSignal(Event->os, Signal, gcvINFINITE));
2773
2774     /* Success. */
2775     gcmkFOOTER_NO();
2776     return gcvSTATUS_OK;
2777
2778 OnError:
2779
2780     /* Return the status. */
2781     gcmkFOOTER();
2782     return status;
2783 }
2784
2785 static void
2786 _PrintRecord(
2787     gcsEVENT_PTR record
2788     )
2789 {
2790     switch (record->info.command)
2791     {
2792     case gcvHAL_FREE_NON_PAGED_MEMORY:
2793         gcmkPRINT("      gcvHAL_FREE_NON_PAGED_MEMORY");
2794             break;
2795
2796     case gcvHAL_FREE_CONTIGUOUS_MEMORY:
2797         gcmkPRINT("      gcvHAL_FREE_CONTIGUOUS_MEMORY");
2798             break;
2799
2800     case gcvHAL_FREE_VIDEO_MEMORY:
2801         gcmkPRINT("      gcvHAL_FREE_VIDEO_MEMORY");
2802             break;
2803
2804     case gcvHAL_WRITE_DATA:
2805         gcmkPRINT("      gcvHAL_WRITE_DATA");
2806        break;
2807
2808     case gcvHAL_UNLOCK_VIDEO_MEMORY:
2809         gcmkPRINT("      gcvHAL_UNLOCK_VIDEO_MEMORY");
2810         break;
2811
2812     case gcvHAL_SIGNAL:
2813         gcmkPRINT("      gcvHAL_SIGNAL process=%d signal=0x%x",
2814                   record->info.u.Signal.process,
2815                   record->info.u.Signal.signal);
2816         break;
2817
2818     case gcvHAL_UNMAP_USER_MEMORY:
2819         gcmkPRINT("      gcvHAL_UNMAP_USER_MEMORY");
2820        break;
2821
2822     case gcvHAL_TIMESTAMP:
2823         gcmkPRINT("      gcvHAL_TIMESTAMP");
2824         break;
2825
2826     case gcvHAL_COMMIT_DONE:
2827         gcmkPRINT("      gcvHAL_COMMIT_DONE");
2828         break;
2829
2830     case gcvHAL_FREE_VIRTUAL_COMMAND_BUFFER:
2831         gcmkPRINT("      gcvHAL_FREE_VIRTUAL_COMMAND_BUFFER logical=0x%08x",
2832                   record->info.u.FreeVirtualCommandBuffer.logical);
2833         break;
2834
2835     default:
2836         gcmkPRINT("      Illegal Event %d", record->info.command);
2837         break;
2838     }
2839 }
2840
2841 /*******************************************************************************
2842 ** gckEVENT_Dump
2843 **
2844 ** Dump record in event queue when stuck happens.
2845 ** No protection for the event queue.
2846 **/
2847 gceSTATUS
2848 gckEVENT_Dump(
2849     IN gckEVENT Event
2850     )
2851 {
2852     gcsEVENT_QUEUE_PTR queueHead = Event->queueHead;
2853     gcsEVENT_QUEUE_PTR queue;
2854     gcsEVENT_PTR record = gcvNULL;
2855     gctINT i;
2856
2857     gcmkHEADER_ARG("Event=0x%x", Event);
2858
2859     gcmkPRINT("**************************\n");
2860     gcmkPRINT("***  EVENT STATE DUMP  ***\n");
2861     gcmkPRINT("**************************\n");
2862
2863
2864     gcmkPRINT("  Unsumbitted Event:");
2865     while(queueHead)
2866     {
2867         queue = queueHead;
2868         record = queueHead->head;
2869
2870         gcmkPRINT("    [%x]:", queue);
2871         while(record)
2872         {
2873             _PrintRecord(record);
2874             record = record->next;
2875         }
2876
2877         if (queueHead == Event->queueTail)
2878         {
2879             queueHead = gcvNULL;
2880         }
2881         else
2882         {
2883             queueHead = queueHead->next;
2884         }
2885     }
2886
2887     gcmkPRINT("  Untriggered Event:");
2888     for (i = 0; i < 30; i++)
2889     {
2890         queue = &Event->queues[i];
2891         record = queue->head;
2892
2893         gcmkPRINT("    [%d]:", i);
2894         while(record)
2895         {
2896             _PrintRecord(record);
2897             record = record->next;
2898         }
2899     }
2900
2901     gcmkFOOTER_NO();
2902     return gcvSTATUS_OK;
2903 }
2904
2905 gceSTATUS gckEVENT_WaitEmpty(gckEVENT Event)
2906 {
2907     gctBOOL isEmpty;
2908
2909     while (Event->inNotify || (gcmIS_SUCCESS(gckEVENT_IsEmpty(Event, &isEmpty)) && !isEmpty)) ;
2910
2911     return gcvSTATUS_OK;
2912 }