drivers/mxc/gpu-viv/hal/kernel/gc_hal_kernel_video_memory.c

   1 /****************************************************************************
   2 *
   3 *    Copyright (C) 2005 - 2013 by Vivante Corp.
   4 *
   5 *    This program is free software; you can redistribute it and/or modify
   6 *    it under the terms of the GNU General Public License as published by
   7 *    the Free Software Foundation; either version 2 of the license, or
   8 *    (at your option) any later version.
   9 *
  10 *    This program is distributed in the hope that it will be useful,
  11 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13 *    GNU General Public License for more details.
  14 *
  15 *    You should have received a copy of the GNU General Public License
  16 *    along with this program; if not write to the Free Software
  17 *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18 *
  19 *****************************************************************************/
  20
  21
  22 #include "gc_hal_kernel_precomp.h"
  23
  24 #define _GC_OBJ_ZONE    gcvZONE_VIDMEM
  25
  26 /******************************************************************************\
  27 ******************************* Private Functions ******************************
  28 \******************************************************************************/
  29
  30 /*******************************************************************************
  31 **
  32 **  _Split
  33 **
  34 **  Split a node on the required byte boundary.
  35 **
  36 **  INPUT:
  37 **
  38 **      gckOS Os
  39 **          Pointer to an gckOS object.
  40 **
  41 **      gcuVIDMEM_NODE_PTR Node
  42 **          Pointer to the node to split.
  43 **
  44 **      gctSIZE_T Bytes
  45 **          Number of bytes to keep in the node.
  46 **
  47 **  OUTPUT:
  48 **
  49 **      Nothing.
  50 **
  51 **  RETURNS:
  52 **
  53 **      gctBOOL
  54 **          gcvTRUE if the node was split successfully, or gcvFALSE if there is an
  55 **          error.
  56 **
  57 */
  58 static gctBOOL
  59 _Split(
  60     IN gckOS Os,
  61     IN gcuVIDMEM_NODE_PTR Node,
  62     IN gctSIZE_T Bytes
  63     )
  64 {
  65     gcuVIDMEM_NODE_PTR node;
  66     gctPOINTER pointer = gcvNULL;
  67
  68     /* Make sure the byte boundary makes sense. */
  69     if ((Bytes <= 0) || (Bytes > Node->VidMem.bytes))
  70     {
  71         return gcvFALSE;
  72     }
  73
  74     /* Allocate a new gcuVIDMEM_NODE object. */
  75     if (gcmIS_ERROR(gckOS_Allocate(Os,
  76                                    gcmSIZEOF(gcuVIDMEM_NODE),
  77                                    &pointer)))
  78     {
  79         /* Error. */
  80         return gcvFALSE;
  81     }
  82
  83     node = pointer;
  84
  85     /* Initialize gcuVIDMEM_NODE structure. */
  86     node->VidMem.offset    = Node->VidMem.offset + Bytes;
  87     node->VidMem.bytes     = Node->VidMem.bytes  - Bytes;
  88     node->VidMem.alignment = 0;
  89     node->VidMem.locked    = 0;
  90     node->VidMem.memory    = Node->VidMem.memory;
  91     node->VidMem.pool      = Node->VidMem.pool;
  92     node->VidMem.physical  = Node->VidMem.physical;
  93 #ifdef __QNXNTO__
  94 #if gcdUSE_VIDMEM_PER_PID
  95     gcmkASSERT(Node->VidMem.physical != 0);
  96     gcmkASSERT(Node->VidMem.logical != gcvNULL);
  97     node->VidMem.processID = Node->VidMem.processID;
  98     node->VidMem.physical  = Node->VidMem.physical + Bytes;
  99     node->VidMem.logical   = Node->VidMem.logical + Bytes;
 100 #else
 101     node->VidMem.processID = 0;
 102     node->VidMem.logical   = gcvNULL;
 103 #endif
 104 #endif
 105
 106     /* Insert node behind specified node. */
 107     node->VidMem.next = Node->VidMem.next;
 108     node->VidMem.prev = Node;
 109     Node->VidMem.next = node->VidMem.next->VidMem.prev = node;
 110
 111     /* Insert free node behind specified node. */
 112     node->VidMem.nextFree = Node->VidMem.nextFree;
 113     node->VidMem.prevFree = Node;
 114     Node->VidMem.nextFree = node->VidMem.nextFree->VidMem.prevFree = node;
 115
 116     /* Adjust size of specified node. */
 117     Node->VidMem.bytes = Bytes;
 118
 119     /* Success. */
 120     return gcvTRUE;
 121 }
 122
 123 /*******************************************************************************
 124 **
 125 **  _Merge
 126 **
 127 **  Merge two adjacent nodes together.
 128 **
 129 **  INPUT:
 130 **
 131 **      gckOS Os
 132 **          Pointer to an gckOS object.
 133 **
 134 **      gcuVIDMEM_NODE_PTR Node
 135 **          Pointer to the first of the two nodes to merge.
 136 **
 137 **  OUTPUT:
 138 **
 139 **      Nothing.
 140 **
 141 */
 142 static gceSTATUS
 143 _Merge(
 144     IN gckOS Os,
 145     IN gcuVIDMEM_NODE_PTR Node
 146     )
 147 {
 148     gcuVIDMEM_NODE_PTR node;
 149     gceSTATUS status;
 150
 151     /* Save pointer to next node. */
 152     node = Node->VidMem.next;
 153 #if gcdUSE_VIDMEM_PER_PID
 154     /* Check if the nodes are adjacent physically. */
 155     if ( ((Node->VidMem.physical + Node->VidMem.bytes) != node->VidMem.physical) ||
 156           ((Node->VidMem.logical + Node->VidMem.bytes) != node->VidMem.logical) )
 157     {
 158         /* Can't merge. */
 159         return gcvSTATUS_OK;
 160     }
 161 #else
 162
 163     /* This is a good time to make sure the heap is not corrupted. */
 164     if (Node->VidMem.offset + Node->VidMem.bytes != node->VidMem.offset)
 165     {
 166         /* Corrupted heap. */
 167         gcmkASSERT(
 168             Node->VidMem.offset + Node->VidMem.bytes == node->VidMem.offset);
 169         return gcvSTATUS_HEAP_CORRUPTED;
 170     }
 171 #endif
 172
 173     /* Adjust byte count. */
 174     Node->VidMem.bytes += node->VidMem.bytes;
 175
 176     /* Unlink next node from linked list. */
 177     Node->VidMem.next     = node->VidMem.next;
 178     Node->VidMem.nextFree = node->VidMem.nextFree;
 179
 180     Node->VidMem.next->VidMem.prev         =
 181     Node->VidMem.nextFree->VidMem.prevFree = Node;
 182
 183     /* Free next node. */
 184     status = gcmkOS_SAFE_FREE(Os, node);
 185     return status;
 186 }
 187
 188 /******************************************************************************\
 189 ******************************* gckVIDMEM API Code ******************************
 190 \******************************************************************************/
 191
 192 /*******************************************************************************
 193 **
 194 **  gckVIDMEM_ConstructVirtual
 195 **
 196 **  Construct a new gcuVIDMEM_NODE union for virtual memory.
 197 **
 198 **  INPUT:
 199 **
 200 **      gckKERNEL Kernel
 201 **          Pointer to an gckKERNEL object.
 202 **
 203 **      gctSIZE_T Bytes
 204 **          Number of byte to allocate.
 205 **
 206 **  OUTPUT:
 207 **
 208 **      gcuVIDMEM_NODE_PTR * Node
 209 **          Pointer to a variable that receives the gcuVIDMEM_NODE union pointer.
 210 */
 211 gceSTATUS
 212 gckVIDMEM_ConstructVirtual(
 213     IN gckKERNEL Kernel,
 214     IN gctBOOL Contiguous,
 215     IN gctSIZE_T Bytes,
 216     OUT gcuVIDMEM_NODE_PTR * Node
 217     )
 218 {
 219     gckOS os;
 220     gceSTATUS status;
 221     gcuVIDMEM_NODE_PTR node = gcvNULL;
 222     gctPOINTER pointer = gcvNULL;
 223     gctINT i;
 224
 225     gcmkHEADER_ARG("Kernel=0x%x Contiguous=%d Bytes=%lu", Kernel, Contiguous, Bytes);
 226
 227     /* Verify the arguments. */
 228     gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
 229     gcmkVERIFY_ARGUMENT(Bytes > 0);
 230     gcmkVERIFY_ARGUMENT(Node != gcvNULL);
 231
 232     /* Extract the gckOS object pointer. */
 233     os = Kernel->os;
 234     gcmkVERIFY_OBJECT(os, gcvOBJ_OS);
 235
 236     /* Allocate an gcuVIDMEM_NODE union. */
 237     gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcuVIDMEM_NODE), &pointer));
 238
 239     node = pointer;
 240
 241     /* Initialize gcuVIDMEM_NODE union for virtual memory. */
 242     node->Virtual.kernel        = Kernel;
 243     node->Virtual.contiguous    = Contiguous;
 244     node->Virtual.logical       = gcvNULL;
 245
 246     for (i = 0; i < gcdMAX_GPU_COUNT; i++)
 247     {
 248         node->Virtual.lockeds[i]        = 0;
 249         node->Virtual.pageTables[i]     = gcvNULL;
 250         node->Virtual.lockKernels[i]    = gcvNULL;
 251     }
 252
 253     node->Virtual.mutex         = gcvNULL;
 254
 255     gcmkONERROR(gckOS_GetProcessID(&node->Virtual.processID));
 256
 257 #ifdef __QNXNTO__
 258     node->Virtual.next          = gcvNULL;
 259     node->Virtual.freePending   = gcvFALSE;
 260     for (i = 0; i < gcdMAX_GPU_COUNT; i++)
 261     {
 262         node->Virtual.unlockPendings[i] = gcvFALSE;
 263     }
 264 #endif
 265
 266     node->Virtual.freed         = gcvFALSE;
 267
 268     gcmkONERROR(gckOS_ZeroMemory(&node->Virtual.sharedInfo, gcmSIZEOF(gcsVIDMEM_NODE_SHARED_INFO)));
 269
 270     /* Create the mutex. */
 271     gcmkONERROR(
 272         gckOS_CreateMutex(os, &node->Virtual.mutex));
 273
 274     /* Allocate the virtual memory. */
 275     gcmkONERROR(
 276         gckOS_AllocatePagedMemoryEx(os,
 277                                     node->Virtual.contiguous,
 278                                     node->Virtual.bytes = Bytes,
 279                                     &node->Virtual.physical));
 280
 281 #ifdef __QNXNTO__
 282     /* Register. */
 283 #if gcdENABLE_VG
 284     if (Kernel->core != gcvCORE_VG)
 285 #endif
 286     {
 287         gckMMU_InsertNode(Kernel->mmu, node);
 288     }
 289 #endif
 290
 291     /* Return pointer to the gcuVIDMEM_NODE union. */
 292     *Node = node;
 293
 294     gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
 295                    "Created virtual node 0x%x for %u bytes @ 0x%x",
 296                    node, Bytes, node->Virtual.physical);
 297
 298     /* Success. */
 299     gcmkFOOTER_ARG("*Node=0x%x", *Node);
 300     return gcvSTATUS_OK;
 301
 302 OnError:
 303     /* Roll back. */
 304     if (node != gcvNULL)
 305     {
 306         if (node->Virtual.mutex != gcvNULL)
 307         {
 308             /* Destroy the mutex. */
 309             gcmkVERIFY_OK(gckOS_DeleteMutex(os, node->Virtual.mutex));
 310         }
 311
 312         /* Free the structure. */
 313         gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, node));
 314     }
 315
 316     /* Return the status. */
 317     gcmkFOOTER();
 318     return status;
 319 }
 320
 321 /*******************************************************************************
 322 **
 323 **  gckVIDMEM_DestroyVirtual
 324 **
 325 **  Destroy an gcuVIDMEM_NODE union for virtual memory.
 326 **
 327 **  INPUT:
 328 **
 329 **      gcuVIDMEM_NODE_PTR Node
 330 **          Pointer to a gcuVIDMEM_NODE union.
 331 **
 332 **  OUTPUT:
 333 **
 334 **      Nothing.
 335 */
 336 gceSTATUS
 337 gckVIDMEM_DestroyVirtual(
 338     IN gcuVIDMEM_NODE_PTR Node
 339     )
 340 {
 341     gckOS os;
 342     gctINT i;
 343
 344     gcmkHEADER_ARG("Node=0x%x", Node);
 345
 346     /* Verify the arguments. */
 347     gcmkVERIFY_OBJECT(Node->Virtual.kernel, gcvOBJ_KERNEL);
 348
 349     /* Extact the gckOS object pointer. */
 350     os = Node->Virtual.kernel->os;
 351     gcmkVERIFY_OBJECT(os, gcvOBJ_OS);
 352
 353 #ifdef __QNXNTO__
 354     /* Unregister. */
 355 #if gcdENABLE_VG
 356     if (Node->Virtual.kernel->core != gcvCORE_VG)
 357 #endif
 358     {
 359         gcmkVERIFY_OK(
 360                 gckMMU_RemoveNode(Node->Virtual.kernel->mmu, Node));
 361     }
 362 #endif
 363
 364     /* Delete the mutex. */
 365     gcmkVERIFY_OK(gckOS_DeleteMutex(os, Node->Virtual.mutex));
 366
 367     for (i = 0; i < gcdMAX_GPU_COUNT; i++)
 368     {
 369         if (Node->Virtual.pageTables[i] != gcvNULL)
 370         {
 371 #if gcdENABLE_VG
 372             if (i == gcvCORE_VG)
 373             {
 374                 /* Free the pages. */
 375                 gcmkVERIFY_OK(gckVGMMU_FreePages(Node->Virtual.lockKernels[i]->vg->mmu,
 376                                                Node->Virtual.pageTables[i],
 377                                                Node->Virtual.pageCount));
 378             }
 379             else
 380 #endif
 381             {
 382                 /* Free the pages. */
 383                 gcmkVERIFY_OK(gckMMU_FreePages(Node->Virtual.lockKernels[i]->mmu,
 384                                                Node->Virtual.pageTables[i],
 385                                                Node->Virtual.pageCount));
 386             }
 387         }
 388     }
 389
 390     /* Delete the gcuVIDMEM_NODE union. */
 391     gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, Node));
 392
 393     /* Success. */
 394     gcmkFOOTER_NO();
 395     return gcvSTATUS_OK;
 396 }
 397
 398 /*******************************************************************************
 399 **
 400 **  gckVIDMEM_Construct
 401 **
 402 **  Construct a new gckVIDMEM object.
 403 **
 404 **  INPUT:
 405 **
 406 **      gckOS Os
 407 **          Pointer to an gckOS object.
 408 **
 409 **      gctUINT32 BaseAddress
 410 **          Base address for the video memory heap.
 411 **
 412 **      gctSIZE_T Bytes
 413 **          Number of bytes in the video memory heap.
 414 **
 415 **      gctSIZE_T Threshold
 416 **          Minimum number of bytes beyond am allocation before the node is
 417 **          split.  Can be used as a minimum alignment requirement.
 418 **
 419 **      gctSIZE_T BankSize
 420 **          Number of bytes per physical memory bank.  Used by bank
 421 **          optimization.
 422 **
 423 **  OUTPUT:
 424 **
 425 **      gckVIDMEM * Memory
 426 **          Pointer to a variable that will hold the pointer to the gckVIDMEM
 427 **          object.
 428 */
 429 gceSTATUS
 430 gckVIDMEM_Construct(
 431     IN gckOS Os,
 432     IN gctUINT32 BaseAddress,
 433     IN gctSIZE_T Bytes,
 434     IN gctSIZE_T Threshold,
 435     IN gctSIZE_T BankSize,
 436     OUT gckVIDMEM * Memory
 437     )
 438 {
 439     gckVIDMEM memory = gcvNULL;
 440     gceSTATUS status;
 441     gcuVIDMEM_NODE_PTR node;
 442     gctINT i, banks = 0;
 443     gctPOINTER pointer = gcvNULL;
 444
 445     gcmkHEADER_ARG("Os=0x%x BaseAddress=%08x Bytes=%lu Threshold=%lu "
 446                    "BankSize=%lu",
 447                    Os, BaseAddress, Bytes, Threshold, BankSize);
 448
 449     /* Verify the arguments. */
 450     gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
 451     gcmkVERIFY_ARGUMENT(Bytes > 0);
 452     gcmkVERIFY_ARGUMENT(Memory != gcvNULL);
 453
 454     /* Allocate the gckVIDMEM object. */
 455     gcmkONERROR(gckOS_Allocate(Os, gcmSIZEOF(struct _gckVIDMEM), &pointer));
 456
 457     memory = pointer;
 458
 459     /* Initialize the gckVIDMEM object. */
 460     memory->object.type = gcvOBJ_VIDMEM;
 461     memory->os          = Os;
 462
 463     /* Set video memory heap information. */
 464     memory->baseAddress = BaseAddress;
 465     memory->bytes       = Bytes;
 466     memory->freeBytes   = Bytes;
 467     memory->threshold   = Threshold;
 468     memory->mutex       = gcvNULL;
 469 #if gcdUSE_VIDMEM_PER_PID
 470     gcmkONERROR(gckOS_GetProcessID(&memory->pid));
 471 #endif
 472
 473     BaseAddress = 0;
 474
 475     /* Walk all possible banks. */
 476     for (i = 0; i < gcmCOUNTOF(memory->sentinel); ++i)
 477     {
 478         gctSIZE_T bytes;
 479
 480         if (BankSize == 0)
 481         {
 482             /* Use all bytes for the first bank. */
 483             bytes = Bytes;
 484         }
 485         else
 486         {
 487             /* Compute number of bytes for this bank. */
 488             bytes = gcmALIGN(BaseAddress + 1, BankSize) - BaseAddress;
 489
 490             if (bytes > Bytes)
 491             {
 492                 /* Make sure we don't exceed the total number of bytes. */
 493                 bytes = Bytes;
 494             }
 495         }
 496
 497         if (bytes == 0)
 498         {
 499             /* Mark heap is not used. */
 500             memory->sentinel[i].VidMem.next     =
 501             memory->sentinel[i].VidMem.prev     =
 502             memory->sentinel[i].VidMem.nextFree =
 503             memory->sentinel[i].VidMem.prevFree = gcvNULL;
 504             continue;
 505         }
 506
 507         /* Allocate one gcuVIDMEM_NODE union. */
 508         gcmkONERROR(gckOS_Allocate(Os, gcmSIZEOF(gcuVIDMEM_NODE), &pointer));
 509
 510         node = pointer;
 511
 512         /* Initialize gcuVIDMEM_NODE union. */
 513         node->VidMem.memory    = memory;
 514
 515         node->VidMem.next      =
 516         node->VidMem.prev      =
 517         node->VidMem.nextFree  =
 518         node->VidMem.prevFree  = &memory->sentinel[i];
 519
 520         node->VidMem.offset    = BaseAddress;
 521         node->VidMem.bytes     = bytes;
 522         node->VidMem.alignment = 0;
 523         node->VidMem.physical  = 0;
 524         node->VidMem.pool      = gcvPOOL_UNKNOWN;
 525
 526         node->VidMem.locked    = 0;
 527
 528 #if gcdDYNAMIC_MAP_RESERVED_MEMORY && gcdENABLE_VG
 529         node->VidMem.kernelVirtual = gcvNULL;
 530 #endif
 531
 532         gcmkONERROR(gckOS_ZeroMemory(&node->VidMem.sharedInfo, gcmSIZEOF(gcsVIDMEM_NODE_SHARED_INFO)));
 533
 534 #ifdef __QNXNTO__
 535 #if gcdUSE_VIDMEM_PER_PID
 536         node->VidMem.processID = memory->pid;
 537         node->VidMem.physical  = memory->baseAddress + BaseAddress;
 538         gcmkONERROR(gckOS_GetLogicalAddressProcess(Os,
 539                     node->VidMem.processID,
 540                     node->VidMem.physical,
 541                     &node->VidMem.logical));
 542 #else
 543         node->VidMem.processID = 0;
 544         node->VidMem.logical   = gcvNULL;
 545 #endif
 546 #endif
 547
 548         /* Initialize the linked list of nodes. */
 549         memory->sentinel[i].VidMem.next     =
 550         memory->sentinel[i].VidMem.prev     =
 551         memory->sentinel[i].VidMem.nextFree =
 552         memory->sentinel[i].VidMem.prevFree = node;
 553
 554         /* Mark sentinel. */
 555         memory->sentinel[i].VidMem.bytes = 0;
 556
 557         /* Adjust address for next bank. */
 558         BaseAddress += bytes;
 559         Bytes       -= bytes;
 560         banks       ++;
 561     }
 562
 563     /* Assign all the bank mappings. */
 564     memory->mapping[gcvSURF_RENDER_TARGET]      = banks - 1;
 565     memory->mapping[gcvSURF_BITMAP]             = banks - 1;
 566     if (banks > 1) --banks;
 567     memory->mapping[gcvSURF_DEPTH]              = banks - 1;
 568     memory->mapping[gcvSURF_HIERARCHICAL_DEPTH] = banks - 1;
 569     if (banks > 1) --banks;
 570     memory->mapping[gcvSURF_TEXTURE]            = banks - 1;
 571     if (banks > 1) --banks;
 572     memory->mapping[gcvSURF_VERTEX]             = banks - 1;
 573     if (banks > 1) --banks;
 574     memory->mapping[gcvSURF_INDEX]              = banks - 1;
 575     if (banks > 1) --banks;
 576     memory->mapping[gcvSURF_TILE_STATUS]        = banks - 1;
 577     if (banks > 1) --banks;
 578     memory->mapping[gcvSURF_TYPE_UNKNOWN]       = 0;
 579
 580 #if gcdENABLE_VG
 581     memory->mapping[gcvSURF_IMAGE]   = 0;
 582     memory->mapping[gcvSURF_MASK]    = 0;
 583     memory->mapping[gcvSURF_SCISSOR] = 0;
 584 #endif
 585
 586     gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
 587                   "[GALCORE] INDEX:         bank %d",
 588                   memory->mapping[gcvSURF_INDEX]);
 589     gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
 590                   "[GALCORE] VERTEX:        bank %d",
 591                   memory->mapping[gcvSURF_VERTEX]);
 592     gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
 593                   "[GALCORE] TEXTURE:       bank %d",
 594                   memory->mapping[gcvSURF_TEXTURE]);
 595     gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
 596                   "[GALCORE] RENDER_TARGET: bank %d",
 597                   memory->mapping[gcvSURF_RENDER_TARGET]);
 598     gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
 599                   "[GALCORE] DEPTH:         bank %d",
 600                   memory->mapping[gcvSURF_DEPTH]);
 601     gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
 602                   "[GALCORE] TILE_STATUS:   bank %d",
 603                   memory->mapping[gcvSURF_TILE_STATUS]);
 604
 605     /* Allocate the mutex. */
 606     gcmkONERROR(gckOS_CreateMutex(Os, &memory->mutex));
 607
 608     /* Return pointer to the gckVIDMEM object. */
 609     *Memory = memory;
 610
 611     /* Success. */
 612     gcmkFOOTER_ARG("*Memory=0x%x", *Memory);
 613     return gcvSTATUS_OK;
 614
 615 OnError:
 616     /* Roll back. */
 617     if (memory != gcvNULL)
 618     {
 619         if (memory->mutex != gcvNULL)
 620         {
 621             /* Delete the mutex. */
 622             gcmkVERIFY_OK(gckOS_DeleteMutex(Os, memory->mutex));
 623         }
 624
 625         for (i = 0; i < banks; ++i)
 626         {
 627             /* Free the heap. */
 628             gcmkASSERT(memory->sentinel[i].VidMem.next != gcvNULL);
 629             gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Os, memory->sentinel[i].VidMem.next));
 630         }
 631
 632         /* Free the object. */
 633         gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Os, memory));
 634     }
 635
 636     /* Return the status. */
 637     gcmkFOOTER();
 638     return status;
 639 }
 640
 641 /*******************************************************************************
 642 **
 643 **  gckVIDMEM_Destroy
 644 **
 645 **  Destroy an gckVIDMEM object.
 646 **
 647 **  INPUT:
 648 **
 649 **      gckVIDMEM Memory
 650 **          Pointer to an gckVIDMEM object to destroy.
 651 **
 652 **  OUTPUT:
 653 **
 654 **      Nothing.
 655 */
 656 gceSTATUS
 657 gckVIDMEM_Destroy(
 658     IN gckVIDMEM Memory
 659     )
 660 {
 661     gcuVIDMEM_NODE_PTR node, next;
 662     gctINT i;
 663
 664     gcmkHEADER_ARG("Memory=0x%x", Memory);
 665
 666     /* Verify the arguments. */
 667     gcmkVERIFY_OBJECT(Memory, gcvOBJ_VIDMEM);
 668
 669     /* Walk all sentinels. */
 670     for (i = 0; i < gcmCOUNTOF(Memory->sentinel); ++i)
 671     {
 672         /* Bail out of the heap is not used. */
 673         if (Memory->sentinel[i].VidMem.next == gcvNULL)
 674         {
 675             break;
 676         }
 677
 678         /* Walk all the nodes until we reach the sentinel. */
 679         for (node = Memory->sentinel[i].VidMem.next;
 680              node->VidMem.bytes != 0;
 681              node = next)
 682         {
 683             /* Save pointer to the next node. */
 684             next = node->VidMem.next;
 685
 686             /* Free the node. */
 687             gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Memory->os, node));
 688         }
 689     }
 690
 691     /* Free the mutex. */
 692     gcmkVERIFY_OK(gckOS_DeleteMutex(Memory->os, Memory->mutex));
 693
 694     /* Mark the object as unknown. */
 695     Memory->object.type = gcvOBJ_UNKNOWN;
 696
 697     /* Free the gckVIDMEM object. */
 698     gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Memory->os, Memory));
 699
 700     /* Success. */
 701     gcmkFOOTER_NO();
 702     return gcvSTATUS_OK;
 703 }
 704
 705 /*******************************************************************************
 706 **
 707 **  gckVIDMEM_Allocate
 708 **
 709 **  Allocate rectangular memory from the gckVIDMEM object.
 710 **
 711 **  INPUT:
 712 **
 713 **      gckVIDMEM Memory
 714 **          Pointer to an gckVIDMEM object.
 715 **
 716 **      gctUINT Width
 717 **          Width of rectangle to allocate.  Make sure the width is properly
 718 **          aligned.
 719 **
 720 **      gctUINT Height
 721 **          Height of rectangle to allocate.  Make sure the height is properly
 722 **          aligned.
 723 **
 724 **      gctUINT Depth
 725 **          Depth of rectangle to allocate.  This equals to the number of
 726 **          rectangles to allocate contiguously (i.e., for cubic maps and volume
 727 **          textures).
 728 **
 729 **      gctUINT BytesPerPixel
 730 **          Number of bytes per pixel.
 731 **
 732 **      gctUINT32 Alignment
 733 **          Byte alignment for allocation.
 734 **
 735 **      gceSURF_TYPE Type
 736 **          Type of surface to allocate (use by bank optimization).
 737 **
 738 **  OUTPUT:
 739 **
 740 **      gcuVIDMEM_NODE_PTR * Node
 741 **          Pointer to a variable that will hold the allocated memory node.
 742 */
 743 gceSTATUS
 744 gckVIDMEM_Allocate(
 745     IN gckVIDMEM Memory,
 746     IN gctUINT Width,
 747     IN gctUINT Height,
 748     IN gctUINT Depth,
 749     IN gctUINT BytesPerPixel,
 750     IN gctUINT32 Alignment,
 751     IN gceSURF_TYPE Type,
 752     OUT gcuVIDMEM_NODE_PTR * Node
 753     )
 754 {
 755     gctSIZE_T bytes;
 756     gceSTATUS status;
 757
 758     gcmkHEADER_ARG("Memory=0x%x Width=%u Height=%u Depth=%u BytesPerPixel=%u "
 759                    "Alignment=%u Type=%d",
 760                    Memory, Width, Height, Depth, BytesPerPixel, Alignment,
 761                    Type);
 762
 763     /* Verify the arguments. */
 764     gcmkVERIFY_OBJECT(Memory, gcvOBJ_VIDMEM);
 765     gcmkVERIFY_ARGUMENT(Width > 0);
 766     gcmkVERIFY_ARGUMENT(Height > 0);
 767     gcmkVERIFY_ARGUMENT(Depth > 0);
 768     gcmkVERIFY_ARGUMENT(BytesPerPixel > 0);
 769     gcmkVERIFY_ARGUMENT(Node != gcvNULL);
 770
 771     /* Compute linear size. */
 772     bytes = Width * Height * Depth * BytesPerPixel;
 773
 774     /* Allocate through linear function. */
 775     gcmkONERROR(
 776         gckVIDMEM_AllocateLinear(Memory, bytes, Alignment, Type, Node));
 777
 778     /* Success. */
 779     gcmkFOOTER_ARG("*Node=0x%x", *Node);
 780     return gcvSTATUS_OK;
 781
 782 OnError:
 783     /* Return the status. */
 784     gcmkFOOTER();
 785     return status;
 786 }
 787
 788 #if gcdENABLE_BANK_ALIGNMENT
 789
 790 #if !gcdBANK_BIT_START
 791 #error gcdBANK_BIT_START not defined.
 792 #endif
 793
 794 #if !gcdBANK_BIT_END
 795 #error gcdBANK_BIT_END not defined.
 796 #endif
 797 /*******************************************************************************
 798 **  _GetSurfaceBankAlignment
 799 **
 800 **  Return the required offset alignment required to the make BaseAddress
 801 **  aligned properly.
 802 **
 803 **  INPUT:
 804 **
 805 **      gckOS Os
 806 **          Pointer to gcoOS object.
 807 **
 808 **      gceSURF_TYPE Type
 809 **          Type of allocation.
 810 **
 811 **      gctUINT32 BaseAddress
 812 **          Base address of current video memory node.
 813 **
 814 **  OUTPUT:
 815 **
 816 **      gctUINT32_PTR AlignmentOffset
 817 **          Pointer to a variable that will hold the number of bytes to skip in
 818 **          the current video memory node in order to make the alignment bank
 819 **          aligned.
 820 */
 821 static gceSTATUS
 822 _GetSurfaceBankAlignment(
 823     IN gceSURF_TYPE Type,
 824     IN gctUINT32 BaseAddress,
 825     OUT gctUINT32_PTR AlignmentOffset
 826     )
 827 {
 828     gctUINT32 bank;
 829     /* To retrieve the bank. */
 830     static const gctUINT32 bankMask = (0xFFFFFFFF << gcdBANK_BIT_START)
 831                                     ^ (0xFFFFFFFF << (gcdBANK_BIT_END + 1));
 832
 833     /* To retrieve the bank and all the lower bytes. */
 834     static const gctUINT32 byteMask = ~(0xFFFFFFFF << (gcdBANK_BIT_END + 1));
 835
 836     gcmkHEADER_ARG("Type=%d BaseAddress=0x%x ", Type, BaseAddress);
 837
 838     /* Verify the arguments. */
 839     gcmkVERIFY_ARGUMENT(AlignmentOffset != gcvNULL);
 840
 841     switch (Type)
 842     {
 843     case gcvSURF_RENDER_TARGET:
 844         bank = (BaseAddress & bankMask) >> (gcdBANK_BIT_START);
 845
 846         /* Align to the first bank. */
 847         *AlignmentOffset = (bank == 0) ?
 848             0 :
 849             ((1 << (gcdBANK_BIT_END + 1)) + 0) -  (BaseAddress & byteMask);
 850         break;
 851
 852     case gcvSURF_DEPTH:
 853         bank = (BaseAddress & bankMask) >> (gcdBANK_BIT_START);
 854
 855         /* Align to the third bank. */
 856         *AlignmentOffset = (bank == 2) ?
 857             0 :
 858             ((1 << (gcdBANK_BIT_END + 1)) + (2 << gcdBANK_BIT_START)) -  (BaseAddress & byteMask);
 859
 860         /* Add a channel offset at the channel bit. */
 861         *AlignmentOffset += (1 << gcdBANK_CHANNEL_BIT);
 862         break;
 863
 864     default:
 865         /* no alignment needed. */
 866         *AlignmentOffset = 0;
 867     }
 868
 869     /* Return the status. */
 870     gcmkFOOTER_ARG("*AlignmentOffset=%u", *AlignmentOffset);
 871     return gcvSTATUS_OK;
 872 }
 873 #endif
 874
 875 static gcuVIDMEM_NODE_PTR
 876 _FindNode(
 877     IN gckVIDMEM Memory,
 878     IN gctINT Bank,
 879     IN gctSIZE_T Bytes,
 880     IN gceSURF_TYPE Type,
 881     IN OUT gctUINT32_PTR Alignment
 882     )
 883 {
 884     gcuVIDMEM_NODE_PTR node;
 885     gctUINT32 alignment;
 886
 887 #if gcdENABLE_BANK_ALIGNMENT
 888     gctUINT32 bankAlignment;
 889     gceSTATUS status;
 890 #endif
 891
 892     if (Memory->sentinel[Bank].VidMem.nextFree == gcvNULL)
 893     {
 894         /* No free nodes left. */
 895         return gcvNULL;
 896     }
 897
 898 #if gcdENABLE_BANK_ALIGNMENT
 899     /* Walk all free nodes until we have one that is big enough or we have
 900     ** reached the sentinel. */
 901     for (node = Memory->sentinel[Bank].VidMem.nextFree;
 902          node->VidMem.bytes != 0;
 903          node = node->VidMem.nextFree)
 904     {
 905         gcmkONERROR(_GetSurfaceBankAlignment(
 906             Type,
 907             node->VidMem.memory->baseAddress + node->VidMem.offset,
 908             &bankAlignment));
 909
 910         bankAlignment = gcmALIGN(bankAlignment, *Alignment);
 911
 912         /* Compute number of bytes to skip for alignment. */
 913         alignment = (*Alignment == 0)
 914                   ? 0
 915                   : (*Alignment - (node->VidMem.offset % *Alignment));
 916
 917         if (alignment == *Alignment)
 918         {
 919             /* Node is already aligned. */
 920             alignment = 0;
 921         }
 922
 923         if (node->VidMem.bytes >= Bytes + alignment + bankAlignment)
 924         {
 925             /* This node is big enough. */
 926             *Alignment = alignment + bankAlignment;
 927             return node;
 928         }
 929     }
 930 #endif
 931
 932     /* Walk all free nodes until we have one that is big enough or we have
 933        reached the sentinel. */
 934     for (node = Memory->sentinel[Bank].VidMem.nextFree;
 935          node->VidMem.bytes != 0;
 936          node = node->VidMem.nextFree)
 937     {
 938
 939         gctINT modulo = gckMATH_ModuloInt(node->VidMem.offset, *Alignment);
 940
 941         /* Compute number of bytes to skip for alignment. */
 942         alignment = (*Alignment == 0) ? 0 : (*Alignment - modulo);
 943
 944         if (alignment == *Alignment)
 945         {
 946             /* Node is already aligned. */
 947             alignment = 0;
 948         }
 949
 950         if (node->VidMem.bytes >= Bytes + alignment)
 951         {
 952             /* This node is big enough. */
 953             *Alignment = alignment;
 954             return node;
 955         }
 956     }
 957
 958 #if gcdENABLE_BANK_ALIGNMENT
 959 OnError:
 960 #endif
 961     /* Not enough memory. */
 962     return gcvNULL;
 963 }
 964
 965 /*******************************************************************************
 966 **
 967 **  gckVIDMEM_AllocateLinear
 968 **
 969 **  Allocate linear memory from the gckVIDMEM object.
 970 **
 971 **  INPUT:
 972 **
 973 **      gckVIDMEM Memory
 974 **          Pointer to an gckVIDMEM object.
 975 **
 976 **      gctSIZE_T Bytes
 977 **          Number of bytes to allocate.
 978 **
 979 **      gctUINT32 Alignment
 980 **          Byte alignment for allocation.
 981 **
 982 **      gceSURF_TYPE Type
 983 **          Type of surface to allocate (use by bank optimization).
 984 **
 985 **  OUTPUT:
 986 **
 987 **      gcuVIDMEM_NODE_PTR * Node
 988 **          Pointer to a variable that will hold the allocated memory node.
 989 */
 990 gceSTATUS
 991 gckVIDMEM_AllocateLinear(
 992     IN gckVIDMEM Memory,
 993     IN gctSIZE_T Bytes,
 994     IN gctUINT32 Alignment,
 995     IN gceSURF_TYPE Type,
 996     OUT gcuVIDMEM_NODE_PTR * Node
 997     )
 998 {
 999     gceSTATUS status;
1000     gcuVIDMEM_NODE_PTR node;
1001     gctUINT32 alignment;
1002     gctINT bank, i;
1003     gctBOOL acquired = gcvFALSE;
1004
1005     gcmkHEADER_ARG("Memory=0x%x Bytes=%lu Alignment=%u Type=%d",
1006                    Memory, Bytes, Alignment, Type);
1007
1008     /* Verify the arguments. */
1009     gcmkVERIFY_OBJECT(Memory, gcvOBJ_VIDMEM);
1010     gcmkVERIFY_ARGUMENT(Bytes > 0);
1011     gcmkVERIFY_ARGUMENT(Node != gcvNULL);
1012     gcmkVERIFY_ARGUMENT(Type < gcvSURF_NUM_TYPES);
1013
1014     /* Acquire the mutex. */
1015     gcmkONERROR(gckOS_AcquireMutex(Memory->os, Memory->mutex, gcvINFINITE));
1016
1017     acquired = gcvTRUE;
1018 #if !gcdUSE_VIDMEM_PER_PID
1019
1020     if (Bytes > Memory->freeBytes)
1021     {
1022         /* Not enough memory. */
1023         status = gcvSTATUS_OUT_OF_MEMORY;
1024         goto OnError;
1025     }
1026 #endif
1027
1028 #if gcdSMALL_BLOCK_SIZE
1029     if ((Memory->freeBytes < (Memory->bytes/gcdRATIO_FOR_SMALL_MEMORY))
1030     &&  (Bytes >= gcdSMALL_BLOCK_SIZE)
1031     )
1032     {
1033         /* The left memory is for small memory.*/
1034         status = gcvSTATUS_OUT_OF_MEMORY;
1035         goto OnError;
1036     }
1037 #endif
1038
1039     /* Find the default bank for this surface type. */
1040     gcmkASSERT((gctINT) Type < gcmCOUNTOF(Memory->mapping));
1041     bank      = Memory->mapping[Type];
1042     alignment = Alignment;
1043
1044 #if gcdUSE_VIDMEM_PER_PID
1045     if (Bytes <= Memory->freeBytes)
1046     {
1047 #endif
1048     /* Find a free node in the default bank. */
1049     node = _FindNode(Memory, bank, Bytes, Type, &alignment);
1050
1051     /* Out of memory? */
1052     if (node == gcvNULL)
1053     {
1054         /* Walk all lower banks. */
1055         for (i = bank - 1; i >= 0; --i)
1056         {
1057             /* Find a free node inside the current bank. */
1058             node = _FindNode(Memory, i, Bytes, Type, &alignment);
1059             if (node != gcvNULL)
1060             {
1061                 break;
1062             }
1063         }
1064     }
1065
1066     if (node == gcvNULL)
1067     {
1068         /* Walk all upper banks. */
1069         for (i = bank + 1; i < gcmCOUNTOF(Memory->sentinel); ++i)
1070         {
1071             if (Memory->sentinel[i].VidMem.nextFree == gcvNULL)
1072             {
1073                 /* Abort when we reach unused banks. */
1074                 break;
1075             }
1076
1077             /* Find a free node inside the current bank. */
1078             node = _FindNode(Memory, i, Bytes, Type, &alignment);
1079             if (node != gcvNULL)
1080             {
1081                 break;
1082             }
1083         }
1084     }
1085 #if gcdUSE_VIDMEM_PER_PID
1086     }
1087 #endif
1088
1089     if (node == gcvNULL)
1090     {
1091         /* Out of memory. */
1092 #if gcdUSE_VIDMEM_PER_PID
1093         /* Allocate more memory from shared pool. */
1094         gctSIZE_T bytes;
1095         gctPHYS_ADDR physical_temp;
1096         gctUINT32 physical;
1097         gctPOINTER logical;
1098
1099         bytes = gcmALIGN(Bytes, gcdUSE_VIDMEM_PER_PID_SIZE);
1100
1101         gcmkONERROR(gckOS_AllocateContiguous(Memory->os,
1102                 gcvTRUE,
1103                 &bytes,
1104                 &physical_temp,
1105                 &logical));
1106
1107         /* physical address is returned as 0 for user space. workaround. */
1108         if (physical_temp == gcvNULL)
1109         {
1110         gcmkONERROR(gckOS_GetPhysicalAddress(Memory->os, logical, &physical));
1111         }
1112
1113         /* Allocate one gcuVIDMEM_NODE union. */
1114         gcmkONERROR(
1115             gckOS_Allocate(Memory->os,
1116                            gcmSIZEOF(gcuVIDMEM_NODE),
1117                            (gctPOINTER *) &node));
1118
1119         /* Initialize gcuVIDMEM_NODE union. */
1120         node->VidMem.memory    = Memory;
1121
1122         node->VidMem.offset    = 0;
1123         node->VidMem.bytes     = bytes;
1124         node->VidMem.alignment = 0;
1125         node->VidMem.physical  = physical;
1126         node->VidMem.pool      = gcvPOOL_UNKNOWN;
1127
1128         node->VidMem.locked    = 0;
1129
1130 #ifdef __QNXNTO__
1131         gcmkONERROR(gckOS_GetProcessID(&node->VidMem.processID));
1132         node->VidMem.logical   = logical;
1133         gcmkASSERT(logical != gcvNULL);
1134 #endif
1135
1136         /* Insert node behind sentinel node. */
1137         node->VidMem.next = Memory->sentinel[bank].VidMem.next;
1138         node->VidMem.prev = &Memory->sentinel[bank];
1139         Memory->sentinel[bank].VidMem.next = node->VidMem.next->VidMem.prev = node;
1140
1141         /* Insert free node behind sentinel node. */
1142         node->VidMem.nextFree = Memory->sentinel[bank].VidMem.nextFree;
1143         node->VidMem.prevFree = &Memory->sentinel[bank];
1144         Memory->sentinel[bank].VidMem.nextFree = node->VidMem.nextFree->VidMem.prevFree = node;
1145
1146         Memory->freeBytes += bytes;
1147 #else
1148         status = gcvSTATUS_OUT_OF_MEMORY;
1149         goto OnError;
1150 #endif
1151     }
1152
1153     /* Do we have an alignment? */
1154     if (alignment > 0)
1155     {
1156         /* Split the node so it is aligned. */
1157         if (_Split(Memory->os, node, alignment))
1158         {
1159             /* Successful split, move to aligned node. */
1160             node = node->VidMem.next;
1161
1162             /* Remove alignment. */
1163             alignment = 0;
1164         }
1165     }
1166
1167     /* Do we have enough memory after the allocation to split it? */
1168     if (node->VidMem.bytes - Bytes > Memory->threshold)
1169     {
1170         /* Adjust the node size. */
1171         _Split(Memory->os, node, Bytes);
1172     }
1173
1174     /* Remove the node from the free list. */
1175     node->VidMem.prevFree->VidMem.nextFree = node->VidMem.nextFree;
1176     node->VidMem.nextFree->VidMem.prevFree = node->VidMem.prevFree;
1177     node->VidMem.nextFree                  =
1178     node->VidMem.prevFree                  = gcvNULL;
1179
1180     /* Fill in the information. */
1181     node->VidMem.alignment = alignment;
1182     node->VidMem.memory    = Memory;
1183 #ifdef __QNXNTO__
1184 #if !gcdUSE_VIDMEM_PER_PID
1185     node->VidMem.logical   = gcvNULL;
1186     gcmkONERROR(gckOS_GetProcessID(&node->VidMem.processID));
1187 #else
1188     gcmkASSERT(node->VidMem.logical != gcvNULL);
1189 #endif
1190 #endif
1191
1192     /* Adjust the number of free bytes. */
1193     Memory->freeBytes -= node->VidMem.bytes;
1194
1195     node->VidMem.freePending = gcvFALSE;
1196
1197 #if gcdDYNAMIC_MAP_RESERVED_MEMORY && gcdENABLE_VG
1198     node->VidMem.kernelVirtual = gcvNULL;
1199 #endif
1200
1201     /* Release the mutex. */
1202     gcmkVERIFY_OK(gckOS_ReleaseMutex(Memory->os, Memory->mutex));
1203
1204     /* Return the pointer to the node. */
1205     *Node = node;
1206
1207     gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
1208                    "Allocated %u bytes @ 0x%x [0x%08X]",
1209                    node->VidMem.bytes, node, node->VidMem.offset);
1210
1211     /* Success. */
1212     gcmkFOOTER_ARG("*Node=0x%x", *Node);
1213     return gcvSTATUS_OK;
1214
1215 OnError:
1216     if (acquired)
1217     {
1218      /* Release the mutex. */
1219         gcmkVERIFY_OK(gckOS_ReleaseMutex(Memory->os, Memory->mutex));
1220     }
1221
1222     /* Return the status. */
1223     gcmkFOOTER();
1224     return status;
1225 }
1226
1227 /*******************************************************************************
1228 **
1229 **  gckVIDMEM_Free
1230 **
1231 **  Free an allocated video memory node.
1232 **
1233 **  INPUT:
1234 **
1235 **      gcuVIDMEM_NODE_PTR Node
1236 **          Pointer to a gcuVIDMEM_NODE object.
1237 **
1238 **  OUTPUT:
1239 **
1240 **      Nothing.
1241 */
1242 gceSTATUS
1243 gckVIDMEM_Free(
1244     IN gcuVIDMEM_NODE_PTR Node
1245     )
1246 {
1247     gceSTATUS status;
1248     gckKERNEL kernel = gcvNULL;
1249     gckVIDMEM memory = gcvNULL;
1250     gcuVIDMEM_NODE_PTR node;
1251     gctBOOL mutexAcquired = gcvFALSE;
1252     gckOS os = gcvNULL;
1253     gctBOOL acquired = gcvFALSE;
1254     gctINT32 i, totalLocked;
1255
1256     gcmkHEADER_ARG("Node=0x%x", Node);
1257
1258     /* Verify the arguments. */
1259     if ((Node == gcvNULL)
1260     ||  (Node->VidMem.memory == gcvNULL)
1261     )
1262     {
1263         /* Invalid object. */
1264         gcmkONERROR(gcvSTATUS_INVALID_OBJECT);
1265     }
1266
1267     /**************************** Video Memory ********************************/
1268
1269     if (Node->VidMem.memory->object.type == gcvOBJ_VIDMEM)
1270     {
1271         if (Node->VidMem.locked > 0)
1272         {
1273             /* Client still has a lock, defer free op 'till when lock reaches 0. */
1274             Node->VidMem.freePending = gcvTRUE;
1275
1276             gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
1277                            "Node 0x%x is locked (%d)... deferring free.",
1278                            Node, Node->VidMem.locked);
1279
1280             gcmkFOOTER_NO();
1281             return gcvSTATUS_OK;
1282         }
1283
1284         /* Extract pointer to gckVIDMEM object owning the node. */
1285         memory = Node->VidMem.memory;
1286
1287         /* Acquire the mutex. */
1288         gcmkONERROR(
1289             gckOS_AcquireMutex(memory->os, memory->mutex, gcvINFINITE));
1290
1291         mutexAcquired = gcvTRUE;
1292
1293 #ifdef __QNXNTO__
1294 #if !gcdUSE_VIDMEM_PER_PID
1295         /* Reset. */
1296         Node->VidMem.processID = 0;
1297         Node->VidMem.logical = gcvNULL;
1298 #endif
1299
1300         /* Don't try to re-free an already freed node. */
1301         if ((Node->VidMem.nextFree == gcvNULL)
1302         &&  (Node->VidMem.prevFree == gcvNULL)
1303         )
1304 #endif
1305         {
1306 #if gcdDYNAMIC_MAP_RESERVED_MEMORY && gcdENABLE_VG
1307             if (Node->VidMem.kernelVirtual)
1308             {
1309                 gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
1310                         "%s(%d) Unmap %x from kernel space.",
1311                         __FUNCTION__, __LINE__,
1312                         Node->VidMem.kernelVirtual);
1313
1314                 gcmkVERIFY_OK(
1315                     gckOS_UnmapPhysical(memory->os,
1316                                         Node->VidMem.kernelVirtual,
1317                                         Node->VidMem.bytes));
1318
1319                 Node->VidMem.kernelVirtual = gcvNULL;
1320             }
1321 #endif
1322
1323             /* Check if Node is already freed. */
1324             if (Node->VidMem.nextFree)
1325             {
1326                 /* Node is alread freed. */
1327                 gcmkONERROR(gcvSTATUS_INVALID_DATA);
1328             }
1329
1330             /* Update the number of free bytes. */
1331             memory->freeBytes += Node->VidMem.bytes;
1332
1333             /* Find the next free node. */
1334             for (node = Node->VidMem.next;
1335                  node != gcvNULL && node->VidMem.nextFree == gcvNULL;
1336                  node = node->VidMem.next) ;
1337
1338             /* Insert this node in the free list. */
1339             Node->VidMem.nextFree = node;
1340             Node->VidMem.prevFree = node->VidMem.prevFree;
1341
1342             Node->VidMem.prevFree->VidMem.nextFree =
1343             node->VidMem.prevFree                  = Node;
1344
1345             /* Is the next node a free node and not the sentinel? */
1346             if ((Node->VidMem.next == Node->VidMem.nextFree)
1347             &&  (Node->VidMem.next->VidMem.bytes != 0)
1348             )
1349             {
1350                 /* Merge this node with the next node. */
1351                 gcmkONERROR(_Merge(memory->os, node = Node));
1352                 gcmkASSERT(node->VidMem.nextFree != node);
1353                 gcmkASSERT(node->VidMem.prevFree != node);
1354             }
1355
1356             /* Is the previous node a free node and not the sentinel? */
1357             if ((Node->VidMem.prev == Node->VidMem.prevFree)
1358             &&  (Node->VidMem.prev->VidMem.bytes != 0)
1359             )
1360             {
1361                 /* Merge this node with the previous node. */
1362                 gcmkONERROR(_Merge(memory->os, node = Node->VidMem.prev));
1363                 gcmkASSERT(node->VidMem.nextFree != node);
1364                 gcmkASSERT(node->VidMem.prevFree != node);
1365             }
1366         }
1367
1368         /* Release the mutex. */
1369         gcmkVERIFY_OK(gckOS_ReleaseMutex(memory->os, memory->mutex));
1370
1371         gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
1372                        "Node 0x%x is freed.",
1373                        Node);
1374
1375         /* Success. */
1376         gcmkFOOTER_NO();
1377         return gcvSTATUS_OK;
1378     }
1379
1380     /*************************** Virtual Memory *******************************/
1381
1382     /* Get gckKERNEL object. */
1383     kernel = Node->Virtual.kernel;
1384
1385     /* Verify the gckKERNEL object pointer. */
1386     gcmkVERIFY_OBJECT(kernel, gcvOBJ_KERNEL);
1387
1388     /* Get the gckOS object pointer. */
1389     os = kernel->os;
1390     gcmkVERIFY_OBJECT(os, gcvOBJ_OS);
1391
1392     /* Grab the mutex. */
1393     gcmkONERROR(
1394         gckOS_AcquireMutex(os, Node->Virtual.mutex, gcvINFINITE));
1395
1396     acquired = gcvTRUE;
1397
1398     for (i = 0, totalLocked = 0; i < gcdMAX_GPU_COUNT; i++)
1399     {
1400         totalLocked += Node->Virtual.lockeds[i];
1401     }
1402
1403     if (totalLocked > 0)
1404     {
1405         gcmkTRACE_ZONE(gcvLEVEL_ERROR, gcvZONE_VIDMEM,
1406                        "gckVIDMEM_Free: Virtual node 0x%x is locked (%d)",
1407                        Node, totalLocked);
1408
1409         /* Set Flag */
1410         Node->Virtual.freed = gcvTRUE;
1411
1412         gcmkVERIFY_OK(gckOS_ReleaseMutex(os, Node->Virtual.mutex));
1413     }
1414     else
1415     {
1416         /* Free the virtual memory. */
1417         gcmkVERIFY_OK(gckOS_FreePagedMemory(kernel->os,
1418                                             Node->Virtual.physical,
1419                                             Node->Virtual.bytes));
1420
1421         gcmkVERIFY_OK(gckOS_ReleaseMutex(os, Node->Virtual.mutex));
1422
1423         /* Destroy the gcuVIDMEM_NODE union. */
1424         gcmkVERIFY_OK(gckVIDMEM_DestroyVirtual(Node));
1425     }
1426
1427     /* Success. */
1428     gcmkFOOTER_NO();
1429     return gcvSTATUS_OK;
1430
1431 OnError:
1432     if (mutexAcquired)
1433     {
1434         /* Release the mutex. */
1435         gcmkVERIFY_OK(gckOS_ReleaseMutex(
1436             memory->os, memory->mutex
1437             ));
1438     }
1439
1440     if (acquired)
1441     {
1442        gcmkVERIFY_OK(gckOS_ReleaseMutex(os, Node->Virtual.mutex));
1443     }
1444
1445     /* Return the status. */
1446     gcmkFOOTER();
1447     return status;
1448 }
1449
1450
1451 #ifdef __QNXNTO__
1452 /*******************************************************************************
1453 **
1454 **  gcoVIDMEM_FreeHandleMemory
1455 **
1456 **  Free all allocated video memory nodes for a handle.
1457 **
1458 **  INPUT:
1459 **
1460 **      gcoVIDMEM Memory
1461 **          Pointer to an gcoVIDMEM object..
1462 **
1463 **  OUTPUT:
1464 **
1465 **      Nothing.
1466 */
1467 gceSTATUS
1468 gckVIDMEM_FreeHandleMemory(
1469     IN gckKERNEL Kernel,
1470     IN gckVIDMEM Memory,
1471     IN gctUINT32 Pid
1472     )
1473 {
1474     gceSTATUS status;
1475     gctBOOL mutex = gcvFALSE;
1476     gcuVIDMEM_NODE_PTR node;
1477     gctINT i;
1478     gctUINT32 nodeCount = 0, byteCount = 0;
1479     gctBOOL again;
1480
1481     gcmkHEADER_ARG("Kernel=0x%x, Memory=0x%x Pid=0x%u", Kernel, Memory, Pid);
1482
1483     gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
1484     gcmkVERIFY_OBJECT(Memory, gcvOBJ_VIDMEM);
1485
1486     gcmkONERROR(gckOS_AcquireMutex(Memory->os, Memory->mutex, gcvINFINITE));
1487     mutex = gcvTRUE;
1488
1489     /* Walk all sentinels. */
1490     for (i = 0; i < gcmCOUNTOF(Memory->sentinel); ++i)
1491     {
1492         /* Bail out of the heap if it is not used. */
1493         if (Memory->sentinel[i].VidMem.next == gcvNULL)
1494         {
1495             break;
1496         }
1497
1498         do
1499         {
1500             again = gcvFALSE;
1501
1502             /* Walk all the nodes until we reach the sentinel. */
1503             for (node = Memory->sentinel[i].VidMem.next;
1504                  node->VidMem.bytes != 0;
1505                  node = node->VidMem.next)
1506             {
1507                 /* Free the node if it was allocated by Handle. */
1508                 if (node->VidMem.processID == Pid)
1509                 {
1510                     /* Unlock video memory. */
1511                     while (node->VidMem.locked > 0)
1512                     {
1513                         gckVIDMEM_Unlock(Kernel, node, gcvSURF_TYPE_UNKNOWN, gcvNULL);
1514                     }
1515
1516                     nodeCount++;
1517                     byteCount += node->VidMem.bytes;
1518
1519                     /* Free video memory. */
1520                     gcmkVERIFY_OK(gckVIDMEM_Free(node));
1521
1522                     /*
1523                      * Freeing may cause a merge which will invalidate our iteration.
1524                      * Don't be clever, just restart.
1525                      */
1526                     again = gcvTRUE;
1527
1528                     break;
1529                 }
1530 #if gcdUSE_VIDMEM_PER_PID
1531                 else
1532                 {
1533                     gcmkASSERT(node->VidMem.processID == Pid);
1534                 }
1535 #endif
1536             }
1537         }
1538         while (again);
1539     }
1540
1541     gcmkVERIFY_OK(gckOS_ReleaseMutex(Memory->os, Memory->mutex));
1542     gcmkFOOTER();
1543     return gcvSTATUS_OK;
1544
1545 OnError:
1546     if (mutex)
1547     {
1548         gcmkVERIFY_OK(gckOS_ReleaseMutex(Memory->os, Memory->mutex));
1549     }
1550
1551     gcmkFOOTER();
1552     return status;
1553 }
1554 #endif
1555
1556 /*******************************************************************************
1557 **
1558 ** _NeedVirtualMapping
1559 **
1560 **  Whether setup GPU page table for video node.
1561 **
1562 **  INPUT:
1563 **      gckKERNEL Kernel
1564 **          Pointer to an gckKERNEL object.
1565 **
1566 **      gcuVIDMEM_NODE_PTR Node
1567 **          Pointer to a gcuVIDMEM_NODE union.
1568 **
1569 **      gceCORE  Core
1570 **          Id of current GPU.
1571 **
1572 **  OUTPUT:
1573 **      gctBOOL * NeedMapping
1574 **          A pointer hold the result whether Node should be mapping.
1575 */
1576 static gceSTATUS
1577 _NeedVirtualMapping(
1578     IN gckKERNEL Kernel,
1579     IN gceCORE  Core,
1580     IN gcuVIDMEM_NODE_PTR Node,
1581     OUT gctBOOL * NeedMapping
1582 )
1583 {
1584     gceSTATUS status;
1585     gctUINT32 phys;
1586     gctUINT32 end;
1587     gcePOOL pool;
1588     gctUINT32 offset;
1589     gctUINT32 baseAddress;
1590
1591     gcmkHEADER_ARG("Node=0x%X", Node);
1592
1593     /* Verify the arguments. */
1594     gcmkVERIFY_ARGUMENT(Kernel != gcvNULL);
1595     gcmkVERIFY_ARGUMENT(Node != gcvNULL);
1596     gcmkVERIFY_ARGUMENT(NeedMapping != gcvNULL);
1597     gcmkVERIFY_ARGUMENT(Core < gcdMAX_GPU_COUNT);
1598
1599     if (Node->Virtual.contiguous)
1600     {
1601 #if gcdENABLE_VG
1602         if (Core == gcvCORE_VG)
1603         {
1604             *NeedMapping = gcvFALSE;
1605         }
1606         else
1607 #endif
1608         {
1609             /* Convert logical address into a physical address. */
1610             gcmkONERROR(
1611                 gckOS_GetPhysicalAddress(Kernel->os, Node->Virtual.logical, &phys));
1612
1613             gcmkONERROR(gckOS_GetBaseAddress(Kernel->os, &baseAddress));
1614
1615             gcmkASSERT(phys >= baseAddress);
1616
1617             /* Subtract baseAddress to get a GPU address used for programming. */
1618             phys -= baseAddress;
1619
1620             /* If part of region is belong to gcvPOOL_VIRTUAL,
1621             ** whole region has to be mapped. */
1622             end = phys + Node->Virtual.bytes - 1;
1623
1624             gcmkONERROR(gckHARDWARE_SplitMemory(
1625                         Kernel->hardware, end, &pool, &offset
1626                         ));
1627
1628             *NeedMapping = (pool == gcvPOOL_VIRTUAL);
1629         }
1630     }
1631     else
1632     {
1633         *NeedMapping = gcvTRUE;
1634     }
1635
1636     gcmkFOOTER_ARG("*NeedMapping=%d", *NeedMapping);
1637     return gcvSTATUS_OK;
1638
1639 OnError:
1640     gcmkFOOTER();
1641     return status;
1642 }
1643
1644 /*******************************************************************************
1645 **
1646 **  gckVIDMEM_Lock
1647 **
1648 **  Lock a video memory node and return its hardware specific address.
1649 **
1650 **  INPUT:
1651 **
1652 **      gckKERNEL Kernel
1653 **          Pointer to an gckKERNEL object.
1654 **
1655 **      gcuVIDMEM_NODE_PTR Node
1656 **          Pointer to a gcuVIDMEM_NODE union.
1657 **
1658 **  OUTPUT:
1659 **
1660 **      gctUINT32 * Address
1661 **          Pointer to a variable that will hold the hardware specific address.
1662 */
1663 gceSTATUS
1664 gckVIDMEM_Lock(
1665     IN gckKERNEL Kernel,
1666     IN gcuVIDMEM_NODE_PTR Node,
1667     IN gctBOOL Cacheable,
1668     OUT gctUINT32 * Address
1669     )
1670 {
1671     gceSTATUS status;
1672     gctBOOL acquired = gcvFALSE;
1673     gctBOOL locked = gcvFALSE;
1674     gckOS os = gcvNULL;
1675     gctBOOL needMapping;
1676     gctUINT32 baseAddress;
1677
1678     gcmkHEADER_ARG("Node=0x%x", Node);
1679
1680     /* Verify the arguments. */
1681     gcmkVERIFY_ARGUMENT(Address != gcvNULL);
1682
1683     if ((Node == gcvNULL)
1684     ||  (Node->VidMem.memory == gcvNULL)
1685     )
1686     {
1687         /* Invalid object. */
1688         gcmkONERROR(gcvSTATUS_INVALID_OBJECT);
1689     }
1690
1691     /**************************** Video Memory ********************************/
1692
1693     if (Node->VidMem.memory->object.type == gcvOBJ_VIDMEM)
1694     {
1695         if (Cacheable == gcvTRUE)
1696         {
1697             gcmkONERROR(gcvSTATUS_INVALID_REQUEST);
1698         }
1699
1700         /* Increment the lock count. */
1701         Node->VidMem.locked ++;
1702
1703         /* Return the physical address of the node. */
1704 #if !gcdUSE_VIDMEM_PER_PID
1705         *Address = Node->VidMem.memory->baseAddress
1706                  + Node->VidMem.offset
1707                  + Node->VidMem.alignment;
1708 #else
1709         *Address = Node->VidMem.physical;
1710 #endif
1711
1712         /* Get hardware specific address. */
1713 #if gcdENABLE_VG
1714         if (Kernel->vg == gcvNULL)
1715 #endif
1716         {
1717             if (Kernel->hardware->mmuVersion == 0)
1718             {
1719                 /* Convert physical to GPU address for old mmu. */
1720                 gcmkONERROR(gckOS_GetBaseAddress(Kernel->os, &baseAddress));
1721                 gcmkASSERT(*Address > baseAddress);
1722                 *Address -= baseAddress;
1723             }
1724         }
1725
1726         gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
1727                       "Locked node 0x%x (%d) @ 0x%08X",
1728                       Node,
1729                       Node->VidMem.locked,
1730                       *Address);
1731     }
1732
1733     /*************************** Virtual Memory *******************************/
1734
1735     else
1736     {
1737         /* Verify the gckKERNEL object pointer. */
1738         gcmkVERIFY_OBJECT(Node->Virtual.kernel, gcvOBJ_KERNEL);
1739
1740         /* Extract the gckOS object pointer. */
1741         os = Node->Virtual.kernel->os;
1742         gcmkVERIFY_OBJECT(os, gcvOBJ_OS);
1743
1744         /* Grab the mutex. */
1745         gcmkONERROR(gckOS_AcquireMutex(os, Node->Virtual.mutex, gcvINFINITE));
1746         acquired = gcvTRUE;
1747
1748 #if gcdPAGED_MEMORY_CACHEABLE
1749         /* Force video memory cacheable. */
1750         Cacheable = gcvTRUE;
1751 #endif
1752
1753         gcmkONERROR(
1754             gckOS_LockPages(os,
1755                             Node->Virtual.physical,
1756                             Node->Virtual.bytes,
1757                             Cacheable,
1758                             &Node->Virtual.logical,
1759                             &Node->Virtual.pageCount));
1760
1761         /* Increment the lock count. */
1762         if (Node->Virtual.lockeds[Kernel->core] ++ == 0)
1763         {
1764             /* Is this node pending for a final unlock? */
1765 #ifdef __QNXNTO__
1766             if (!Node->Virtual.contiguous && Node->Virtual.unlockPendings[Kernel->core])
1767             {
1768                 /* Make sure we have a page table. */
1769                 gcmkASSERT(Node->Virtual.pageTables[Kernel->core] != gcvNULL);
1770
1771                 /* Remove pending unlock. */
1772                 Node->Virtual.unlockPendings[Kernel->core] = gcvFALSE;
1773             }
1774
1775             /* First lock - create a page table. */
1776             gcmkASSERT(Node->Virtual.pageTables[Kernel->core] == gcvNULL);
1777
1778             /* Make sure we mark our node as not flushed. */
1779             Node->Virtual.unlockPendings[Kernel->core] = gcvFALSE;
1780 #endif
1781
1782             locked = gcvTRUE;
1783
1784             gcmkONERROR(_NeedVirtualMapping(Kernel, Kernel->core, Node, &needMapping));
1785
1786             if (needMapping == gcvFALSE)
1787             {
1788                 /* Get hardware specific address. */
1789 #if gcdENABLE_VG
1790                 if (Kernel->vg != gcvNULL)
1791                 {
1792                     gcmkONERROR(gckVGHARDWARE_ConvertLogical(Kernel->vg->hardware,
1793                                 Node->Virtual.logical,
1794                                 &Node->Virtual.addresses[Kernel->core]));
1795                 }
1796                 else
1797 #endif
1798                 {
1799                     gcmkONERROR(gckHARDWARE_ConvertLogical(Kernel->hardware,
1800                                 Node->Virtual.logical,
1801                                 &Node->Virtual.addresses[Kernel->core]));
1802                 }
1803             }
1804             else
1805             {
1806 #if gcdENABLE_VG
1807                 if (Kernel->vg != gcvNULL)
1808                 {
1809                     /* Allocate pages inside the MMU. */
1810                     gcmkONERROR(
1811                         gckVGMMU_AllocatePages(Kernel->vg->mmu,
1812                                              Node->Virtual.pageCount,
1813                                              &Node->Virtual.pageTables[Kernel->core],
1814                                              &Node->Virtual.addresses[Kernel->core]));
1815                 }
1816                 else
1817 #endif
1818                 {
1819                     /* Allocate pages inside the MMU. */
1820                     gcmkONERROR(
1821                         gckMMU_AllocatePagesEx(Kernel->mmu,
1822                                              Node->Virtual.pageCount,
1823                                              Node->Virtual.type,
1824                                              &Node->Virtual.pageTables[Kernel->core],
1825                                              &Node->Virtual.addresses[Kernel->core]));
1826                 }
1827
1828                 Node->Virtual.lockKernels[Kernel->core] = Kernel;
1829
1830                 /* Map the pages. */
1831 #ifdef __QNXNTO__
1832                 gcmkONERROR(
1833                     gckOS_MapPagesEx(os,
1834                                      Kernel->core,
1835                                      Node->Virtual.physical,
1836                                      Node->Virtual.logical,
1837                                      Node->Virtual.pageCount,
1838                                      Node->Virtual.pageTables[Kernel->core]));
1839 #else
1840                 gcmkONERROR(
1841                     gckOS_MapPagesEx(os,
1842                                      Kernel->core,
1843                                      Node->Virtual.physical,
1844                                      Node->Virtual.pageCount,
1845                                      Node->Virtual.pageTables[Kernel->core]));
1846 #endif
1847
1848 #if gcdENABLE_VG
1849                 if (Kernel->core == gcvCORE_VG)
1850                 {
1851                     gcmkONERROR(gckVGMMU_Flush(Kernel->vg->mmu));
1852                 }
1853                 else
1854 #endif
1855                 {
1856                     gcmkONERROR(gckMMU_Flush(Kernel->mmu));
1857                 }
1858             }
1859             gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
1860                            "Mapped virtual node 0x%x to 0x%08X",
1861                            Node,
1862                            Node->Virtual.addresses[Kernel->core]);
1863         }
1864
1865         /* Return hardware address. */
1866         *Address = Node->Virtual.addresses[Kernel->core];
1867
1868         /* Release the mutex. */
1869         gcmkVERIFY_OK(gckOS_ReleaseMutex(os, Node->Virtual.mutex));
1870     }
1871
1872     /* Success. */
1873     gcmkFOOTER_ARG("*Address=%08x", *Address);
1874     return gcvSTATUS_OK;
1875
1876 OnError:
1877     if (locked)
1878     {
1879         if (Node->Virtual.pageTables[Kernel->core] != gcvNULL)
1880         {
1881 #if gcdENABLE_VG
1882             if (Kernel->vg != gcvNULL)
1883             {
1884                 /* Free the pages from the MMU. */
1885                 gcmkVERIFY_OK(
1886                     gckVGMMU_FreePages(Kernel->vg->mmu,
1887                                      Node->Virtual.pageTables[Kernel->core],
1888                                      Node->Virtual.pageCount));
1889             }
1890             else
1891 #endif
1892             {
1893                 /* Free the pages from the MMU. */
1894                 gcmkVERIFY_OK(
1895                     gckMMU_FreePages(Kernel->mmu,
1896                                      Node->Virtual.pageTables[Kernel->core],
1897                                      Node->Virtual.pageCount));
1898             }
1899             Node->Virtual.pageTables[Kernel->core]  = gcvNULL;
1900             Node->Virtual.lockKernels[Kernel->core] = gcvNULL;
1901         }
1902
1903         /* Unlock the pages. */
1904         gcmkVERIFY_OK(
1905             gckOS_UnlockPages(os,
1906                               Node->Virtual.physical,
1907                               Node->Virtual.bytes,
1908                               Node->Virtual.logical
1909                               ));
1910
1911         Node->Virtual.lockeds[Kernel->core]--;
1912     }
1913
1914     if (acquired)
1915     {
1916         /* Release the mutex. */
1917         gcmkVERIFY_OK(gckOS_ReleaseMutex(os, Node->Virtual.mutex));
1918     }
1919
1920     /* Return the status. */
1921     gcmkFOOTER();
1922     return status;
1923 }
1924
1925 /*******************************************************************************
1926 **
1927 **  gckVIDMEM_Unlock
1928 **
1929 **  Unlock a video memory node.
1930 **
1931 **  INPUT:
1932 **
1933 **      gckKERNEL Kernel
1934 **          Pointer to an gckKERNEL object.
1935 **
1936 **      gcuVIDMEM_NODE_PTR Node
1937 **          Pointer to a locked gcuVIDMEM_NODE union.
1938 **
1939 **      gceSURF_TYPE Type
1940 **          Type of surface to unlock.
1941 **
1942 **      gctBOOL * Asynchroneous
1943 **          Pointer to a variable specifying whether the surface should be
1944 **          unlocked asynchroneously or not.
1945 **
1946 **  OUTPUT:
1947 **
1948 **      gctBOOL * Asynchroneous
1949 **          Pointer to a variable receiving the number of bytes used in the
1950 **          command buffer specified by 'Commands'.  If gcvNULL, there is no
1951 **          command buffer.
1952 */
1953 gceSTATUS
1954 gckVIDMEM_Unlock(
1955     IN gckKERNEL Kernel,
1956     IN gcuVIDMEM_NODE_PTR Node,
1957     IN gceSURF_TYPE Type,
1958     IN OUT gctBOOL * Asynchroneous
1959     )
1960 {
1961     gceSTATUS status;
1962     gckHARDWARE hardware;
1963     gctPOINTER buffer;
1964     gctSIZE_T requested, bufferSize;
1965     gckCOMMAND command = gcvNULL;
1966     gceKERNEL_FLUSH flush;
1967     gckOS os = gcvNULL;
1968     gctBOOL acquired = gcvFALSE;
1969     gctBOOL commitEntered = gcvFALSE;
1970     gctINT32 i, totalLocked;
1971
1972     gcmkHEADER_ARG("Node=0x%x Type=%d *Asynchroneous=%d",
1973                    Node, Type, gcmOPT_VALUE(Asynchroneous));
1974
1975     /* Verify the arguments. */
1976     if ((Node == gcvNULL)
1977     ||  (Node->VidMem.memory == gcvNULL)
1978     )
1979     {
1980         /* Invalid object. */
1981         gcmkONERROR(gcvSTATUS_INVALID_OBJECT);
1982     }
1983
1984     /**************************** Video Memory ********************************/
1985
1986     if (Node->VidMem.memory->object.type == gcvOBJ_VIDMEM)
1987     {
1988         if (Node->VidMem.locked <= 0)
1989         {
1990             /* The surface was not locked. */
1991             status = gcvSTATUS_MEMORY_UNLOCKED;
1992             goto OnError;
1993         }
1994
1995         /* Decrement the lock count. */
1996         Node->VidMem.locked --;
1997
1998         if (Asynchroneous != gcvNULL)
1999         {
2000             /* No need for any events. */
2001             *Asynchroneous = gcvFALSE;
2002         }
2003
2004         gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
2005                       "Unlocked node 0x%x (%d)",
2006                       Node,
2007                       Node->VidMem.locked);
2008
2009 #ifdef __QNXNTO__
2010         /* Unmap the video memory */
2011         if ((Node->VidMem.locked == 0) && (Node->VidMem.logical != gcvNULL))
2012         {
2013             if (Kernel->core == gcvCORE_VG)
2014             {
2015                 gckKERNEL_UnmapVideoMemory(Kernel,
2016                                            Node->VidMem.logical,
2017                                            Node->VidMem.processID,
2018                                            Node->VidMem.bytes);
2019                 Node->VidMem.logical = gcvNULL;
2020             }
2021         }
2022 #endif /* __QNXNTO__ */
2023
2024         if (Node->VidMem.freePending && (Node->VidMem.locked == 0))
2025         {
2026             /* Client has unlocked node previously attempted to be freed by compositor. Free now. */
2027             Node->VidMem.freePending = gcvFALSE;
2028             gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
2029                            "Deferred-freeing Node 0x%x.",
2030                            Node);
2031             gcmkONERROR(gckVIDMEM_Free(Node));
2032         }
2033     }
2034
2035     /*************************** Virtual Memory *******************************/
2036
2037     else
2038     {
2039         /* Verify the gckHARDWARE object pointer. */
2040         hardware = Kernel->hardware;
2041         gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE);
2042
2043         /* Verify the gckCOMMAND object pointer. */
2044         command = Kernel->command;
2045         gcmkVERIFY_OBJECT(command, gcvOBJ_COMMAND);
2046
2047         /* Get the gckOS object pointer. */
2048         os = Kernel->os;
2049         gcmkVERIFY_OBJECT(os, gcvOBJ_OS);
2050
2051         /* Grab the mutex. */
2052         gcmkONERROR(
2053             gckOS_AcquireMutex(os, Node->Virtual.mutex, gcvINFINITE));
2054
2055         acquired = gcvTRUE;
2056
2057         if (Asynchroneous == gcvNULL)
2058         {
2059             if (Node->Virtual.lockeds[Kernel->core] == 0)
2060             {
2061                 status = gcvSTATUS_MEMORY_UNLOCKED;
2062                 goto OnError;
2063             }
2064
2065             /* Decrement lock count. */
2066             -- Node->Virtual.lockeds[Kernel->core];
2067
2068             /* See if we can unlock the resources. */
2069             if (Node->Virtual.lockeds[Kernel->core] == 0)
2070             {
2071                 /* Free the page table. */
2072                 if (Node->Virtual.pageTables[Kernel->core] != gcvNULL)
2073                 {
2074 #if gcdENABLE_VG
2075                     if (Kernel->vg != gcvNULL)
2076                     {
2077                         gcmkONERROR(
2078                             gckVGMMU_FreePages(Kernel->vg->mmu,
2079                                              Node->Virtual.pageTables[Kernel->core],
2080                                              Node->Virtual.pageCount));
2081                     }
2082                     else
2083 #endif
2084                     {
2085                         gcmkONERROR(
2086                             gckMMU_FreePages(Kernel->mmu,
2087                                              Node->Virtual.pageTables[Kernel->core],
2088                                              Node->Virtual.pageCount));
2089                     }
2090                     /* Mark page table as freed. */
2091                     Node->Virtual.pageTables[Kernel->core] = gcvNULL;
2092                     Node->Virtual.lockKernels[Kernel->core] = gcvNULL;
2093                 }
2094
2095 #ifdef __QNXNTO__
2096                 /* Mark node as unlocked. */
2097                 Node->Virtual.unlockPendings[Kernel->core] = gcvFALSE;
2098 #endif
2099             }
2100
2101             for (i = 0, totalLocked = 0; i < gcdMAX_GPU_COUNT; i++)
2102             {
2103                 totalLocked += Node->Virtual.lockeds[i];
2104             }
2105
2106             if (totalLocked == 0)
2107             {
2108                 /* Owner have already freed this node
2109                 ** and we are the last one to unlock, do
2110                 ** real free */
2111                 if (Node->Virtual.freed)
2112                 {
2113                     /* Free the virtual memory. */
2114                     gcmkVERIFY_OK(gckOS_FreePagedMemory(Kernel->os,
2115                                                         Node->Virtual.physical,
2116                                                         Node->Virtual.bytes));
2117
2118                     /* Release mutex before node is destroyed */
2119                     gcmkVERIFY_OK(gckOS_ReleaseMutex(os, Node->Virtual.mutex));
2120
2121                     acquired = gcvFALSE;
2122
2123                     /* Destroy the gcuVIDMEM_NODE union. */
2124                     gcmkVERIFY_OK(gckVIDMEM_DestroyVirtual(Node));
2125
2126                     /* Node has been destroyed, so we should not touch it any more */
2127                     gcmkFOOTER();
2128                     return gcvSTATUS_OK;
2129                 }
2130             }
2131
2132             gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
2133                            "Unmapped virtual node 0x%x from 0x%08X",
2134                            Node, Node->Virtual.addresses[Kernel->core]);
2135
2136         }
2137
2138         else
2139         {
2140             /* If we need to unlock a node from virtual memory we have to be
2141             ** very carefull.  If the node is still inside the caches we
2142             ** might get a bus error later if the cache line needs to be
2143             ** replaced.  So - we have to flush the caches before we do
2144             ** anything. */
2145
2146             /* gckCommand_EnterCommit() can't be called in interrupt handler because
2147             ** of a dead lock situation:
2148             ** process call Command_Commit(), and acquire Command->mutexQueue in
2149             ** gckCOMMAND_EnterCommit(). Then it will wait for a signal which depends
2150             ** on interrupt handler to generate, if interrupt handler enter
2151             ** gckCommand_EnterCommit(), process will never get the signal. */
2152
2153             /* So, flush cache when we still in process context, and then ask caller to
2154             ** schedule a event. */
2155
2156             gcmkONERROR(
2157                 gckOS_UnlockPages(os,
2158                               Node->Virtual.physical,
2159                               Node->Virtual.bytes,
2160                               Node->Virtual.logical));
2161
2162             if (!Node->Virtual.contiguous
2163             &&  (Node->Virtual.lockeds[Kernel->core] == 1)
2164 #if gcdENABLE_VG
2165             && (Kernel->vg == gcvNULL)
2166 #endif
2167             )
2168             {
2169                 if (Type == gcvSURF_BITMAP)
2170                 {
2171                     /* Flush 2D cache. */
2172                     flush = gcvFLUSH_2D;
2173                 }
2174                 else if (Type == gcvSURF_RENDER_TARGET)
2175                 {
2176                     /* Flush color cache. */
2177                     flush = gcvFLUSH_COLOR;
2178                 }
2179                 else if (Type == gcvSURF_DEPTH)
2180                 {
2181                     /* Flush depth cache. */
2182                     flush = gcvFLUSH_DEPTH;
2183                 }
2184                 else
2185                 {
2186                     /* No flush required. */
2187                     flush = (gceKERNEL_FLUSH) 0;
2188                 }
2189                 if(hardware)
2190                 {
2191                     gcmkONERROR(
2192                         gckHARDWARE_Flush(hardware, flush, gcvNULL, &requested));
2193
2194                     if (requested != 0)
2195                     {
2196                         /* Acquire the command queue. */
2197                         gcmkONERROR(gckCOMMAND_EnterCommit(command, gcvFALSE));
2198                         commitEntered = gcvTRUE;
2199
2200                         gcmkONERROR(gckCOMMAND_Reserve(
2201                             command, requested, &buffer, &bufferSize
2202                             ));
2203
2204                         gcmkONERROR(gckHARDWARE_Flush(
2205                             hardware, flush, buffer, &bufferSize
2206                             ));
2207
2208                         /* Mark node as pending. */
2209 #ifdef __QNXNTO__
2210                         Node->Virtual.unlockPendings[Kernel->core] = gcvTRUE;
2211 #endif
2212
2213                         gcmkONERROR(gckCOMMAND_Execute(command, requested));
2214
2215                         /* Release the command queue. */
2216                         gcmkONERROR(gckCOMMAND_ExitCommit(command, gcvFALSE));
2217                         commitEntered = gcvFALSE;
2218                     }
2219                 }
2220                 else
2221                 {
2222                     gckOS_Print("Hardware already is freed.\n");
2223                 }
2224             }
2225
2226             gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
2227                            "Scheduled unlock for virtual node 0x%x",
2228                            Node);
2229
2230             /* Schedule the surface to be unlocked. */
2231             *Asynchroneous = gcvTRUE;
2232         }
2233
2234         /* Release the mutex. */
2235         gcmkVERIFY_OK(gckOS_ReleaseMutex(os, Node->Virtual.mutex));
2236
2237         acquired = gcvFALSE;
2238     }
2239
2240     /* Success. */
2241     gcmkFOOTER_ARG("*Asynchroneous=%d", gcmOPT_VALUE(Asynchroneous));
2242     return gcvSTATUS_OK;
2243
2244 OnError:
2245     if (commitEntered)
2246     {
2247         /* Release the command queue mutex. */
2248         gcmkVERIFY_OK(gckCOMMAND_ExitCommit(command, gcvFALSE));
2249     }
2250
2251     if (acquired)
2252     {
2253         /* Release the mutex. */
2254         gcmkVERIFY_OK(gckOS_ReleaseMutex(os, Node->Virtual.mutex));
2255     }
2256
2257     /* Return the status. */
2258     gcmkFOOTER();
2259     return status;
2260 }