drivers/gpu/drm/drm_mm.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2006 Tungsten Graphics, Inc., Bismarck, ND., USA.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  *
  27  **************************************************************************/
  28
  29 /*
  30  * Generic simple memory manager implementation. Intended to be used as a base
  31  * class implementation for more advanced memory managers.
  32  *
  33  * Note that the algorithm used is quite simple and there might be substantial
  34  * performance gains if a smarter free list is implemented. Currently it is just an
  35  * unordered stack of free regions. This could easily be improved if an RB-tree
  36  * is used instead. At least if we expect heavy fragmentation.
  37  *
  38  * Aligned allocations can also see improvement.
  39  *
  40  * Authors:
  41  * Thomas Hellström <thomas-at-tungstengraphics-dot-com>
  42  */
  43
  44 #include <drm/drmP.h>
  45 #include <drm/drm_mm.h>
  46 #include <linux/slab.h>
  47 #include <linux/seq_file.h>
  48 #include <linux/export.h>
  49
  50 #define MM_UNUSED_TARGET 4
  51
  52 static struct drm_mm_node *drm_mm_search_free_generic(const struct drm_mm *mm,
  53                                                 unsigned long size,
  54                                                 unsigned alignment,
  55                                                 unsigned long color,
  56                                                 enum drm_mm_search_flags flags);
  57 static struct drm_mm_node *drm_mm_search_free_in_range_generic(const struct drm_mm *mm,
  58                                                 unsigned long size,
  59                                                 unsigned alignment,
  60                                                 unsigned long color,
  61                                                 unsigned long start,
  62                                                 unsigned long end,
  63                                                 enum drm_mm_search_flags flags);
  64
  65 static void drm_mm_insert_helper(struct drm_mm_node *hole_node,
  66                                  struct drm_mm_node *node,
  67                                  unsigned long size, unsigned alignment,
  68                                  unsigned long color)
  69 {
  70         struct drm_mm *mm = hole_node->mm;
  71         unsigned long hole_start = drm_mm_hole_node_start(hole_node);
  72         unsigned long hole_end = drm_mm_hole_node_end(hole_node);
  73         unsigned long adj_start = hole_start;
  74         unsigned long adj_end = hole_end;
  75
  76         BUG_ON(node->allocated);
  77
  78         if (mm->color_adjust)
  79                 mm->color_adjust(hole_node, color, &adj_start, &adj_end);
  80
  81         if (alignment) {
  82                 unsigned tmp = adj_start % alignment;
  83                 if (tmp)
  84                         adj_start += alignment - tmp;
  85         }
  86
  87         if (adj_start == hole_start) {
  88                 hole_node->hole_follows = 0;
  89                 list_del(&hole_node->hole_stack);
  90         }
  91
  92         node->start = adj_start;
  93         node->size = size;
  94         node->mm = mm;
  95         node->color = color;
  96         node->allocated = 1;
  97
  98         INIT_LIST_HEAD(&node->hole_stack);
  99         list_add(&node->node_list, &hole_node->node_list);
 100
 101         BUG_ON(node->start + node->size > adj_end);
 102
 103         node->hole_follows = 0;
 104         if (__drm_mm_hole_node_start(node) < hole_end) {
 105                 list_add(&node->hole_stack, &mm->hole_stack);
 106                 node->hole_follows = 1;
 107         }
 108 }
 109
 110 int drm_mm_reserve_node(struct drm_mm *mm, struct drm_mm_node *node)
 111 {
 112         struct drm_mm_node *hole;
 113         unsigned long end = node->start + node->size;
 114         unsigned long hole_start;
 115         unsigned long hole_end;
 116
 117         BUG_ON(node == NULL);
 118
 119         /* Find the relevant hole to add our node to */
 120         drm_mm_for_each_hole(hole, mm, hole_start, hole_end) {
 121                 if (hole_start > node->start || hole_end < end)
 122                         continue;
 123
 124                 node->mm = mm;
 125                 node->allocated = 1;
 126
 127                 INIT_LIST_HEAD(&node->hole_stack);
 128                 list_add(&node->node_list, &hole->node_list);
 129
 130                 if (node->start == hole_start) {
 131                         hole->hole_follows = 0;
 132                         list_del_init(&hole->hole_stack);
 133                 }
 134
 135                 node->hole_follows = 0;
 136                 if (end != hole_end) {
 137                         list_add(&node->hole_stack, &mm->hole_stack);
 138                         node->hole_follows = 1;
 139                 }
 140
 141                 return 0;
 142         }
 143
 144         WARN(1, "no hole found for node 0x%lx + 0x%lx\n",
 145              node->start, node->size);
 146         return -ENOSPC;
 147 }
 148 EXPORT_SYMBOL(drm_mm_reserve_node);
 149
 150 /**
 151  * Search for free space and insert a preallocated memory node. Returns
 152  * -ENOSPC if no suitable free area is available. The preallocated memory node
 153  * must be cleared.
 154  */
 155 int drm_mm_insert_node_generic(struct drm_mm *mm, struct drm_mm_node *node,
 156                                unsigned long size, unsigned alignment,
 157                                unsigned long color,
 158                                enum drm_mm_search_flags flags)
 159 {
 160         struct drm_mm_node *hole_node;
 161
 162         hole_node = drm_mm_search_free_generic(mm, size, alignment,
 163                                                color, flags);
 164         if (!hole_node)
 165                 return -ENOSPC;
 166
 167         drm_mm_insert_helper(hole_node, node, size, alignment, color);
 168         return 0;
 169 }
 170 EXPORT_SYMBOL(drm_mm_insert_node_generic);
 171
 172 static void drm_mm_insert_helper_range(struct drm_mm_node *hole_node,
 173                                        struct drm_mm_node *node,
 174                                        unsigned long size, unsigned alignment,
 175                                        unsigned long color,
 176                                        unsigned long start, unsigned long end)
 177 {
 178         struct drm_mm *mm = hole_node->mm;
 179         unsigned long hole_start = drm_mm_hole_node_start(hole_node);
 180         unsigned long hole_end = drm_mm_hole_node_end(hole_node);
 181         unsigned long adj_start = hole_start;
 182         unsigned long adj_end = hole_end;
 183
 184         BUG_ON(!hole_node->hole_follows || node->allocated);
 185
 186         if (adj_start < start)
 187                 adj_start = start;
 188         if (adj_end > end)
 189                 adj_end = end;
 190
 191         if (mm->color_adjust)
 192                 mm->color_adjust(hole_node, color, &adj_start, &adj_end);
 193
 194         if (alignment) {
 195                 unsigned tmp = adj_start % alignment;
 196                 if (tmp)
 197                         adj_start += alignment - tmp;
 198         }
 199
 200         if (adj_start == hole_start) {
 201                 hole_node->hole_follows = 0;
 202                 list_del(&hole_node->hole_stack);
 203         }
 204
 205         node->start = adj_start;
 206         node->size = size;
 207         node->mm = mm;
 208         node->color = color;
 209         node->allocated = 1;
 210
 211         INIT_LIST_HEAD(&node->hole_stack);
 212         list_add(&node->node_list, &hole_node->node_list);
 213
 214         BUG_ON(node->start + node->size > adj_end);
 215         BUG_ON(node->start + node->size > end);
 216
 217         node->hole_follows = 0;
 218         if (__drm_mm_hole_node_start(node) < hole_end) {
 219                 list_add(&node->hole_stack, &mm->hole_stack);
 220                 node->hole_follows = 1;
 221         }
 222 }
 223
 224 /**
 225  * Search for free space and insert a preallocated memory node. Returns
 226  * -ENOSPC if no suitable free area is available. This is for range
 227  * restricted allocations. The preallocated memory node must be cleared.
 228  */
 229 int drm_mm_insert_node_in_range_generic(struct drm_mm *mm, struct drm_mm_node *node,
 230                                         unsigned long size, unsigned alignment, unsigned long color,
 231                                         unsigned long start, unsigned long end,
 232                                         enum drm_mm_search_flags flags)
 233 {
 234         struct drm_mm_node *hole_node;
 235
 236         hole_node = drm_mm_search_free_in_range_generic(mm,
 237                                                         size, alignment, color,
 238                                                         start, end, flags);
 239         if (!hole_node)
 240                 return -ENOSPC;
 241
 242         drm_mm_insert_helper_range(hole_node, node,
 243                                    size, alignment, color,
 244                                    start, end);
 245         return 0;
 246 }
 247 EXPORT_SYMBOL(drm_mm_insert_node_in_range_generic);
 248
 249 /**
 250  * Remove a memory node from the allocator.
 251  */
 252 void drm_mm_remove_node(struct drm_mm_node *node)
 253 {
 254         struct drm_mm *mm = node->mm;
 255         struct drm_mm_node *prev_node;
 256
 257         BUG_ON(node->scanned_block || node->scanned_prev_free
 258                                    || node->scanned_next_free);
 259
 260         prev_node =
 261             list_entry(node->node_list.prev, struct drm_mm_node, node_list);
 262
 263         if (node->hole_follows) {
 264                 BUG_ON(__drm_mm_hole_node_start(node) ==
 265                        __drm_mm_hole_node_end(node));
 266                 list_del(&node->hole_stack);
 267         } else
 268                 BUG_ON(__drm_mm_hole_node_start(node) !=
 269                        __drm_mm_hole_node_end(node));
 270
 271
 272         if (!prev_node->hole_follows) {
 273                 prev_node->hole_follows = 1;
 274                 list_add(&prev_node->hole_stack, &mm->hole_stack);
 275         } else
 276                 list_move(&prev_node->hole_stack, &mm->hole_stack);
 277
 278         list_del(&node->node_list);
 279         node->allocated = 0;
 280 }
 281 EXPORT_SYMBOL(drm_mm_remove_node);
 282
 283 static int check_free_hole(unsigned long start, unsigned long end,
 284                            unsigned long size, unsigned alignment)
 285 {
 286         if (end - start < size)
 287                 return 0;
 288
 289         if (alignment) {
 290                 unsigned tmp = start % alignment;
 291                 if (tmp)
 292                         start += alignment - tmp;
 293         }
 294
 295         return end >= start + size;
 296 }
 297
 298 static struct drm_mm_node *drm_mm_search_free_generic(const struct drm_mm *mm,
 299                                                       unsigned long size,
 300                                                       unsigned alignment,
 301                                                       unsigned long color,
 302                                                       enum drm_mm_search_flags flags)
 303 {
 304         struct drm_mm_node *entry;
 305         struct drm_mm_node *best;
 306         unsigned long adj_start;
 307         unsigned long adj_end;
 308         unsigned long best_size;
 309
 310         BUG_ON(mm->scanned_blocks);
 311
 312         best = NULL;
 313         best_size = ~0UL;
 314
 315         drm_mm_for_each_hole(entry, mm, adj_start, adj_end) {
 316                 if (mm->color_adjust) {
 317                         mm->color_adjust(entry, color, &adj_start, &adj_end);
 318                         if (adj_end <= adj_start)
 319                                 continue;
 320                 }
 321
 322                 if (!check_free_hole(adj_start, adj_end, size, alignment))
 323                         continue;
 324
 325                 if (!(flags & DRM_MM_SEARCH_BEST))
 326                         return entry;
 327
 328                 if (entry->size < best_size) {
 329                         best = entry;
 330                         best_size = entry->size;
 331                 }
 332         }
 333
 334         return best;
 335 }
 336
 337 static struct drm_mm_node *drm_mm_search_free_in_range_generic(const struct drm_mm *mm,
 338                                                         unsigned long size,
 339                                                         unsigned alignment,
 340                                                         unsigned long color,
 341                                                         unsigned long start,
 342                                                         unsigned long end,
 343                                                         enum drm_mm_search_flags flags)
 344 {
 345         struct drm_mm_node *entry;
 346         struct drm_mm_node *best;
 347         unsigned long adj_start;
 348         unsigned long adj_end;
 349         unsigned long best_size;
 350
 351         BUG_ON(mm->scanned_blocks);
 352
 353         best = NULL;
 354         best_size = ~0UL;
 355
 356         drm_mm_for_each_hole(entry, mm, adj_start, adj_end) {
 357                 if (adj_start < start)
 358                         adj_start = start;
 359                 if (adj_end > end)
 360                         adj_end = end;
 361
 362                 if (mm->color_adjust) {
 363                         mm->color_adjust(entry, color, &adj_start, &adj_end);
 364                         if (adj_end <= adj_start)
 365                                 continue;
 366                 }
 367
 368                 if (!check_free_hole(adj_start, adj_end, size, alignment))
 369                         continue;
 370
 371                 if (!(flags & DRM_MM_SEARCH_BEST))
 372                         return entry;
 373
 374                 if (entry->size < best_size) {
 375                         best = entry;
 376                         best_size = entry->size;
 377                 }
 378         }
 379
 380         return best;
 381 }
 382
 383 /**
 384  * Moves an allocation. To be used with embedded struct drm_mm_node.
 385  */
 386 void drm_mm_replace_node(struct drm_mm_node *old, struct drm_mm_node *new)
 387 {
 388         list_replace(&old->node_list, &new->node_list);
 389         list_replace(&old->hole_stack, &new->hole_stack);
 390         new->hole_follows = old->hole_follows;
 391         new->mm = old->mm;
 392         new->start = old->start;
 393         new->size = old->size;
 394         new->color = old->color;
 395
 396         old->allocated = 0;
 397         new->allocated = 1;
 398 }
 399 EXPORT_SYMBOL(drm_mm_replace_node);
 400
 401 /**
 402  * Initializa lru scanning.
 403  *
 404  * This simply sets up the scanning routines with the parameters for the desired
 405  * hole.
 406  *
 407  * Warning: As long as the scan list is non-empty, no other operations than
 408  * adding/removing nodes to/from the scan list are allowed.
 409  */
 410 void drm_mm_init_scan(struct drm_mm *mm,
 411                       unsigned long size,
 412                       unsigned alignment,
 413                       unsigned long color)
 414 {
 415         mm->scan_color = color;
 416         mm->scan_alignment = alignment;
 417         mm->scan_size = size;
 418         mm->scanned_blocks = 0;
 419         mm->scan_hit_start = 0;
 420         mm->scan_hit_end = 0;
 421         mm->scan_check_range = 0;
 422         mm->prev_scanned_node = NULL;
 423 }
 424 EXPORT_SYMBOL(drm_mm_init_scan);
 425
 426 /**
 427  * Initializa lru scanning.
 428  *
 429  * This simply sets up the scanning routines with the parameters for the desired
 430  * hole. This version is for range-restricted scans.
 431  *
 432  * Warning: As long as the scan list is non-empty, no other operations than
 433  * adding/removing nodes to/from the scan list are allowed.
 434  */
 435 void drm_mm_init_scan_with_range(struct drm_mm *mm,
 436                                  unsigned long size,
 437                                  unsigned alignment,
 438                                  unsigned long color,
 439                                  unsigned long start,
 440                                  unsigned long end)
 441 {
 442         mm->scan_color = color;
 443         mm->scan_alignment = alignment;
 444         mm->scan_size = size;
 445         mm->scanned_blocks = 0;
 446         mm->scan_hit_start = 0;
 447         mm->scan_hit_end = 0;
 448         mm->scan_start = start;
 449         mm->scan_end = end;
 450         mm->scan_check_range = 1;
 451         mm->prev_scanned_node = NULL;
 452 }
 453 EXPORT_SYMBOL(drm_mm_init_scan_with_range);
 454
 455 /**
 456  * Add a node to the scan list that might be freed to make space for the desired
 457  * hole.
 458  *
 459  * Returns non-zero, if a hole has been found, zero otherwise.
 460  */
 461 int drm_mm_scan_add_block(struct drm_mm_node *node)
 462 {
 463         struct drm_mm *mm = node->mm;
 464         struct drm_mm_node *prev_node;
 465         unsigned long hole_start, hole_end;
 466         unsigned long adj_start, adj_end;
 467
 468         mm->scanned_blocks++;
 469
 470         BUG_ON(node->scanned_block);
 471         node->scanned_block = 1;
 472
 473         prev_node = list_entry(node->node_list.prev, struct drm_mm_node,
 474                                node_list);
 475
 476         node->scanned_preceeds_hole = prev_node->hole_follows;
 477         prev_node->hole_follows = 1;
 478         list_del(&node->node_list);
 479         node->node_list.prev = &prev_node->node_list;
 480         node->node_list.next = &mm->prev_scanned_node->node_list;
 481         mm->prev_scanned_node = node;
 482
 483         adj_start = hole_start = drm_mm_hole_node_start(prev_node);
 484         adj_end = hole_end = drm_mm_hole_node_end(prev_node);
 485
 486         if (mm->scan_check_range) {
 487                 if (adj_start < mm->scan_start)
 488                         adj_start = mm->scan_start;
 489                 if (adj_end > mm->scan_end)
 490                         adj_end = mm->scan_end;
 491         }
 492
 493         if (mm->color_adjust)
 494                 mm->color_adjust(prev_node, mm->scan_color,
 495                                  &adj_start, &adj_end);
 496
 497         if (check_free_hole(adj_start, adj_end,
 498                             mm->scan_size, mm->scan_alignment)) {
 499                 mm->scan_hit_start = hole_start;
 500                 mm->scan_hit_end = hole_end;
 501                 return 1;
 502         }
 503
 504         return 0;
 505 }
 506 EXPORT_SYMBOL(drm_mm_scan_add_block);
 507
 508 /**
 509  * Remove a node from the scan list.
 510  *
 511  * Nodes _must_ be removed in the exact same order from the scan list as they
 512  * have been added, otherwise the internal state of the memory manager will be
 513  * corrupted.
 514  *
 515  * When the scan list is empty, the selected memory nodes can be freed. An
 516  * immediately following drm_mm_search_free with !DRM_MM_SEARCH_BEST will then
 517  * return the just freed block (because its at the top of the free_stack list).
 518  *
 519  * Returns one if this block should be evicted, zero otherwise. Will always
 520  * return zero when no hole has been found.
 521  */
 522 int drm_mm_scan_remove_block(struct drm_mm_node *node)
 523 {
 524         struct drm_mm *mm = node->mm;
 525         struct drm_mm_node *prev_node;
 526
 527         mm->scanned_blocks--;
 528
 529         BUG_ON(!node->scanned_block);
 530         node->scanned_block = 0;
 531
 532         prev_node = list_entry(node->node_list.prev, struct drm_mm_node,
 533                                node_list);
 534
 535         prev_node->hole_follows = node->scanned_preceeds_hole;
 536         list_add(&node->node_list, &prev_node->node_list);
 537
 538          return (drm_mm_hole_node_end(node) > mm->scan_hit_start &&
 539                  node->start < mm->scan_hit_end);
 540 }
 541 EXPORT_SYMBOL(drm_mm_scan_remove_block);
 542
 543 int drm_mm_clean(struct drm_mm * mm)
 544 {
 545         struct list_head *head = &mm->head_node.node_list;
 546
 547         return (head->next->next == head);
 548 }
 549 EXPORT_SYMBOL(drm_mm_clean);
 550
 551 void drm_mm_init(struct drm_mm * mm, unsigned long start, unsigned long size)
 552 {
 553         INIT_LIST_HEAD(&mm->hole_stack);
 554         mm->scanned_blocks = 0;
 555
 556         /* Clever trick to avoid a special case in the free hole tracking. */
 557         INIT_LIST_HEAD(&mm->head_node.node_list);
 558         INIT_LIST_HEAD(&mm->head_node.hole_stack);
 559         mm->head_node.hole_follows = 1;
 560         mm->head_node.scanned_block = 0;
 561         mm->head_node.scanned_prev_free = 0;
 562         mm->head_node.scanned_next_free = 0;
 563         mm->head_node.mm = mm;
 564         mm->head_node.start = start + size;
 565         mm->head_node.size = start - mm->head_node.start;
 566         list_add_tail(&mm->head_node.hole_stack, &mm->hole_stack);
 567
 568         mm->color_adjust = NULL;
 569 }
 570 EXPORT_SYMBOL(drm_mm_init);
 571
 572 void drm_mm_takedown(struct drm_mm * mm)
 573 {
 574         WARN(!list_empty(&mm->head_node.node_list),
 575              "Memory manager not clean during takedown.\n");
 576 }
 577 EXPORT_SYMBOL(drm_mm_takedown);
 578
 579 static unsigned long drm_mm_debug_hole(struct drm_mm_node *entry,
 580                                        const char *prefix)
 581 {
 582         unsigned long hole_start, hole_end, hole_size;
 583
 584         if (entry->hole_follows) {
 585                 hole_start = drm_mm_hole_node_start(entry);
 586                 hole_end = drm_mm_hole_node_end(entry);
 587                 hole_size = hole_end - hole_start;
 588                 printk(KERN_DEBUG "%s 0x%08lx-0x%08lx: %8lu: free\n",
 589                         prefix, hole_start, hole_end,
 590                         hole_size);
 591                 return hole_size;
 592         }
 593
 594         return 0;
 595 }
 596
 597 void drm_mm_debug_table(struct drm_mm *mm, const char *prefix)
 598 {
 599         struct drm_mm_node *entry;
 600         unsigned long total_used = 0, total_free = 0, total = 0;
 601
 602         total_free += drm_mm_debug_hole(&mm->head_node, prefix);
 603
 604         drm_mm_for_each_node(entry, mm) {
 605                 printk(KERN_DEBUG "%s 0x%08lx-0x%08lx: %8lu: used\n",
 606                         prefix, entry->start, entry->start + entry->size,
 607                         entry->size);
 608                 total_used += entry->size;
 609                 total_free += drm_mm_debug_hole(entry, prefix);
 610         }
 611         total = total_free + total_used;
 612
 613         printk(KERN_DEBUG "%s total: %lu, used %lu free %lu\n", prefix, total,
 614                 total_used, total_free);
 615 }
 616 EXPORT_SYMBOL(drm_mm_debug_table);
 617
 618 #if defined(CONFIG_DEBUG_FS)
 619 static unsigned long drm_mm_dump_hole(struct seq_file *m, struct drm_mm_node *entry)
 620 {
 621         unsigned long hole_start, hole_end, hole_size;
 622
 623         if (entry->hole_follows) {
 624                 hole_start = drm_mm_hole_node_start(entry);
 625                 hole_end = drm_mm_hole_node_end(entry);
 626                 hole_size = hole_end - hole_start;
 627                 seq_printf(m, "0x%08lx-0x%08lx: 0x%08lx: free\n",
 628                                 hole_start, hole_end, hole_size);
 629                 return hole_size;
 630         }
 631
 632         return 0;
 633 }
 634
 635 int drm_mm_dump_table(struct seq_file *m, struct drm_mm *mm)
 636 {
 637         struct drm_mm_node *entry;
 638         unsigned long total_used = 0, total_free = 0, total = 0;
 639
 640         total_free += drm_mm_dump_hole(m, &mm->head_node);
 641
 642         drm_mm_for_each_node(entry, mm) {
 643                 seq_printf(m, "0x%08lx-0x%08lx: 0x%08lx: used\n",
 644                                 entry->start, entry->start + entry->size,
 645                                 entry->size);
 646                 total_used += entry->size;
 647                 total_free += drm_mm_dump_hole(m, entry);
 648         }
 649         total = total_free + total_used;
 650
 651         seq_printf(m, "total: %lu, used %lu free %lu\n", total, total_used, total_free);
 652         return 0;
 653 }
 654 EXPORT_SYMBOL(drm_mm_dump_table);
 655 #endif