drivers/gpu/drm/nouveau/nouveau_fence.c

   1 /*
   2  * Copyright (C) 2007 Ben Skeggs.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining
   6  * a copy of this software and associated documentation files (the
   7  * "Software"), to deal in the Software without restriction, including
   8  * without limitation the rights to use, copy, modify, merge, publish,
   9  * distribute, sublicense, and/or sell copies of the Software, and to
  10  * permit persons to whom the Software is furnished to do so, subject to
  11  * the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the
  14  * next paragraph) shall be included in all copies or substantial
  15  * portions of the Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  20  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  21  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  22  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  24  *
  25  */
  26
  27 #include "drmP.h"
  28 #include "drm.h"
  29
  30 #include <linux/ktime.h>
  31 #include <linux/hrtimer.h>
  32
  33 #include "nouveau_drv.h"
  34 #include "nouveau_ramht.h"
  35 #include "nouveau_dma.h"
  36
  37 #define USE_REFCNT(dev) (nouveau_private(dev)->chipset >= 0x10)
  38 #define USE_SEMA(dev) (nouveau_private(dev)->chipset >= 0x17)
  39
  40 struct nouveau_fence {
  41         struct nouveau_channel *channel;
  42         struct kref refcount;
  43         struct list_head entry;
  44
  45         uint32_t sequence;
  46         bool signalled;
  47
  48         void (*work)(void *priv, bool signalled);
  49         void *priv;
  50 };
  51
  52 struct nouveau_semaphore {
  53         struct kref ref;
  54         struct drm_device *dev;
  55         struct drm_mm_node *mem;
  56 };
  57
  58 static inline struct nouveau_fence *
  59 nouveau_fence(void *sync_obj)
  60 {
  61         return (struct nouveau_fence *)sync_obj;
  62 }
  63
  64 static void
  65 nouveau_fence_del(struct kref *ref)
  66 {
  67         struct nouveau_fence *fence =
  68                 container_of(ref, struct nouveau_fence, refcount);
  69
  70         nouveau_channel_ref(NULL, &fence->channel);
  71         kfree(fence);
  72 }
  73
  74 void
  75 nouveau_fence_update(struct nouveau_channel *chan)
  76 {
  77         struct drm_device *dev = chan->dev;
  78         struct nouveau_fence *tmp, *fence;
  79         uint32_t sequence;
  80
  81         spin_lock(&chan->fence.lock);
  82
  83         /* Fetch the last sequence if the channel is still up and running */
  84         if (likely(!list_empty(&chan->fence.pending))) {
  85                 if (USE_REFCNT(dev))
  86                         sequence = nvchan_rd32(chan, 0x48);
  87                 else
  88                         sequence = atomic_read(&chan->fence.last_sequence_irq);
  89
  90                 if (chan->fence.sequence_ack == sequence)
  91                         goto out;
  92                 chan->fence.sequence_ack = sequence;
  93         }
  94
  95         list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
  96                 if (fence->sequence > chan->fence.sequence_ack)
  97                         break;
  98
  99                 fence->signalled = true;
 100                 list_del(&fence->entry);
 101                 if (fence->work)
 102                         fence->work(fence->priv, true);
 103
 104                 kref_put(&fence->refcount, nouveau_fence_del);
 105         }
 106
 107 out:
 108         spin_unlock(&chan->fence.lock);
 109 }
 110
 111 int
 112 nouveau_fence_new(struct nouveau_channel *chan, struct nouveau_fence **pfence,
 113                   bool emit)
 114 {
 115         struct nouveau_fence *fence;
 116         int ret = 0;
 117
 118         fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 119         if (!fence)
 120                 return -ENOMEM;
 121         kref_init(&fence->refcount);
 122         nouveau_channel_ref(chan, &fence->channel);
 123
 124         if (emit)
 125                 ret = nouveau_fence_emit(fence);
 126
 127         if (ret)
 128                 nouveau_fence_unref(&fence);
 129         *pfence = fence;
 130         return ret;
 131 }
 132
 133 struct nouveau_channel *
 134 nouveau_fence_channel(struct nouveau_fence *fence)
 135 {
 136         return fence ? nouveau_channel_get_unlocked(fence->channel) : NULL;
 137 }
 138
 139 int
 140 nouveau_fence_emit(struct nouveau_fence *fence)
 141 {
 142         struct nouveau_channel *chan = fence->channel;
 143         struct drm_device *dev = chan->dev;
 144         struct drm_nouveau_private *dev_priv = dev->dev_private;
 145         int ret;
 146
 147         ret = RING_SPACE(chan, 2);
 148         if (ret)
 149                 return ret;
 150
 151         if (unlikely(chan->fence.sequence == chan->fence.sequence_ack - 1)) {
 152                 nouveau_fence_update(chan);
 153
 154                 BUG_ON(chan->fence.sequence ==
 155                        chan->fence.sequence_ack - 1);
 156         }
 157
 158         fence->sequence = ++chan->fence.sequence;
 159
 160         kref_get(&fence->refcount);
 161         spin_lock(&chan->fence.lock);
 162         list_add_tail(&fence->entry, &chan->fence.pending);
 163         spin_unlock(&chan->fence.lock);
 164
 165         if (USE_REFCNT(dev)) {
 166                 if (dev_priv->card_type < NV_C0)
 167                         BEGIN_RING(chan, 0, NV10_SUBCHAN_REF_CNT, 1);
 168                 else
 169                         BEGIN_NVC0(chan, 2, 0, NV10_SUBCHAN_REF_CNT, 1);
 170         } else {
 171                 BEGIN_RING(chan, NvSubSw, 0x0150, 1);
 172         }
 173         OUT_RING (chan, fence->sequence);
 174         FIRE_RING(chan);
 175
 176         return 0;
 177 }
 178
 179 void
 180 nouveau_fence_work(struct nouveau_fence *fence,
 181                    void (*work)(void *priv, bool signalled),
 182                    void *priv)
 183 {
 184         BUG_ON(fence->work);
 185
 186         spin_lock(&fence->channel->fence.lock);
 187
 188         if (fence->signalled) {
 189                 work(priv, true);
 190         } else {
 191                 fence->work = work;
 192                 fence->priv = priv;
 193         }
 194
 195         spin_unlock(&fence->channel->fence.lock);
 196 }
 197
 198 void
 199 __nouveau_fence_unref(void **sync_obj)
 200 {
 201         struct nouveau_fence *fence = nouveau_fence(*sync_obj);
 202
 203         if (fence)
 204                 kref_put(&fence->refcount, nouveau_fence_del);
 205         *sync_obj = NULL;
 206 }
 207
 208 void *
 209 __nouveau_fence_ref(void *sync_obj)
 210 {
 211         struct nouveau_fence *fence = nouveau_fence(sync_obj);
 212
 213         kref_get(&fence->refcount);
 214         return sync_obj;
 215 }
 216
 217 bool
 218 __nouveau_fence_signalled(void *sync_obj, void *sync_arg)
 219 {
 220         struct nouveau_fence *fence = nouveau_fence(sync_obj);
 221         struct nouveau_channel *chan = fence->channel;
 222
 223         if (fence->signalled)
 224                 return true;
 225
 226         nouveau_fence_update(chan);
 227         return fence->signalled;
 228 }
 229
 230 int
 231 __nouveau_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr)
 232 {
 233         unsigned long timeout = jiffies + (3 * DRM_HZ);
 234         unsigned long sleep_time = NSEC_PER_MSEC / 1000;
 235         ktime_t t;
 236         int ret = 0;
 237
 238         while (1) {
 239                 if (__nouveau_fence_signalled(sync_obj, sync_arg))
 240                         break;
 241
 242                 if (time_after_eq(jiffies, timeout)) {
 243                         ret = -EBUSY;
 244                         break;
 245                 }
 246
 247                 __set_current_state(intr ? TASK_INTERRUPTIBLE
 248                         : TASK_UNINTERRUPTIBLE);
 249                 if (lazy) {
 250                         t = ktime_set(0, sleep_time);
 251                         schedule_hrtimeout(&t, HRTIMER_MODE_REL);
 252                         sleep_time *= 2;
 253                         if (sleep_time > NSEC_PER_MSEC)
 254                                 sleep_time = NSEC_PER_MSEC;
 255                 }
 256
 257                 if (intr && signal_pending(current)) {
 258                         ret = -ERESTARTSYS;
 259                         break;
 260                 }
 261         }
 262
 263         __set_current_state(TASK_RUNNING);
 264
 265         return ret;
 266 }
 267
 268 static struct nouveau_semaphore *
 269 semaphore_alloc(struct drm_device *dev)
 270 {
 271         struct drm_nouveau_private *dev_priv = dev->dev_private;
 272         struct nouveau_semaphore *sema;
 273         int size = (dev_priv->chipset < 0x84) ? 4 : 16;
 274         int ret, i;
 275
 276         if (!USE_SEMA(dev))
 277                 return NULL;
 278
 279         sema = kmalloc(sizeof(*sema), GFP_KERNEL);
 280         if (!sema)
 281                 goto fail;
 282
 283         ret = drm_mm_pre_get(&dev_priv->fence.heap);
 284         if (ret)
 285                 goto fail;
 286
 287         spin_lock(&dev_priv->fence.lock);
 288         sema->mem = drm_mm_search_free(&dev_priv->fence.heap, size, 0, 0);
 289         if (sema->mem)
 290                 sema->mem = drm_mm_get_block_atomic(sema->mem, size, 0);
 291         spin_unlock(&dev_priv->fence.lock);
 292
 293         if (!sema->mem)
 294                 goto fail;
 295
 296         kref_init(&sema->ref);
 297         sema->dev = dev;
 298         for (i = sema->mem->start; i < sema->mem->start + size; i += 4)
 299                 nouveau_bo_wr32(dev_priv->fence.bo, i / 4, 0);
 300
 301         return sema;
 302 fail:
 303         kfree(sema);
 304         return NULL;
 305 }
 306
 307 static void
 308 semaphore_free(struct kref *ref)
 309 {
 310         struct nouveau_semaphore *sema =
 311                 container_of(ref, struct nouveau_semaphore, ref);
 312         struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
 313
 314         spin_lock(&dev_priv->fence.lock);
 315         drm_mm_put_block(sema->mem);
 316         spin_unlock(&dev_priv->fence.lock);
 317
 318         kfree(sema);
 319 }
 320
 321 static void
 322 semaphore_work(void *priv, bool signalled)
 323 {
 324         struct nouveau_semaphore *sema = priv;
 325         struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
 326
 327         if (unlikely(!signalled))
 328                 nouveau_bo_wr32(dev_priv->fence.bo, sema->mem->start / 4, 1);
 329
 330         kref_put(&sema->ref, semaphore_free);
 331 }
 332
 333 static int
 334 semaphore_acquire(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
 335 {
 336         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
 337         struct nouveau_fence *fence = NULL;
 338         u64 offset = chan->fence.vma.offset + sema->mem->start;
 339         int ret;
 340
 341         if (dev_priv->chipset < 0x84) {
 342                 ret = RING_SPACE(chan, 4);
 343                 if (ret)
 344                         return ret;
 345
 346                 BEGIN_RING(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 3);
 347                 OUT_RING  (chan, NvSema);
 348                 OUT_RING  (chan, offset);
 349                 OUT_RING  (chan, 1);
 350         } else
 351         if (dev_priv->chipset < 0xc0) {
 352                 ret = RING_SPACE(chan, 7);
 353                 if (ret)
 354                         return ret;
 355
 356                 BEGIN_RING(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 1);
 357                 OUT_RING  (chan, chan->vram_handle);
 358                 BEGIN_RING(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4);
 359                 OUT_RING  (chan, upper_32_bits(offset));
 360                 OUT_RING  (chan, lower_32_bits(offset));
 361                 OUT_RING  (chan, 1);
 362                 OUT_RING  (chan, 1); /* ACQUIRE_EQ */
 363         } else {
 364                 ret = RING_SPACE(chan, 5);
 365                 if (ret)
 366                         return ret;
 367
 368                 BEGIN_NVC0(chan, 2, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4);
 369                 OUT_RING  (chan, upper_32_bits(offset));
 370                 OUT_RING  (chan, lower_32_bits(offset));
 371                 OUT_RING  (chan, 1);
 372                 OUT_RING  (chan, 0x1001); /* ACQUIRE_EQ */
 373         }
 374
 375         /* Delay semaphore destruction until its work is done */
 376         ret = nouveau_fence_new(chan, &fence, true);
 377         if (ret)
 378                 return ret;
 379
 380         kref_get(&sema->ref);
 381         nouveau_fence_work(fence, semaphore_work, sema);
 382         nouveau_fence_unref(&fence);
 383         return 0;
 384 }
 385
 386 static int
 387 semaphore_release(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
 388 {
 389         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
 390         struct nouveau_fence *fence = NULL;
 391         u64 offset = chan->fence.vma.offset + sema->mem->start;
 392         int ret;
 393
 394         if (dev_priv->chipset < 0x84) {
 395                 ret = RING_SPACE(chan, 5);
 396                 if (ret)
 397                         return ret;
 398
 399                 BEGIN_RING(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 2);
 400                 OUT_RING  (chan, NvSema);
 401                 OUT_RING  (chan, offset);
 402                 BEGIN_RING(chan, 0, NV11_SUBCHAN_SEMAPHORE_RELEASE, 1);
 403                 OUT_RING  (chan, 1);
 404         } else
 405         if (dev_priv->chipset < 0xc0) {
 406                 ret = RING_SPACE(chan, 7);
 407                 if (ret)
 408                         return ret;
 409
 410                 BEGIN_RING(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 1);
 411                 OUT_RING  (chan, chan->vram_handle);
 412                 BEGIN_RING(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4);
 413                 OUT_RING  (chan, upper_32_bits(offset));
 414                 OUT_RING  (chan, lower_32_bits(offset));
 415                 OUT_RING  (chan, 1);
 416                 OUT_RING  (chan, 2); /* RELEASE */
 417         } else {
 418                 ret = RING_SPACE(chan, 5);
 419                 if (ret)
 420                         return ret;
 421
 422                 BEGIN_NVC0(chan, 2, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4);
 423                 OUT_RING  (chan, upper_32_bits(offset));
 424                 OUT_RING  (chan, lower_32_bits(offset));
 425                 OUT_RING  (chan, 1);
 426                 OUT_RING  (chan, 0x1002); /* RELEASE */
 427         }
 428
 429         /* Delay semaphore destruction until its work is done */
 430         ret = nouveau_fence_new(chan, &fence, true);
 431         if (ret)
 432                 return ret;
 433
 434         kref_get(&sema->ref);
 435         nouveau_fence_work(fence, semaphore_work, sema);
 436         nouveau_fence_unref(&fence);
 437         return 0;
 438 }
 439
 440 int
 441 nouveau_fence_sync(struct nouveau_fence *fence,
 442                    struct nouveau_channel *wchan)
 443 {
 444         struct nouveau_channel *chan = nouveau_fence_channel(fence);
 445         struct drm_device *dev = wchan->dev;
 446         struct nouveau_semaphore *sema;
 447         int ret = 0;
 448
 449         if (likely(!chan || chan == wchan ||
 450                    nouveau_fence_signalled(fence)))
 451                 goto out;
 452
 453         sema = semaphore_alloc(dev);
 454         if (!sema) {
 455                 /* Early card or broken userspace, fall back to
 456                  * software sync. */
 457                 ret = nouveau_fence_wait(fence, true, false);
 458                 goto out;
 459         }
 460
 461         /* try to take chan's mutex, if we can't take it right away
 462          * we have to fallback to software sync to prevent locking
 463          * order issues
 464          */
 465         if (!mutex_trylock(&chan->mutex)) {
 466                 ret = nouveau_fence_wait(fence, true, false);
 467                 goto out_unref;
 468         }
 469
 470         /* Make wchan wait until it gets signalled */
 471         ret = semaphore_acquire(wchan, sema);
 472         if (ret)
 473                 goto out_unlock;
 474
 475         /* Signal the semaphore from chan */
 476         ret = semaphore_release(chan, sema);
 477
 478 out_unlock:
 479         mutex_unlock(&chan->mutex);
 480 out_unref:
 481         kref_put(&sema->ref, semaphore_free);
 482 out:
 483         if (chan)
 484                 nouveau_channel_put_unlocked(&chan);
 485         return ret;
 486 }
 487
 488 int
 489 __nouveau_fence_flush(void *sync_obj, void *sync_arg)
 490 {
 491         return 0;
 492 }
 493
 494 int
 495 nouveau_fence_channel_init(struct nouveau_channel *chan)
 496 {
 497         struct drm_device *dev = chan->dev;
 498         struct drm_nouveau_private *dev_priv = dev->dev_private;
 499         struct nouveau_gpuobj *obj = NULL;
 500         int ret;
 501
 502         if (dev_priv->card_type < NV_C0) {
 503                 /* Create an NV_SW object for various sync purposes */
 504                 ret = nouveau_gpuobj_gr_new(chan, NvSw, NV_SW);
 505                 if (ret)
 506                         return ret;
 507
 508                 ret = RING_SPACE(chan, 2);
 509                 if (ret)
 510                         return ret;
 511
 512                 BEGIN_RING(chan, NvSubSw, NV01_SUBCHAN_OBJECT, 1);
 513                 OUT_RING  (chan, NvSw);
 514                 FIRE_RING (chan);
 515         }
 516
 517         /* Setup area of memory shared between all channels for x-chan sync */
 518         if (USE_SEMA(dev) && dev_priv->chipset < 0x84) {
 519                 struct ttm_mem_reg *mem = &dev_priv->fence.bo->bo.mem;
 520
 521                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_FROM_MEMORY,
 522                                              mem->start << PAGE_SHIFT,
 523                                              mem->size, NV_MEM_ACCESS_RW,
 524                                              NV_MEM_TARGET_VRAM, &obj);
 525                 if (ret)
 526                         return ret;
 527
 528                 ret = nouveau_ramht_insert(chan, NvSema, obj);
 529                 nouveau_gpuobj_ref(NULL, &obj);
 530                 if (ret)
 531                         return ret;
 532         } else
 533         if (USE_SEMA(dev)) {
 534                 /* map fence bo into channel's vm */
 535                 ret = nouveau_bo_vma_add(dev_priv->fence.bo, chan->vm,
 536                                          &chan->fence.vma);
 537                 if (ret)
 538                         return ret;
 539         }
 540
 541         atomic_set(&chan->fence.last_sequence_irq, 0);
 542         return 0;
 543 }
 544
 545 void
 546 nouveau_fence_channel_fini(struct nouveau_channel *chan)
 547 {
 548         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
 549         struct nouveau_fence *tmp, *fence;
 550
 551         spin_lock(&chan->fence.lock);
 552         list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
 553                 fence->signalled = true;
 554                 list_del(&fence->entry);
 555
 556                 if (unlikely(fence->work))
 557                         fence->work(fence->priv, false);
 558
 559                 kref_put(&fence->refcount, nouveau_fence_del);
 560         }
 561         spin_unlock(&chan->fence.lock);
 562
 563         nouveau_bo_vma_del(dev_priv->fence.bo, &chan->fence.vma);
 564 }
 565
 566 int
 567 nouveau_fence_init(struct drm_device *dev)
 568 {
 569         struct drm_nouveau_private *dev_priv = dev->dev_private;
 570         int size = (dev_priv->chipset < 0x84) ? 4096 : 16384;
 571         int ret;
 572
 573         /* Create a shared VRAM heap for cross-channel sync. */
 574         if (USE_SEMA(dev)) {
 575                 ret = nouveau_bo_new(dev, size, 0, TTM_PL_FLAG_VRAM,
 576                                      0, 0, &dev_priv->fence.bo);
 577                 if (ret)
 578                         return ret;
 579
 580                 ret = nouveau_bo_pin(dev_priv->fence.bo, TTM_PL_FLAG_VRAM);
 581                 if (ret)
 582                         goto fail;
 583
 584                 ret = nouveau_bo_map(dev_priv->fence.bo);
 585                 if (ret)
 586                         goto fail;
 587
 588                 ret = drm_mm_init(&dev_priv->fence.heap, 0,
 589                                   dev_priv->fence.bo->bo.mem.size);
 590                 if (ret)
 591                         goto fail;
 592
 593                 spin_lock_init(&dev_priv->fence.lock);
 594         }
 595
 596         return 0;
 597 fail:
 598         nouveau_bo_unmap(dev_priv->fence.bo);
 599         nouveau_bo_ref(NULL, &dev_priv->fence.bo);
 600         return ret;
 601 }
 602
 603 void
 604 nouveau_fence_fini(struct drm_device *dev)
 605 {
 606         struct drm_nouveau_private *dev_priv = dev->dev_private;
 607
 608         if (USE_SEMA(dev)) {
 609                 drm_mm_takedown(&dev_priv->fence.heap);
 610                 nouveau_bo_unmap(dev_priv->fence.bo);
 611                 nouveau_bo_unpin(dev_priv->fence.bo);
 612                 nouveau_bo_ref(NULL, &dev_priv->fence.bo);
 613         }
 614 }