drivers/gpu/drm/nouveau/nouveau_fence.c

   1 /*
   2  * Copyright (C) 2007 Ben Skeggs.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining
   6  * a copy of this software and associated documentation files (the
   7  * "Software"), to deal in the Software without restriction, including
   8  * without limitation the rights to use, copy, modify, merge, publish,
   9  * distribute, sublicense, and/or sell copies of the Software, and to
  10  * permit persons to whom the Software is furnished to do so, subject to
  11  * the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the
  14  * next paragraph) shall be included in all copies or substantial
  15  * portions of the Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  20  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  21  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  22  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  24  *
  25  */
  26
  27 #include "drmP.h"
  28 #include "drm.h"
  29
  30 #include "nouveau_drv.h"
  31 #include "nouveau_ramht.h"
  32 #include "nouveau_dma.h"
  33
  34 #define USE_REFCNT(dev) (nouveau_private(dev)->chipset >= 0x10)
  35 #define USE_SEMA(dev) (nouveau_private(dev)->chipset >= 0x17)
  36
  37 struct nouveau_fence {
  38         struct nouveau_channel *channel;
  39         struct kref refcount;
  40         struct list_head entry;
  41
  42         uint32_t sequence;
  43         bool signalled;
  44
  45         void (*work)(void *priv, bool signalled);
  46         void *priv;
  47 };
  48
  49 struct nouveau_semaphore {
  50         struct kref ref;
  51         struct drm_device *dev;
  52         struct drm_mm_node *mem;
  53 };
  54
  55 static inline struct nouveau_fence *
  56 nouveau_fence(void *sync_obj)
  57 {
  58         return (struct nouveau_fence *)sync_obj;
  59 }
  60
  61 static void
  62 nouveau_fence_del(struct kref *ref)
  63 {
  64         struct nouveau_fence *fence =
  65                 container_of(ref, struct nouveau_fence, refcount);
  66
  67         nouveau_channel_ref(NULL, &fence->channel);
  68         kfree(fence);
  69 }
  70
  71 void
  72 nouveau_fence_update(struct nouveau_channel *chan)
  73 {
  74         struct drm_device *dev = chan->dev;
  75         struct nouveau_fence *tmp, *fence;
  76         uint32_t sequence;
  77
  78         spin_lock(&chan->fence.lock);
  79
  80         /* Fetch the last sequence if the channel is still up and running */
  81         if (likely(!list_empty(&chan->fence.pending))) {
  82                 if (USE_REFCNT(dev))
  83                         sequence = nvchan_rd32(chan, 0x48);
  84                 else
  85                         sequence = atomic_read(&chan->fence.last_sequence_irq);
  86
  87                 if (chan->fence.sequence_ack == sequence)
  88                         goto out;
  89                 chan->fence.sequence_ack = sequence;
  90         }
  91
  92         list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
  93                 sequence = fence->sequence;
  94                 fence->signalled = true;
  95                 list_del(&fence->entry);
  96
  97                 if (unlikely(fence->work))
  98                         fence->work(fence->priv, true);
  99
 100                 kref_put(&fence->refcount, nouveau_fence_del);
 101
 102                 if (sequence == chan->fence.sequence_ack)
 103                         break;
 104         }
 105 out:
 106         spin_unlock(&chan->fence.lock);
 107 }
 108
 109 int
 110 nouveau_fence_new(struct nouveau_channel *chan, struct nouveau_fence **pfence,
 111                   bool emit)
 112 {
 113         struct nouveau_fence *fence;
 114         int ret = 0;
 115
 116         fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 117         if (!fence)
 118                 return -ENOMEM;
 119         kref_init(&fence->refcount);
 120         nouveau_channel_ref(chan, &fence->channel);
 121
 122         if (emit)
 123                 ret = nouveau_fence_emit(fence);
 124
 125         if (ret)
 126                 nouveau_fence_unref(&fence);
 127         *pfence = fence;
 128         return ret;
 129 }
 130
 131 struct nouveau_channel *
 132 nouveau_fence_channel(struct nouveau_fence *fence)
 133 {
 134         return fence ? nouveau_channel_get_unlocked(fence->channel) : NULL;
 135 }
 136
 137 int
 138 nouveau_fence_emit(struct nouveau_fence *fence)
 139 {
 140         struct nouveau_channel *chan = fence->channel;
 141         struct drm_device *dev = chan->dev;
 142         int ret;
 143
 144         ret = RING_SPACE(chan, 2);
 145         if (ret)
 146                 return ret;
 147
 148         if (unlikely(chan->fence.sequence == chan->fence.sequence_ack - 1)) {
 149                 nouveau_fence_update(chan);
 150
 151                 BUG_ON(chan->fence.sequence ==
 152                        chan->fence.sequence_ack - 1);
 153         }
 154
 155         fence->sequence = ++chan->fence.sequence;
 156
 157         kref_get(&fence->refcount);
 158         spin_lock(&chan->fence.lock);
 159         list_add_tail(&fence->entry, &chan->fence.pending);
 160         spin_unlock(&chan->fence.lock);
 161
 162         BEGIN_RING(chan, NvSubSw, USE_REFCNT(dev) ? 0x0050 : 0x0150, 1);
 163         OUT_RING(chan, fence->sequence);
 164         FIRE_RING(chan);
 165
 166         return 0;
 167 }
 168
 169 void
 170 nouveau_fence_work(struct nouveau_fence *fence,
 171                    void (*work)(void *priv, bool signalled),
 172                    void *priv)
 173 {
 174         BUG_ON(fence->work);
 175
 176         spin_lock(&fence->channel->fence.lock);
 177
 178         if (fence->signalled) {
 179                 work(priv, true);
 180         } else {
 181                 fence->work = work;
 182                 fence->priv = priv;
 183         }
 184
 185         spin_unlock(&fence->channel->fence.lock);
 186 }
 187
 188 void
 189 __nouveau_fence_unref(void **sync_obj)
 190 {
 191         struct nouveau_fence *fence = nouveau_fence(*sync_obj);
 192
 193         if (fence)
 194                 kref_put(&fence->refcount, nouveau_fence_del);
 195         *sync_obj = NULL;
 196 }
 197
 198 void *
 199 __nouveau_fence_ref(void *sync_obj)
 200 {
 201         struct nouveau_fence *fence = nouveau_fence(sync_obj);
 202
 203         kref_get(&fence->refcount);
 204         return sync_obj;
 205 }
 206
 207 bool
 208 __nouveau_fence_signalled(void *sync_obj, void *sync_arg)
 209 {
 210         struct nouveau_fence *fence = nouveau_fence(sync_obj);
 211         struct nouveau_channel *chan = fence->channel;
 212
 213         if (fence->signalled)
 214                 return true;
 215
 216         nouveau_fence_update(chan);
 217         return fence->signalled;
 218 }
 219
 220 int
 221 __nouveau_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr)
 222 {
 223         unsigned long timeout = jiffies + (3 * DRM_HZ);
 224         unsigned long sleep_time = jiffies + 1;
 225         int ret = 0;
 226
 227         while (1) {
 228                 if (__nouveau_fence_signalled(sync_obj, sync_arg))
 229                         break;
 230
 231                 if (time_after_eq(jiffies, timeout)) {
 232                         ret = -EBUSY;
 233                         break;
 234                 }
 235
 236                 __set_current_state(intr ? TASK_INTERRUPTIBLE
 237                         : TASK_UNINTERRUPTIBLE);
 238                 if (lazy && time_after_eq(jiffies, sleep_time))
 239                         schedule_timeout(1);
 240
 241                 if (intr && signal_pending(current)) {
 242                         ret = -ERESTARTSYS;
 243                         break;
 244                 }
 245         }
 246
 247         __set_current_state(TASK_RUNNING);
 248
 249         return ret;
 250 }
 251
 252 static struct nouveau_semaphore *
 253 alloc_semaphore(struct drm_device *dev)
 254 {
 255         struct drm_nouveau_private *dev_priv = dev->dev_private;
 256         struct nouveau_semaphore *sema;
 257         int ret;
 258
 259         if (!USE_SEMA(dev))
 260                 return NULL;
 261
 262         sema = kmalloc(sizeof(*sema), GFP_KERNEL);
 263         if (!sema)
 264                 goto fail;
 265
 266         ret = drm_mm_pre_get(&dev_priv->fence.heap);
 267         if (ret)
 268                 goto fail;
 269
 270         spin_lock(&dev_priv->fence.lock);
 271         sema->mem = drm_mm_search_free(&dev_priv->fence.heap, 4, 0, 0);
 272         if (sema->mem)
 273                 sema->mem = drm_mm_get_block_atomic(sema->mem, 4, 0);
 274         spin_unlock(&dev_priv->fence.lock);
 275
 276         if (!sema->mem)
 277                 goto fail;
 278
 279         kref_init(&sema->ref);
 280         sema->dev = dev;
 281         nouveau_bo_wr32(dev_priv->fence.bo, sema->mem->start / 4, 0);
 282
 283         return sema;
 284 fail:
 285         kfree(sema);
 286         return NULL;
 287 }
 288
 289 static void
 290 free_semaphore(struct kref *ref)
 291 {
 292         struct nouveau_semaphore *sema =
 293                 container_of(ref, struct nouveau_semaphore, ref);
 294         struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
 295
 296         spin_lock(&dev_priv->fence.lock);
 297         drm_mm_put_block(sema->mem);
 298         spin_unlock(&dev_priv->fence.lock);
 299
 300         kfree(sema);
 301 }
 302
 303 static void
 304 semaphore_work(void *priv, bool signalled)
 305 {
 306         struct nouveau_semaphore *sema = priv;
 307         struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
 308
 309         if (unlikely(!signalled))
 310                 nouveau_bo_wr32(dev_priv->fence.bo, sema->mem->start / 4, 1);
 311
 312         kref_put(&sema->ref, free_semaphore);
 313 }
 314
 315 static int
 316 emit_semaphore(struct nouveau_channel *chan, int method,
 317                struct nouveau_semaphore *sema)
 318 {
 319         struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
 320         struct nouveau_fence *fence;
 321         bool smart = (dev_priv->card_type >= NV_50);
 322         int ret;
 323
 324         ret = RING_SPACE(chan, smart ? 8 : 4);
 325         if (ret)
 326                 return ret;
 327
 328         if (smart) {
 329                 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1);
 330                 OUT_RING(chan, NvSema);
 331         }
 332         BEGIN_RING(chan, NvSubSw, NV_SW_SEMAPHORE_OFFSET, 1);
 333         OUT_RING(chan, sema->mem->start);
 334
 335         if (smart && method == NV_SW_SEMAPHORE_ACQUIRE) {
 336                 /*
 337                  * NV50 tries to be too smart and context-switch
 338                  * between semaphores instead of doing a "first come,
 339                  * first served" strategy like previous cards
 340                  * do.
 341                  *
 342                  * That's bad because the ACQUIRE latency can get as
 343                  * large as the PFIFO context time slice in the
 344                  * typical DRI2 case where you have several
 345                  * outstanding semaphores at the same moment.
 346                  *
 347                  * If we're going to ACQUIRE, force the card to
 348                  * context switch before, just in case the matching
 349                  * RELEASE is already scheduled to be executed in
 350                  * another channel.
 351                  */
 352                 BEGIN_RING(chan, NvSubSw, NV_SW_YIELD, 1);
 353                 OUT_RING(chan, 0);
 354         }
 355
 356         BEGIN_RING(chan, NvSubSw, method, 1);
 357         OUT_RING(chan, 1);
 358
 359         if (smart && method == NV_SW_SEMAPHORE_RELEASE) {
 360                 /*
 361                  * Force the card to context switch, there may be
 362                  * another channel waiting for the semaphore we just
 363                  * released.
 364                  */
 365                 BEGIN_RING(chan, NvSubSw, NV_SW_YIELD, 1);
 366                 OUT_RING(chan, 0);
 367         }
 368
 369         /* Delay semaphore destruction until its work is done */
 370         ret = nouveau_fence_new(chan, &fence, true);
 371         if (ret)
 372                 return ret;
 373
 374         kref_get(&sema->ref);
 375         nouveau_fence_work(fence, semaphore_work, sema);
 376         nouveau_fence_unref(&fence);
 377
 378         return 0;
 379 }
 380
 381 int
 382 nouveau_fence_sync(struct nouveau_fence *fence,
 383                    struct nouveau_channel *wchan)
 384 {
 385         struct nouveau_channel *chan = nouveau_fence_channel(fence);
 386         struct drm_device *dev = wchan->dev;
 387         struct nouveau_semaphore *sema;
 388         int ret = 0;
 389
 390         if (likely(!chan || chan == wchan ||
 391                    nouveau_fence_signalled(fence)))
 392                 goto out;
 393
 394         sema = alloc_semaphore(dev);
 395         if (!sema) {
 396                 /* Early card or broken userspace, fall back to
 397                  * software sync. */
 398                 ret = nouveau_fence_wait(fence, true, false);
 399                 goto out;
 400         }
 401
 402         /* try to take chan's mutex, if we can't take it right away
 403          * we have to fallback to software sync to prevent locking
 404          * order issues
 405          */
 406         if (!mutex_trylock(&chan->mutex)) {
 407                 ret = nouveau_fence_wait(fence, true, false);
 408                 goto out_unref;
 409         }
 410
 411         /* Make wchan wait until it gets signalled */
 412         ret = emit_semaphore(wchan, NV_SW_SEMAPHORE_ACQUIRE, sema);
 413         if (ret)
 414                 goto out_unlock;
 415
 416         /* Signal the semaphore from chan */
 417         ret = emit_semaphore(chan, NV_SW_SEMAPHORE_RELEASE, sema);
 418
 419 out_unlock:
 420         mutex_unlock(&chan->mutex);
 421 out_unref:
 422         kref_put(&sema->ref, free_semaphore);
 423 out:
 424         if (chan)
 425                 nouveau_channel_put_unlocked(&chan);
 426         return ret;
 427 }
 428
 429 int
 430 __nouveau_fence_flush(void *sync_obj, void *sync_arg)
 431 {
 432         return 0;
 433 }
 434
 435 int
 436 nouveau_fence_channel_init(struct nouveau_channel *chan)
 437 {
 438         struct drm_device *dev = chan->dev;
 439         struct drm_nouveau_private *dev_priv = dev->dev_private;
 440         struct nouveau_gpuobj *obj = NULL;
 441         int ret;
 442
 443         /* Create an NV_SW object for various sync purposes */
 444         ret = nouveau_gpuobj_gr_new(chan, NvSw, NV_SW);
 445         if (ret)
 446                 return ret;
 447
 448         ret = RING_SPACE(chan, 2);
 449         if (ret)
 450                 return ret;
 451         BEGIN_RING(chan, NvSubSw, 0, 1);
 452         OUT_RING(chan, NvSw);
 453
 454         /* Create a DMA object for the shared cross-channel sync area. */
 455         if (USE_SEMA(dev)) {
 456                 struct ttm_mem_reg *mem = &dev_priv->fence.bo->bo.mem;
 457
 458                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 459                                              mem->start << PAGE_SHIFT,
 460                                              mem->size, NV_MEM_ACCESS_RW,
 461                                              NV_MEM_TARGET_VRAM, &obj);
 462                 if (ret)
 463                         return ret;
 464
 465                 ret = nouveau_ramht_insert(chan, NvSema, obj);
 466                 nouveau_gpuobj_ref(NULL, &obj);
 467                 if (ret)
 468                         return ret;
 469
 470                 ret = RING_SPACE(chan, 2);
 471                 if (ret)
 472                         return ret;
 473                 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1);
 474                 OUT_RING(chan, NvSema);
 475         }
 476
 477         FIRE_RING(chan);
 478
 479         INIT_LIST_HEAD(&chan->fence.pending);
 480         spin_lock_init(&chan->fence.lock);
 481         atomic_set(&chan->fence.last_sequence_irq, 0);
 482
 483         return 0;
 484 }
 485
 486 void
 487 nouveau_fence_channel_fini(struct nouveau_channel *chan)
 488 {
 489         struct nouveau_fence *tmp, *fence;
 490
 491         spin_lock(&chan->fence.lock);
 492
 493         list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
 494                 fence->signalled = true;
 495                 list_del(&fence->entry);
 496
 497                 if (unlikely(fence->work))
 498                         fence->work(fence->priv, false);
 499
 500                 kref_put(&fence->refcount, nouveau_fence_del);
 501         }
 502
 503         spin_unlock(&chan->fence.lock);
 504 }
 505
 506 int
 507 nouveau_fence_init(struct drm_device *dev)
 508 {
 509         struct drm_nouveau_private *dev_priv = dev->dev_private;
 510         int ret;
 511
 512         /* Create a shared VRAM heap for cross-channel sync. */
 513         if (USE_SEMA(dev)) {
 514                 ret = nouveau_bo_new(dev, NULL, 4096, 0, TTM_PL_FLAG_VRAM,
 515                                      0, 0, false, true, &dev_priv->fence.bo);
 516                 if (ret)
 517                         return ret;
 518
 519                 ret = nouveau_bo_pin(dev_priv->fence.bo, TTM_PL_FLAG_VRAM);
 520                 if (ret)
 521                         goto fail;
 522
 523                 ret = nouveau_bo_map(dev_priv->fence.bo);
 524                 if (ret)
 525                         goto fail;
 526
 527                 ret = drm_mm_init(&dev_priv->fence.heap, 0,
 528                                   dev_priv->fence.bo->bo.mem.size);
 529                 if (ret)
 530                         goto fail;
 531
 532                 spin_lock_init(&dev_priv->fence.lock);
 533         }
 534
 535         return 0;
 536 fail:
 537         nouveau_bo_unmap(dev_priv->fence.bo);
 538         nouveau_bo_ref(NULL, &dev_priv->fence.bo);
 539         return ret;
 540 }
 541
 542 void
 543 nouveau_fence_fini(struct drm_device *dev)
 544 {
 545         struct drm_nouveau_private *dev_priv = dev->dev_private;
 546
 547         if (USE_SEMA(dev)) {
 548                 drm_mm_takedown(&dev_priv->fence.heap);
 549                 nouveau_bo_unmap(dev_priv->fence.bo);
 550                 nouveau_bo_unpin(dev_priv->fence.bo);
 551                 nouveau_bo_ref(NULL, &dev_priv->fence.bo);
 552         }
 553 }