drivers/gpu/drm/radeon/radeon_fence.c

   1 /*
   2  * Copyright 2009 Jerome Glisse.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the
   7  * "Software"), to deal in the Software without restriction, including
   8  * without limitation the rights to use, copy, modify, merge, publish,
   9  * distribute, sub license, and/or sell copies of the Software, and to
  10  * permit persons to whom the Software is furnished to do so, subject to
  11  * the following conditions:
  12  *
  13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20  *
  21  * The above copyright notice and this permission notice (including the
  22  * next paragraph) shall be included in all copies or substantial portions
  23  * of the Software.
  24  *
  25  */
  26 /*
  27  * Authors:
  28  *    Jerome Glisse <glisse@freedesktop.org>
  29  *    Dave Airlie
  30  */
  31 #include <linux/seq_file.h>
  32 #include <linux/atomic.h>
  33 #include <linux/wait.h>
  34 #include <linux/list.h>
  35 #include <linux/kref.h>
  36 #include <linux/slab.h>
  37 #include "drmP.h"
  38 #include "drm.h"
  39 #include "radeon_reg.h"
  40 #include "radeon.h"
  41 #include "radeon_trace.h"
  42
  43 static void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring)
  44 {
  45         if (rdev->wb.enabled) {
  46                 *rdev->fence_drv[ring].cpu_addr = cpu_to_le32(seq);
  47         } else {
  48                 WREG32(rdev->fence_drv[ring].scratch_reg, seq);
  49         }
  50 }
  51
  52 static u32 radeon_fence_read(struct radeon_device *rdev, int ring)
  53 {
  54         u32 seq = 0;
  55
  56         if (rdev->wb.enabled) {
  57                 seq = le32_to_cpu(*rdev->fence_drv[ring].cpu_addr);
  58         } else {
  59                 seq = RREG32(rdev->fence_drv[ring].scratch_reg);
  60         }
  61         return seq;
  62 }
  63
  64 int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence)
  65 {
  66         unsigned long irq_flags;
  67
  68         write_lock_irqsave(&rdev->fence_lock, irq_flags);
  69         if (fence->emitted) {
  70                 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
  71                 return 0;
  72         }
  73         fence->seq = atomic_add_return(1, &rdev->fence_drv[fence->ring].seq);
  74         radeon_fence_ring_emit(rdev, fence->ring, fence);
  75         trace_radeon_fence_emit(rdev->ddev, fence->seq);
  76         fence->emitted = true;
  77         /* are we the first fence on a previusly idle ring? */
  78         if (list_empty(&rdev->fence_drv[fence->ring].emitted)) {
  79                 rdev->fence_drv[fence->ring].last_activity = jiffies;
  80         }
  81         list_move_tail(&fence->list, &rdev->fence_drv[fence->ring].emitted);
  82         write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
  83         return 0;
  84 }
  85
  86 static bool radeon_fence_poll_locked(struct radeon_device *rdev, int ring)
  87 {
  88         struct radeon_fence *fence;
  89         struct list_head *i, *n;
  90         uint32_t seq;
  91         bool wake = false;
  92
  93         seq = radeon_fence_read(rdev, ring);
  94         if (seq == rdev->fence_drv[ring].last_seq)
  95                 return false;
  96
  97         rdev->fence_drv[ring].last_seq = seq;
  98         rdev->fence_drv[ring].last_activity = jiffies;
  99
 100         n = NULL;
 101         list_for_each(i, &rdev->fence_drv[ring].emitted) {
 102                 fence = list_entry(i, struct radeon_fence, list);
 103                 if (fence->seq == seq) {
 104                         n = i;
 105                         break;
 106                 }
 107         }
 108         /* all fence previous to this one are considered as signaled */
 109         if (n) {
 110                 i = n;
 111                 do {
 112                         n = i->prev;
 113                         list_move_tail(i, &rdev->fence_drv[ring].signaled);
 114                         fence = list_entry(i, struct radeon_fence, list);
 115                         fence->signaled = true;
 116                         i = n;
 117                 } while (i != &rdev->fence_drv[ring].emitted);
 118                 wake = true;
 119         }
 120         return wake;
 121 }
 122
 123 static void radeon_fence_destroy(struct kref *kref)
 124 {
 125         unsigned long irq_flags;
 126         struct radeon_fence *fence;
 127
 128         fence = container_of(kref, struct radeon_fence, kref);
 129         write_lock_irqsave(&fence->rdev->fence_lock, irq_flags);
 130         list_del(&fence->list);
 131         fence->emitted = false;
 132         write_unlock_irqrestore(&fence->rdev->fence_lock, irq_flags);
 133         if (fence->semaphore)
 134                 radeon_semaphore_free(fence->rdev, fence->semaphore);
 135         kfree(fence);
 136 }
 137
 138 int radeon_fence_create(struct radeon_device *rdev,
 139                         struct radeon_fence **fence,
 140                         int ring)
 141 {
 142         *fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL);
 143         if ((*fence) == NULL) {
 144                 return -ENOMEM;
 145         }
 146         kref_init(&((*fence)->kref));
 147         (*fence)->rdev = rdev;
 148         (*fence)->emitted = false;
 149         (*fence)->signaled = false;
 150         (*fence)->seq = 0;
 151         (*fence)->ring = ring;
 152         (*fence)->semaphore = NULL;
 153         INIT_LIST_HEAD(&(*fence)->list);
 154         return 0;
 155 }
 156
 157 bool radeon_fence_signaled(struct radeon_fence *fence)
 158 {
 159         unsigned long irq_flags;
 160         bool signaled = false;
 161
 162         if (!fence)
 163                 return true;
 164
 165         write_lock_irqsave(&fence->rdev->fence_lock, irq_flags);
 166         signaled = fence->signaled;
 167         /* if we are shuting down report all fence as signaled */
 168         if (fence->rdev->shutdown) {
 169                 signaled = true;
 170         }
 171         if (!fence->emitted) {
 172                 WARN(1, "Querying an unemitted fence : %p !\n", fence);
 173                 signaled = true;
 174         }
 175         if (!signaled) {
 176                 radeon_fence_poll_locked(fence->rdev, fence->ring);
 177                 signaled = fence->signaled;
 178         }
 179         write_unlock_irqrestore(&fence->rdev->fence_lock, irq_flags);
 180         return signaled;
 181 }
 182
 183 int radeon_fence_wait(struct radeon_fence *fence, bool intr)
 184 {
 185         struct radeon_device *rdev;
 186         unsigned long irq_flags, timeout;
 187         u32 seq;
 188         int i, r;
 189         bool signaled;
 190
 191         if (fence == NULL) {
 192                 WARN(1, "Querying an invalid fence : %p !\n", fence);
 193                 return -EINVAL;
 194         }
 195
 196         rdev = fence->rdev;
 197         signaled = radeon_fence_signaled(fence);
 198         while (!signaled) {
 199                 read_lock_irqsave(&rdev->fence_lock, irq_flags);
 200                 timeout = jiffies - RADEON_FENCE_JIFFIES_TIMEOUT;
 201                 if (time_after(rdev->fence_drv[fence->ring].last_activity, timeout)) {
 202                         /* the normal case, timeout is somewhere before last_activity */
 203                         timeout = rdev->fence_drv[fence->ring].last_activity - timeout;
 204                 } else {
 205                         /* either jiffies wrapped around, or no fence was signaled in the last 500ms
 206                          * anyway we will just wait for the minimum amount and then check for a lockup */
 207                         timeout = 1;
 208                 }
 209                 /* save current sequence value used to check for GPU lockups */
 210                 seq = rdev->fence_drv[fence->ring].last_seq;
 211                 read_unlock_irqrestore(&rdev->fence_lock, irq_flags);
 212
 213                 trace_radeon_fence_wait_begin(rdev->ddev, seq);
 214                 radeon_irq_kms_sw_irq_get(rdev, fence->ring);
 215                 if (intr) {
 216                         r = wait_event_interruptible_timeout(
 217                                 rdev->fence_drv[fence->ring].queue,
 218                                 (signaled = radeon_fence_signaled(fence)), timeout);
 219                 } else {
 220                         r = wait_event_timeout(
 221                                 rdev->fence_drv[fence->ring].queue,
 222                                 (signaled = radeon_fence_signaled(fence)), timeout);
 223                 }
 224                 radeon_irq_kms_sw_irq_put(rdev, fence->ring);
 225                 if (unlikely(r < 0)) {
 226                         return r;
 227                 }
 228                 trace_radeon_fence_wait_end(rdev->ddev, seq);
 229
 230                 if (unlikely(!signaled)) {
 231                         /* we were interrupted for some reason and fence
 232                          * isn't signaled yet, resume waiting */
 233                         if (r) {
 234                                 continue;
 235                         }
 236
 237                         write_lock_irqsave(&rdev->fence_lock, irq_flags);
 238                         /* check if sequence value has changed since last_activity */
 239                         if (seq != rdev->fence_drv[fence->ring].last_seq) {
 240                                 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
 241                                 continue;
 242                         }
 243
 244                         /* change sequence value on all rings, so nobody else things there is a lockup */
 245                         for (i = 0; i < RADEON_NUM_RINGS; ++i)
 246                                 rdev->fence_drv[i].last_seq -= 0x10000;
 247
 248                         rdev->fence_drv[fence->ring].last_activity = jiffies;
 249                         write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
 250
 251                         if (radeon_ring_is_lockup(rdev, fence->ring, &rdev->ring[fence->ring])) {
 252
 253                                 /* good news we believe it's a lockup */
 254                                 printk(KERN_WARNING "GPU lockup (waiting for 0x%08X last fence id 0x%08X)\n",
 255                                      fence->seq, seq);
 256
 257                                 /* mark the ring as not ready any more */
 258                                 rdev->ring[fence->ring].ready = false;
 259                                 return -EDEADLK;
 260                         }
 261                 }
 262         }
 263         return 0;
 264 }
 265
 266 int radeon_fence_wait_next(struct radeon_device *rdev, int ring)
 267 {
 268         unsigned long irq_flags;
 269         struct radeon_fence *fence;
 270         int r;
 271
 272         write_lock_irqsave(&rdev->fence_lock, irq_flags);
 273         if (!rdev->ring[ring].ready) {
 274                 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
 275                 return -EBUSY;
 276         }
 277         if (list_empty(&rdev->fence_drv[ring].emitted)) {
 278                 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
 279                 return -ENOENT;
 280         }
 281         fence = list_entry(rdev->fence_drv[ring].emitted.next,
 282                            struct radeon_fence, list);
 283         radeon_fence_ref(fence);
 284         write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
 285         r = radeon_fence_wait(fence, false);
 286         radeon_fence_unref(&fence);
 287         return r;
 288 }
 289
 290 int radeon_fence_wait_empty(struct radeon_device *rdev, int ring)
 291 {
 292         unsigned long irq_flags;
 293         struct radeon_fence *fence;
 294         int r;
 295
 296         write_lock_irqsave(&rdev->fence_lock, irq_flags);
 297         if (!rdev->ring[ring].ready) {
 298                 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
 299                 return -EBUSY;
 300         }
 301         if (list_empty(&rdev->fence_drv[ring].emitted)) {
 302                 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
 303                 return 0;
 304         }
 305         fence = list_entry(rdev->fence_drv[ring].emitted.prev,
 306                            struct radeon_fence, list);
 307         radeon_fence_ref(fence);
 308         write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
 309         r = radeon_fence_wait(fence, false);
 310         radeon_fence_unref(&fence);
 311         return r;
 312 }
 313
 314 struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence)
 315 {
 316         kref_get(&fence->kref);
 317         return fence;
 318 }
 319
 320 void radeon_fence_unref(struct radeon_fence **fence)
 321 {
 322         struct radeon_fence *tmp = *fence;
 323
 324         *fence = NULL;
 325         if (tmp) {
 326                 kref_put(&tmp->kref, radeon_fence_destroy);
 327         }
 328 }
 329
 330 void radeon_fence_process(struct radeon_device *rdev, int ring)
 331 {
 332         unsigned long irq_flags;
 333         bool wake;
 334
 335         write_lock_irqsave(&rdev->fence_lock, irq_flags);
 336         wake = radeon_fence_poll_locked(rdev, ring);
 337         write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
 338         if (wake) {
 339                 wake_up_all(&rdev->fence_drv[ring].queue);
 340         }
 341 }
 342
 343 int radeon_fence_count_emitted(struct radeon_device *rdev, int ring)
 344 {
 345         unsigned long irq_flags;
 346         int not_processed = 0;
 347
 348         read_lock_irqsave(&rdev->fence_lock, irq_flags);
 349         if (!rdev->fence_drv[ring].initialized) {
 350                 read_unlock_irqrestore(&rdev->fence_lock, irq_flags);
 351                 return 0;
 352         }
 353
 354         if (!list_empty(&rdev->fence_drv[ring].emitted)) {
 355                 struct list_head *ptr;
 356                 list_for_each(ptr, &rdev->fence_drv[ring].emitted) {
 357                         /* count up to 3, that's enought info */
 358                         if (++not_processed >= 3)
 359                                 break;
 360                 }
 361         }
 362         read_unlock_irqrestore(&rdev->fence_lock, irq_flags);
 363         return not_processed;
 364 }
 365
 366 int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring)
 367 {
 368         unsigned long irq_flags;
 369         uint64_t index;
 370         int r;
 371
 372         write_lock_irqsave(&rdev->fence_lock, irq_flags);
 373         radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);
 374         if (rdev->wb.use_event) {
 375                 rdev->fence_drv[ring].scratch_reg = 0;
 376                 index = R600_WB_EVENT_OFFSET + ring * 4;
 377         } else {
 378                 r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg);
 379                 if (r) {
 380                         dev_err(rdev->dev, "fence failed to get scratch register\n");
 381                         write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
 382                         return r;
 383                 }
 384                 index = RADEON_WB_SCRATCH_OFFSET +
 385                         rdev->fence_drv[ring].scratch_reg -
 386                         rdev->scratch.reg_base;
 387         }
 388         rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4];
 389         rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index;
 390         radeon_fence_write(rdev, atomic_read(&rdev->fence_drv[ring].seq), ring);
 391         rdev->fence_drv[ring].initialized = true;
 392         DRM_INFO("fence driver on ring %d use gpu addr 0x%08Lx and cpu addr 0x%p\n",
 393                  ring, rdev->fence_drv[ring].gpu_addr, rdev->fence_drv[ring].cpu_addr);
 394         write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
 395         return 0;
 396 }
 397
 398 static void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring)
 399 {
 400         rdev->fence_drv[ring].scratch_reg = -1;
 401         rdev->fence_drv[ring].cpu_addr = NULL;
 402         rdev->fence_drv[ring].gpu_addr = 0;
 403         atomic_set(&rdev->fence_drv[ring].seq, 0);
 404         INIT_LIST_HEAD(&rdev->fence_drv[ring].emitted);
 405         INIT_LIST_HEAD(&rdev->fence_drv[ring].signaled);
 406         init_waitqueue_head(&rdev->fence_drv[ring].queue);
 407         rdev->fence_drv[ring].initialized = false;
 408 }
 409
 410 int radeon_fence_driver_init(struct radeon_device *rdev)
 411 {
 412         unsigned long irq_flags;
 413         int ring;
 414
 415         write_lock_irqsave(&rdev->fence_lock, irq_flags);
 416         for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
 417                 radeon_fence_driver_init_ring(rdev, ring);
 418         }
 419         write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
 420         if (radeon_debugfs_fence_init(rdev)) {
 421                 dev_err(rdev->dev, "fence debugfs file creation failed\n");
 422         }
 423         return 0;
 424 }
 425
 426 void radeon_fence_driver_fini(struct radeon_device *rdev)
 427 {
 428         unsigned long irq_flags;
 429         int ring;
 430
 431         for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
 432                 if (!rdev->fence_drv[ring].initialized)
 433                         continue;
 434                 radeon_fence_wait_empty(rdev, ring);
 435                 wake_up_all(&rdev->fence_drv[ring].queue);
 436                 write_lock_irqsave(&rdev->fence_lock, irq_flags);
 437                 radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);
 438                 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
 439                 rdev->fence_drv[ring].initialized = false;
 440         }
 441 }
 442
 443
 444 /*
 445  * Fence debugfs
 446  */
 447 #if defined(CONFIG_DEBUG_FS)
 448 static int radeon_debugfs_fence_info(struct seq_file *m, void *data)
 449 {
 450         struct drm_info_node *node = (struct drm_info_node *)m->private;
 451         struct drm_device *dev = node->minor->dev;
 452         struct radeon_device *rdev = dev->dev_private;
 453         struct radeon_fence *fence;
 454         int i;
 455
 456         for (i = 0; i < RADEON_NUM_RINGS; ++i) {
 457                 if (!rdev->fence_drv[i].initialized)
 458                         continue;
 459
 460                 seq_printf(m, "--- ring %d ---\n", i);
 461                 seq_printf(m, "Last signaled fence 0x%08X\n",
 462                            radeon_fence_read(rdev, i));
 463                 if (!list_empty(&rdev->fence_drv[i].emitted)) {
 464                         fence = list_entry(rdev->fence_drv[i].emitted.prev,
 465                                            struct radeon_fence, list);
 466                         seq_printf(m, "Last emitted fence %p with 0x%08X\n",
 467                                    fence,  fence->seq);
 468                 }
 469         }
 470         return 0;
 471 }
 472
 473 static struct drm_info_list radeon_debugfs_fence_list[] = {
 474         {"radeon_fence_info", &radeon_debugfs_fence_info, 0, NULL},
 475 };
 476 #endif
 477
 478 int radeon_debugfs_fence_init(struct radeon_device *rdev)
 479 {
 480 #if defined(CONFIG_DEBUG_FS)
 481         return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 1);
 482 #else
 483         return 0;
 484 #endif
 485 }