drivers/gpu/host1x/job.c

   1 /*
   2  * Tegra host1x Job
   3  *
   4  * Copyright (c) 2010-2013, NVIDIA Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  */
  18
  19 #include <linux/dma-mapping.h>
  20 #include <linux/err.h>
  21 #include <linux/kref.h>
  22 #include <linux/module.h>
  23 #include <linux/scatterlist.h>
  24 #include <linux/slab.h>
  25 #include <linux/vmalloc.h>
  26 #include <trace/events/host1x.h>
  27
  28 #include "channel.h"
  29 #include "dev.h"
  30 #include "host1x_bo.h"
  31 #include "job.h"
  32 #include "syncpt.h"
  33
  34 struct host1x_job *host1x_job_alloc(struct host1x_channel *ch,
  35                                     u32 num_cmdbufs, u32 num_relocs,
  36                                     u32 num_waitchks)
  37 {
  38         struct host1x_job *job = NULL;
  39         unsigned int num_unpins = num_cmdbufs + num_relocs;
  40         u64 total;
  41         void *mem;
  42
  43         /* Check that we're not going to overflow */
  44         total = sizeof(struct host1x_job) +
  45                 num_relocs * sizeof(struct host1x_reloc) +
  46                 num_unpins * sizeof(struct host1x_job_unpin_data) +
  47                 num_waitchks * sizeof(struct host1x_waitchk) +
  48                 num_cmdbufs * sizeof(struct host1x_job_gather) +
  49                 num_unpins * sizeof(dma_addr_t) +
  50                 num_unpins * sizeof(u32 *);
  51         if (total > ULONG_MAX)
  52                 return NULL;
  53
  54         mem = job = kzalloc(total, GFP_KERNEL);
  55         if (!job)
  56                 return NULL;
  57
  58         kref_init(&job->ref);
  59         job->channel = ch;
  60
  61         /* Redistribute memory to the structs  */
  62         mem += sizeof(struct host1x_job);
  63         job->relocarray = num_relocs ? mem : NULL;
  64         mem += num_relocs * sizeof(struct host1x_reloc);
  65         job->unpins = num_unpins ? mem : NULL;
  66         mem += num_unpins * sizeof(struct host1x_job_unpin_data);
  67         job->waitchk = num_waitchks ? mem : NULL;
  68         mem += num_waitchks * sizeof(struct host1x_waitchk);
  69         job->gathers = num_cmdbufs ? mem : NULL;
  70         mem += num_cmdbufs * sizeof(struct host1x_job_gather);
  71         job->addr_phys = num_unpins ? mem : NULL;
  72
  73         job->reloc_addr_phys = job->addr_phys;
  74         job->gather_addr_phys = &job->addr_phys[num_relocs];
  75
  76         return job;
  77 }
  78
  79 struct host1x_job *host1x_job_get(struct host1x_job *job)
  80 {
  81         kref_get(&job->ref);
  82         return job;
  83 }
  84
  85 static void job_free(struct kref *ref)
  86 {
  87         struct host1x_job *job = container_of(ref, struct host1x_job, ref);
  88
  89         kfree(job);
  90 }
  91
  92 void host1x_job_put(struct host1x_job *job)
  93 {
  94         kref_put(&job->ref, job_free);
  95 }
  96
  97 void host1x_job_add_gather(struct host1x_job *job, struct host1x_bo *bo,
  98                            u32 words, u32 offset)
  99 {
 100         struct host1x_job_gather *cur_gather = &job->gathers[job->num_gathers];
 101
 102         cur_gather->words = words;
 103         cur_gather->bo = bo;
 104         cur_gather->offset = offset;
 105         job->num_gathers++;
 106 }
 107
 108 /*
 109  * NULL an already satisfied WAIT_SYNCPT host method, by patching its
 110  * args in the command stream. The method data is changed to reference
 111  * a reserved (never given out or incr) HOST1X_SYNCPT_RESERVED syncpt
 112  * with a matching threshold value of 0, so is guaranteed to be popped
 113  * by the host HW.
 114  */
 115 static void host1x_syncpt_patch_offset(struct host1x_syncpt *sp,
 116                                        struct host1x_bo *h, u32 offset)
 117 {
 118         void *patch_addr = NULL;
 119
 120         /* patch the wait */
 121         patch_addr = host1x_bo_kmap(h, offset >> PAGE_SHIFT);
 122         if (patch_addr) {
 123                 host1x_syncpt_patch_wait(sp,
 124                                          patch_addr + (offset & ~PAGE_MASK));
 125                 host1x_bo_kunmap(h, offset >> PAGE_SHIFT, patch_addr);
 126         } else
 127                 pr_err("Could not map cmdbuf for wait check\n");
 128 }
 129
 130 /*
 131  * Check driver supplied waitchk structs for syncpt thresholds
 132  * that have already been satisfied and NULL the comparison (to
 133  * avoid a wrap condition in the HW).
 134  */
 135 static int do_waitchks(struct host1x_job *job, struct host1x *host,
 136                        struct host1x_bo *patch)
 137 {
 138         int i;
 139
 140         /* compare syncpt vs wait threshold */
 141         for (i = 0; i < job->num_waitchk; i++) {
 142                 struct host1x_waitchk *wait = &job->waitchk[i];
 143                 struct host1x_syncpt *sp =
 144                         host1x_syncpt_get(host, wait->syncpt_id);
 145
 146                 /* validate syncpt id */
 147                 if (wait->syncpt_id > host1x_syncpt_nb_pts(host))
 148                         continue;
 149
 150                 /* skip all other gathers */
 151                 if (patch != wait->bo)
 152                         continue;
 153
 154                 trace_host1x_syncpt_wait_check(wait->bo, wait->offset,
 155                                                wait->syncpt_id, wait->thresh,
 156                                                host1x_syncpt_read_min(sp));
 157
 158                 if (host1x_syncpt_is_expired(sp, wait->thresh)) {
 159                         dev_dbg(host->dev,
 160                                 "drop WAIT id %d (%s) thresh 0x%x, min 0x%x\n",
 161                                 wait->syncpt_id, sp->name, wait->thresh,
 162                                 host1x_syncpt_read_min(sp));
 163
 164                         host1x_syncpt_patch_offset(sp, patch, wait->offset);
 165                 }
 166
 167                 wait->bo = NULL;
 168         }
 169
 170         return 0;
 171 }
 172
 173 static unsigned int pin_job(struct host1x_job *job)
 174 {
 175         unsigned int i;
 176
 177         job->num_unpins = 0;
 178
 179         for (i = 0; i < job->num_relocs; i++) {
 180                 struct host1x_reloc *reloc = &job->relocarray[i];
 181                 struct sg_table *sgt;
 182                 dma_addr_t phys_addr;
 183
 184                 reloc->target = host1x_bo_get(reloc->target);
 185                 if (!reloc->target)
 186                         goto unpin;
 187
 188                 phys_addr = host1x_bo_pin(reloc->target, &sgt);
 189                 if (!phys_addr)
 190                         goto unpin;
 191
 192                 job->addr_phys[job->num_unpins] = phys_addr;
 193                 job->unpins[job->num_unpins].bo = reloc->target;
 194                 job->unpins[job->num_unpins].sgt = sgt;
 195                 job->num_unpins++;
 196         }
 197
 198         for (i = 0; i < job->num_gathers; i++) {
 199                 struct host1x_job_gather *g = &job->gathers[i];
 200                 struct sg_table *sgt;
 201                 dma_addr_t phys_addr;
 202
 203                 g->bo = host1x_bo_get(g->bo);
 204                 if (!g->bo)
 205                         goto unpin;
 206
 207                 phys_addr = host1x_bo_pin(g->bo, &sgt);
 208                 if (!phys_addr)
 209                         goto unpin;
 210
 211                 job->addr_phys[job->num_unpins] = phys_addr;
 212                 job->unpins[job->num_unpins].bo = g->bo;
 213                 job->unpins[job->num_unpins].sgt = sgt;
 214                 job->num_unpins++;
 215         }
 216
 217         return job->num_unpins;
 218
 219 unpin:
 220         host1x_job_unpin(job);
 221         return 0;
 222 }
 223
 224 static unsigned int do_relocs(struct host1x_job *job, struct host1x_bo *cmdbuf)
 225 {
 226         int i = 0;
 227         u32 last_page = ~0;
 228         void *cmdbuf_page_addr = NULL;
 229
 230         /* pin & patch the relocs for one gather */
 231         while (i < job->num_relocs) {
 232                 struct host1x_reloc *reloc = &job->relocarray[i];
 233                 u32 reloc_addr = (job->reloc_addr_phys[i] +
 234                         reloc->target_offset) >> reloc->shift;
 235                 u32 *target;
 236
 237                 /* skip all other gathers */
 238                 if (!(reloc->cmdbuf && cmdbuf == reloc->cmdbuf)) {
 239                         i++;
 240                         continue;
 241                 }
 242
 243                 if (last_page != reloc->cmdbuf_offset >> PAGE_SHIFT) {
 244                         if (cmdbuf_page_addr)
 245                                 host1x_bo_kunmap(cmdbuf, last_page,
 246                                                  cmdbuf_page_addr);
 247
 248                         cmdbuf_page_addr = host1x_bo_kmap(cmdbuf,
 249                                         reloc->cmdbuf_offset >> PAGE_SHIFT);
 250                         last_page = reloc->cmdbuf_offset >> PAGE_SHIFT;
 251
 252                         if (unlikely(!cmdbuf_page_addr)) {
 253                                 pr_err("Could not map cmdbuf for relocation\n");
 254                                 return -ENOMEM;
 255                         }
 256                 }
 257
 258                 target = cmdbuf_page_addr + (reloc->cmdbuf_offset & ~PAGE_MASK);
 259                 *target = reloc_addr;
 260
 261                 /* mark this gather as handled */
 262                 reloc->cmdbuf = 0;
 263         }
 264
 265         if (cmdbuf_page_addr)
 266                 host1x_bo_kunmap(cmdbuf, last_page, cmdbuf_page_addr);
 267
 268         return 0;
 269 }
 270
 271 static int check_reloc(struct host1x_reloc *reloc, struct host1x_bo *cmdbuf,
 272                        unsigned int offset)
 273 {
 274         offset *= sizeof(u32);
 275
 276         if (reloc->cmdbuf != cmdbuf || reloc->cmdbuf_offset != offset)
 277                 return -EINVAL;
 278
 279         return 0;
 280 }
 281
 282 struct host1x_firewall {
 283         struct host1x_job *job;
 284         struct device *dev;
 285
 286         unsigned int num_relocs;
 287         struct host1x_reloc *reloc;
 288
 289         struct host1x_bo *cmdbuf_id;
 290         unsigned int offset;
 291
 292         u32 words;
 293         u32 class;
 294         u32 reg;
 295         u32 mask;
 296         u32 count;
 297 };
 298
 299 static int check_mask(struct host1x_firewall *fw)
 300 {
 301         u32 mask = fw->mask;
 302         u32 reg = fw->reg;
 303
 304         while (mask) {
 305                 if (fw->words == 0)
 306                         return -EINVAL;
 307
 308                 if (mask & 1) {
 309                         if (fw->job->is_addr_reg(fw->dev, fw->class, reg)) {
 310                                 bool bad_reloc = check_reloc(fw->reloc,
 311                                                              fw->cmdbuf_id,
 312                                                              fw->offset);
 313                                 if (!fw->num_relocs || bad_reloc)
 314                                         return -EINVAL;
 315                                 fw->reloc++;
 316                                 fw->num_relocs--;
 317                         }
 318                         fw->words--;
 319                         fw->offset++;
 320                 }
 321                 mask >>= 1;
 322                 reg++;
 323         }
 324
 325         return 0;
 326 }
 327
 328 static int check_incr(struct host1x_firewall *fw)
 329 {
 330         u32 count = fw->count;
 331         u32 reg = fw->reg;
 332
 333         while (fw) {
 334                 if (fw->words == 0)
 335                         return -EINVAL;
 336
 337                 if (fw->job->is_addr_reg(fw->dev, fw->class, reg)) {
 338                         bool bad_reloc = check_reloc(fw->reloc, fw->cmdbuf_id,
 339                                                      fw->offset);
 340                         if (!fw->num_relocs || bad_reloc)
 341                                 return -EINVAL;
 342                         fw->reloc++;
 343                         fw->num_relocs--;
 344                 }
 345                 reg++;
 346                 fw->words--;
 347                 fw->offset++;
 348                 count--;
 349         }
 350
 351         return 0;
 352 }
 353
 354 static int check_nonincr(struct host1x_firewall *fw)
 355 {
 356         int is_addr_reg = fw->job->is_addr_reg(fw->dev, fw->class, fw->reg);
 357         u32 count = fw->count;
 358
 359         while (count) {
 360                 if (fw->words == 0)
 361                         return -EINVAL;
 362
 363                 if (is_addr_reg) {
 364                         bool bad_reloc = check_reloc(fw->reloc, fw->cmdbuf_id,
 365                                                      fw->offset);
 366                         if (!fw->num_relocs || bad_reloc)
 367                                 return -EINVAL;
 368                         fw->reloc++;
 369                         fw->num_relocs--;
 370                 }
 371                 fw->words--;
 372                 fw->offset++;
 373                 count--;
 374         }
 375
 376         return 0;
 377 }
 378
 379 static int validate(struct host1x_job *job, struct device *dev,
 380                     struct host1x_job_gather *g)
 381 {
 382         u32 *cmdbuf_base;
 383         int err = 0;
 384         struct host1x_firewall fw;
 385
 386         fw.job = job;
 387         fw.dev = dev;
 388         fw.reloc = job->relocarray;
 389         fw.num_relocs = job->num_relocs;
 390         fw.cmdbuf_id = g->bo;
 391
 392         fw.offset = 0;
 393         fw.class = 0;
 394
 395         if (!job->is_addr_reg)
 396                 return 0;
 397
 398         cmdbuf_base = host1x_bo_mmap(g->bo);
 399         if (!cmdbuf_base)
 400                 return -ENOMEM;
 401
 402         fw.words = g->words;
 403         while (fw.words && !err) {
 404                 u32 word = cmdbuf_base[fw.offset];
 405                 u32 opcode = (word & 0xf0000000) >> 28;
 406
 407                 fw.mask = 0;
 408                 fw.reg = 0;
 409                 fw.count = 0;
 410                 fw.words--;
 411                 fw.offset++;
 412
 413                 switch (opcode) {
 414                 case 0:
 415                         fw.class = word >> 6 & 0x3ff;
 416                         fw.mask = word & 0x3f;
 417                         fw.reg = word >> 16 & 0xfff;
 418                         err = check_mask(&fw);
 419                         if (err)
 420                                 goto out;
 421                         break;
 422                 case 1:
 423                         fw.reg = word >> 16 & 0xfff;
 424                         fw.count = word & 0xffff;
 425                         err = check_incr(&fw);
 426                         if (err)
 427                                 goto out;
 428                         break;
 429
 430                 case 2:
 431                         fw.reg = word >> 16 & 0xfff;
 432                         fw.count = word & 0xffff;
 433                         err = check_nonincr(&fw);
 434                         if (err)
 435                                 goto out;
 436                         break;
 437
 438                 case 3:
 439                         fw.mask = word & 0xffff;
 440                         fw.reg = word >> 16 & 0xfff;
 441                         err = check_mask(&fw);
 442                         if (err)
 443                                 goto out;
 444                         break;
 445                 case 4:
 446                 case 5:
 447                 case 14:
 448                         break;
 449                 default:
 450                         err = -EINVAL;
 451                         break;
 452                 }
 453         }
 454
 455         /* No relocs should remain at this point */
 456         if (fw.num_relocs)
 457                 err = -EINVAL;
 458
 459 out:
 460         host1x_bo_munmap(g->bo, cmdbuf_base);
 461
 462         return err;
 463 }
 464
 465 static inline int copy_gathers(struct host1x_job *job, struct device *dev)
 466 {
 467         size_t size = 0;
 468         size_t offset = 0;
 469         int i;
 470
 471         for (i = 0; i < job->num_gathers; i++) {
 472                 struct host1x_job_gather *g = &job->gathers[i];
 473                 size += g->words * sizeof(u32);
 474         }
 475
 476         job->gather_copy_mapped = dma_alloc_writecombine(dev, size,
 477                                                          &job->gather_copy,
 478                                                          GFP_KERNEL);
 479         if (!job->gather_copy_mapped) {
 480                 int err = PTR_ERR(job->gather_copy_mapped);
 481                 job->gather_copy_mapped = NULL;
 482                 return err;
 483         }
 484
 485         job->gather_copy_size = size;
 486
 487         for (i = 0; i < job->num_gathers; i++) {
 488                 struct host1x_job_gather *g = &job->gathers[i];
 489                 void *gather;
 490
 491                 gather = host1x_bo_mmap(g->bo);
 492                 memcpy(job->gather_copy_mapped + offset, gather + g->offset,
 493                        g->words * sizeof(u32));
 494                 host1x_bo_munmap(g->bo, gather);
 495
 496                 g->base = job->gather_copy;
 497                 g->offset = offset;
 498                 g->bo = NULL;
 499
 500                 offset += g->words * sizeof(u32);
 501         }
 502
 503         return 0;
 504 }
 505
 506 int host1x_job_pin(struct host1x_job *job, struct device *dev)
 507 {
 508         int err;
 509         unsigned int i, j;
 510         struct host1x *host = dev_get_drvdata(dev->parent);
 511         DECLARE_BITMAP(waitchk_mask, host1x_syncpt_nb_pts(host));
 512
 513         bitmap_zero(waitchk_mask, host1x_syncpt_nb_pts(host));
 514         for (i = 0; i < job->num_waitchk; i++) {
 515                 u32 syncpt_id = job->waitchk[i].syncpt_id;
 516                 if (syncpt_id < host1x_syncpt_nb_pts(host))
 517                         set_bit(syncpt_id, waitchk_mask);
 518         }
 519
 520         /* get current syncpt values for waitchk */
 521         for_each_set_bit(i, waitchk_mask, host1x_syncpt_nb_pts(host))
 522                 host1x_syncpt_load(host->syncpt + i);
 523
 524         /* pin memory */
 525         err = pin_job(job);
 526         if (!err)
 527                 goto out;
 528
 529         /* patch gathers */
 530         for (i = 0; i < job->num_gathers; i++) {
 531                 struct host1x_job_gather *g = &job->gathers[i];
 532
 533                 /* process each gather mem only once */
 534                 if (g->handled)
 535                         continue;
 536
 537                 g->base = job->gather_addr_phys[i];
 538
 539                 for (j = 0; j < job->num_gathers; j++)
 540                         if (job->gathers[j].bo == g->bo)
 541                                 job->gathers[j].handled = true;
 542
 543                 err = 0;
 544
 545                 if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL))
 546                         err = validate(job, dev, g);
 547
 548                 if (err)
 549                         dev_err(dev, "Job invalid (err=%d)\n", err);
 550
 551                 if (!err)
 552                         err = do_relocs(job, g->bo);
 553
 554                 if (!err)
 555                         err = do_waitchks(job, host, g->bo);
 556
 557                 if (err)
 558                         break;
 559         }
 560
 561         if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) && !err) {
 562                 err = copy_gathers(job, dev);
 563                 if (err) {
 564                         host1x_job_unpin(job);
 565                         return err;
 566                 }
 567         }
 568
 569 out:
 570         wmb();
 571
 572         return err;
 573 }
 574
 575 void host1x_job_unpin(struct host1x_job *job)
 576 {
 577         unsigned int i;
 578
 579         for (i = 0; i < job->num_unpins; i++) {
 580                 struct host1x_job_unpin_data *unpin = &job->unpins[i];
 581                 host1x_bo_unpin(unpin->bo, unpin->sgt);
 582                 host1x_bo_put(unpin->bo);
 583         }
 584         job->num_unpins = 0;
 585
 586         if (job->gather_copy_size)
 587                 dma_free_writecombine(job->channel->dev, job->gather_copy_size,
 588                                       job->gather_copy_mapped,
 589                                       job->gather_copy);
 590 }
 591
 592 /*
 593  * Debug routine used to dump job entries
 594  */
 595 void host1x_job_dump(struct device *dev, struct host1x_job *job)
 596 {
 597         dev_dbg(dev, "    SYNCPT_ID   %d\n", job->syncpt_id);
 598         dev_dbg(dev, "    SYNCPT_VAL  %d\n", job->syncpt_end);
 599         dev_dbg(dev, "    FIRST_GET   0x%x\n", job->first_get);
 600         dev_dbg(dev, "    TIMEOUT     %d\n", job->timeout);
 601         dev_dbg(dev, "    NUM_SLOTS   %d\n", job->num_slots);
 602         dev_dbg(dev, "    NUM_HANDLES %d\n", job->num_unpins);
 603 }