Pileus Git - ~andy/linux/blob - drivers/gpu/host1x/job.c

   1 /*
   2  * Tegra host1x Job
   3  *
   4  * Copyright (c) 2010-2013, NVIDIA Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  */
  18
  19 #include <linux/dma-mapping.h>
  20 #include <linux/err.h>
  21 #include <linux/host1x.h>
  22 #include <linux/kref.h>
  23 #include <linux/module.h>
  24 #include <linux/scatterlist.h>
  25 #include <linux/slab.h>
  26 #include <linux/vmalloc.h>
  27 #include <trace/events/host1x.h>
  28
  29 #include "channel.h"
  30 #include "dev.h"
  31 #include "job.h"
  32 #include "syncpt.h"
  33
  34 struct host1x_job *host1x_job_alloc(struct host1x_channel *ch,
  35                                     u32 num_cmdbufs, u32 num_relocs,
  36                                     u32 num_waitchks)
  37 {
  38         struct host1x_job *job = NULL;
  39         unsigned int num_unpins = num_cmdbufs + num_relocs;
  40         u64 total;
  41         void *mem;
  42
  43         /* Check that we're not going to overflow */
  44         total = sizeof(struct host1x_job) +
  45                 (u64)num_relocs * sizeof(struct host1x_reloc) +
  46                 (u64)num_unpins * sizeof(struct host1x_job_unpin_data) +
  47                 (u64)num_waitchks * sizeof(struct host1x_waitchk) +
  48                 (u64)num_cmdbufs * sizeof(struct host1x_job_gather) +
  49                 (u64)num_unpins * sizeof(dma_addr_t) +
  50                 (u64)num_unpins * sizeof(u32 *);
  51         if (total > ULONG_MAX)
  52                 return NULL;
  53
  54         mem = job = kzalloc(total, GFP_KERNEL);
  55         if (!job)
  56                 return NULL;
  57
  58         kref_init(&job->ref);
  59         job->channel = ch;
  60
  61         /* Redistribute memory to the structs  */
  62         mem += sizeof(struct host1x_job);
  63         job->relocarray = num_relocs ? mem : NULL;
  64         mem += num_relocs * sizeof(struct host1x_reloc);
  65         job->unpins = num_unpins ? mem : NULL;
  66         mem += num_unpins * sizeof(struct host1x_job_unpin_data);
  67         job->waitchk = num_waitchks ? mem : NULL;
  68         mem += num_waitchks * sizeof(struct host1x_waitchk);
  69         job->gathers = num_cmdbufs ? mem : NULL;
  70         mem += num_cmdbufs * sizeof(struct host1x_job_gather);
  71         job->addr_phys = num_unpins ? mem : NULL;
  72
  73         job->reloc_addr_phys = job->addr_phys;
  74         job->gather_addr_phys = &job->addr_phys[num_relocs];
  75
  76         return job;
  77 }
  78
  79 struct host1x_job *host1x_job_get(struct host1x_job *job)
  80 {
  81         kref_get(&job->ref);
  82         return job;
  83 }
  84
  85 static void job_free(struct kref *ref)
  86 {
  87         struct host1x_job *job = container_of(ref, struct host1x_job, ref);
  88
  89         kfree(job);
  90 }
  91
  92 void host1x_job_put(struct host1x_job *job)
  93 {
  94         kref_put(&job->ref, job_free);
  95 }
  96
  97 void host1x_job_add_gather(struct host1x_job *job, struct host1x_bo *bo,
  98                            u32 words, u32 offset)
  99 {
 100         struct host1x_job_gather *cur_gather = &job->gathers[job->num_gathers];
 101
 102         cur_gather->words = words;
 103         cur_gather->bo = bo;
 104         cur_gather->offset = offset;
 105         job->num_gathers++;
 106 }
 107
 108 /*
 109  * NULL an already satisfied WAIT_SYNCPT host method, by patching its
 110  * args in the command stream. The method data is changed to reference
 111  * a reserved (never given out or incr) HOST1X_SYNCPT_RESERVED syncpt
 112  * with a matching threshold value of 0, so is guaranteed to be popped
 113  * by the host HW.
 114  */
 115 static void host1x_syncpt_patch_offset(struct host1x_syncpt *sp,
 116                                        struct host1x_bo *h, u32 offset)
 117 {
 118         void *patch_addr = NULL;
 119
 120         /* patch the wait */
 121         patch_addr = host1x_bo_kmap(h, offset >> PAGE_SHIFT);
 122         if (patch_addr) {
 123                 host1x_syncpt_patch_wait(sp,
 124                                          patch_addr + (offset & ~PAGE_MASK));
 125                 host1x_bo_kunmap(h, offset >> PAGE_SHIFT, patch_addr);
 126         } else
 127                 pr_err("Could not map cmdbuf for wait check\n");
 128 }
 129
 130 /*
 131  * Check driver supplied waitchk structs for syncpt thresholds
 132  * that have already been satisfied and NULL the comparison (to
 133  * avoid a wrap condition in the HW).
 134  */
 135 static int do_waitchks(struct host1x_job *job, struct host1x *host,
 136                        struct host1x_bo *patch)
 137 {
 138         int i;
 139
 140         /* compare syncpt vs wait threshold */
 141         for (i = 0; i < job->num_waitchk; i++) {
 142                 struct host1x_waitchk *wait = &job->waitchk[i];
 143                 struct host1x_syncpt *sp =
 144                         host1x_syncpt_get(host, wait->syncpt_id);
 145
 146                 /* validate syncpt id */
 147                 if (wait->syncpt_id > host1x_syncpt_nb_pts(host))
 148                         continue;
 149
 150                 /* skip all other gathers */
 151                 if (patch != wait->bo)
 152                         continue;
 153
 154                 trace_host1x_syncpt_wait_check(wait->bo, wait->offset,
 155                                                wait->syncpt_id, wait->thresh,
 156                                                host1x_syncpt_read_min(sp));
 157
 158                 if (host1x_syncpt_is_expired(sp, wait->thresh)) {
 159                         dev_dbg(host->dev,
 160                                 "drop WAIT id %d (%s) thresh 0x%x, min 0x%x\n",
 161                                 wait->syncpt_id, sp->name, wait->thresh,
 162                                 host1x_syncpt_read_min(sp));
 163
 164                         host1x_syncpt_patch_offset(sp, patch, wait->offset);
 165                 }
 166
 167                 wait->bo = NULL;
 168         }
 169
 170         return 0;
 171 }
 172
 173 static unsigned int pin_job(struct host1x_job *job)
 174 {
 175         unsigned int i;
 176
 177         job->num_unpins = 0;
 178
 179         for (i = 0; i < job->num_relocs; i++) {
 180                 struct host1x_reloc *reloc = &job->relocarray[i];
 181                 struct sg_table *sgt;
 182                 dma_addr_t phys_addr;
 183
 184                 reloc->target = host1x_bo_get(reloc->target);
 185                 if (!reloc->target)
 186                         goto unpin;
 187
 188                 phys_addr = host1x_bo_pin(reloc->target, &sgt);
 189                 if (!phys_addr)
 190                         goto unpin;
 191
 192                 job->addr_phys[job->num_unpins] = phys_addr;
 193                 job->unpins[job->num_unpins].bo = reloc->target;
 194                 job->unpins[job->num_unpins].sgt = sgt;
 195                 job->num_unpins++;
 196         }
 197
 198         for (i = 0; i < job->num_gathers; i++) {
 199                 struct host1x_job_gather *g = &job->gathers[i];
 200                 struct sg_table *sgt;
 201                 dma_addr_t phys_addr;
 202
 203                 g->bo = host1x_bo_get(g->bo);
 204                 if (!g->bo)
 205                         goto unpin;
 206
 207                 phys_addr = host1x_bo_pin(g->bo, &sgt);
 208                 if (!phys_addr)
 209                         goto unpin;
 210
 211                 job->addr_phys[job->num_unpins] = phys_addr;
 212                 job->unpins[job->num_unpins].bo = g->bo;
 213                 job->unpins[job->num_unpins].sgt = sgt;
 214                 job->num_unpins++;
 215         }
 216
 217         return job->num_unpins;
 218
 219 unpin:
 220         host1x_job_unpin(job);
 221         return 0;
 222 }
 223
 224 static unsigned int do_relocs(struct host1x_job *job, struct host1x_bo *cmdbuf)
 225 {
 226         int i = 0;
 227         u32 last_page = ~0;
 228         void *cmdbuf_page_addr = NULL;
 229
 230         /* pin & patch the relocs for one gather */
 231         for (i = 0; i < job->num_relocs; i++) {
 232                 struct host1x_reloc *reloc = &job->relocarray[i];
 233                 u32 reloc_addr = (job->reloc_addr_phys[i] +
 234                         reloc->target_offset) >> reloc->shift;
 235                 u32 *target;
 236
 237                 /* skip all other gathers */
 238                 if (cmdbuf != reloc->cmdbuf)
 239                         continue;
 240
 241                 if (last_page != reloc->cmdbuf_offset >> PAGE_SHIFT) {
 242                         if (cmdbuf_page_addr)
 243                                 host1x_bo_kunmap(cmdbuf, last_page,
 244                                                  cmdbuf_page_addr);
 245
 246                         cmdbuf_page_addr = host1x_bo_kmap(cmdbuf,
 247                                         reloc->cmdbuf_offset >> PAGE_SHIFT);
 248                         last_page = reloc->cmdbuf_offset >> PAGE_SHIFT;
 249
 250                         if (unlikely(!cmdbuf_page_addr)) {
 251                                 pr_err("Could not map cmdbuf for relocation\n");
 252                                 return -ENOMEM;
 253                         }
 254                 }
 255
 256                 target = cmdbuf_page_addr + (reloc->cmdbuf_offset & ~PAGE_MASK);
 257                 *target = reloc_addr;
 258         }
 259
 260         if (cmdbuf_page_addr)
 261                 host1x_bo_kunmap(cmdbuf, last_page, cmdbuf_page_addr);
 262
 263         return 0;
 264 }
 265
 266 static bool check_reloc(struct host1x_reloc *reloc, struct host1x_bo *cmdbuf,
 267                         unsigned int offset)
 268 {
 269         offset *= sizeof(u32);
 270
 271         if (reloc->cmdbuf != cmdbuf || reloc->cmdbuf_offset != offset)
 272                 return false;
 273
 274         return true;
 275 }
 276
 277 struct host1x_firewall {
 278         struct host1x_job *job;
 279         struct device *dev;
 280
 281         unsigned int num_relocs;
 282         struct host1x_reloc *reloc;
 283
 284         struct host1x_bo *cmdbuf;
 285         unsigned int offset;
 286
 287         u32 words;
 288         u32 class;
 289         u32 reg;
 290         u32 mask;
 291         u32 count;
 292 };
 293
 294 static int check_register(struct host1x_firewall *fw, unsigned long offset)
 295 {
 296         if (fw->job->is_addr_reg(fw->dev, fw->class, offset)) {
 297                 if (!fw->num_relocs)
 298                         return -EINVAL;
 299
 300                 if (!check_reloc(fw->reloc, fw->cmdbuf, fw->offset))
 301                         return -EINVAL;
 302
 303                 fw->num_relocs--;
 304                 fw->reloc++;
 305         }
 306
 307         return 0;
 308 }
 309
 310 static int check_mask(struct host1x_firewall *fw)
 311 {
 312         u32 mask = fw->mask;
 313         u32 reg = fw->reg;
 314         int ret;
 315
 316         while (mask) {
 317                 if (fw->words == 0)
 318                         return -EINVAL;
 319
 320                 if (mask & 1) {
 321                         ret = check_register(fw, reg);
 322                         if (ret < 0)
 323                                 return ret;
 324
 325                         fw->words--;
 326                         fw->offset++;
 327                 }
 328                 mask >>= 1;
 329                 reg++;
 330         }
 331
 332         return 0;
 333 }
 334
 335 static int check_incr(struct host1x_firewall *fw)
 336 {
 337         u32 count = fw->count;
 338         u32 reg = fw->reg;
 339         int ret;
 340
 341         while (count) {
 342                 if (fw->words == 0)
 343                         return -EINVAL;
 344
 345                 ret = check_register(fw, reg);
 346                 if (ret < 0)
 347                         return ret;
 348
 349                 reg++;
 350                 fw->words--;
 351                 fw->offset++;
 352                 count--;
 353         }
 354
 355         return 0;
 356 }
 357
 358 static int check_nonincr(struct host1x_firewall *fw)
 359 {
 360         u32 count = fw->count;
 361         int ret;
 362
 363         while (count) {
 364                 if (fw->words == 0)
 365                         return -EINVAL;
 366
 367                 ret = check_register(fw, fw->reg);
 368                 if (ret < 0)
 369                         return ret;
 370
 371                 fw->words--;
 372                 fw->offset++;
 373                 count--;
 374         }
 375
 376         return 0;
 377 }
 378
 379 static int validate(struct host1x_firewall *fw, struct host1x_job_gather *g)
 380 {
 381         u32 *cmdbuf_base = (u32 *)fw->job->gather_copy_mapped +
 382                 (g->offset / sizeof(u32));
 383         int err = 0;
 384
 385         if (!fw->job->is_addr_reg)
 386                 return 0;
 387
 388         fw->words = g->words;
 389         fw->cmdbuf = g->bo;
 390         fw->offset = 0;
 391
 392         while (fw->words && !err) {
 393                 u32 word = cmdbuf_base[fw->offset];
 394                 u32 opcode = (word & 0xf0000000) >> 28;
 395
 396                 fw->mask = 0;
 397                 fw->reg = 0;
 398                 fw->count = 0;
 399                 fw->words--;
 400                 fw->offset++;
 401
 402                 switch (opcode) {
 403                 case 0:
 404                         fw->class = word >> 6 & 0x3ff;
 405                         fw->mask = word & 0x3f;
 406                         fw->reg = word >> 16 & 0xfff;
 407                         err = check_mask(fw);
 408                         if (err)
 409                                 goto out;
 410                         break;
 411                 case 1:
 412                         fw->reg = word >> 16 & 0xfff;
 413                         fw->count = word & 0xffff;
 414                         err = check_incr(fw);
 415                         if (err)
 416                                 goto out;
 417                         break;
 418
 419                 case 2:
 420                         fw->reg = word >> 16 & 0xfff;
 421                         fw->count = word & 0xffff;
 422                         err = check_nonincr(fw);
 423                         if (err)
 424                                 goto out;
 425                         break;
 426
 427                 case 3:
 428                         fw->mask = word & 0xffff;
 429                         fw->reg = word >> 16 & 0xfff;
 430                         err = check_mask(fw);
 431                         if (err)
 432                                 goto out;
 433                         break;
 434                 case 4:
 435                 case 5:
 436                 case 14:
 437                         break;
 438                 default:
 439                         err = -EINVAL;
 440                         break;
 441                 }
 442         }
 443
 444 out:
 445         return err;
 446 }
 447
 448 static inline int copy_gathers(struct host1x_job *job, struct device *dev)
 449 {
 450         struct host1x_firewall fw;
 451         size_t size = 0;
 452         size_t offset = 0;
 453         int i;
 454
 455         fw.job = job;
 456         fw.dev = dev;
 457         fw.reloc = job->relocarray;
 458         fw.num_relocs = job->num_relocs;
 459         fw.class = 0;
 460
 461         for (i = 0; i < job->num_gathers; i++) {
 462                 struct host1x_job_gather *g = &job->gathers[i];
 463                 size += g->words * sizeof(u32);
 464         }
 465
 466         job->gather_copy_mapped = dma_alloc_writecombine(dev, size,
 467                                                          &job->gather_copy,
 468                                                          GFP_KERNEL);
 469         if (!job->gather_copy_mapped) {
 470                 job->gather_copy_mapped = NULL;
 471                 return -ENOMEM;
 472         }
 473
 474         job->gather_copy_size = size;
 475
 476         for (i = 0; i < job->num_gathers; i++) {
 477                 struct host1x_job_gather *g = &job->gathers[i];
 478                 void *gather;
 479
 480                 /* Copy the gather */
 481                 gather = host1x_bo_mmap(g->bo);
 482                 memcpy(job->gather_copy_mapped + offset, gather + g->offset,
 483                        g->words * sizeof(u32));
 484                 host1x_bo_munmap(g->bo, gather);
 485
 486                 /* Store the location in the buffer */
 487                 g->base = job->gather_copy;
 488                 g->offset = offset;
 489
 490                 /* Validate the job */
 491                 if (validate(&fw, g))
 492                         return -EINVAL;
 493
 494                 offset += g->words * sizeof(u32);
 495         }
 496
 497         /* No relocs should remain at this point */
 498         if (fw.num_relocs)
 499                 return -EINVAL;
 500
 501         return 0;
 502 }
 503
 504 int host1x_job_pin(struct host1x_job *job, struct device *dev)
 505 {
 506         int err;
 507         unsigned int i, j;
 508         struct host1x *host = dev_get_drvdata(dev->parent);
 509         DECLARE_BITMAP(waitchk_mask, host1x_syncpt_nb_pts(host));
 510
 511         bitmap_zero(waitchk_mask, host1x_syncpt_nb_pts(host));
 512         for (i = 0; i < job->num_waitchk; i++) {
 513                 u32 syncpt_id = job->waitchk[i].syncpt_id;
 514                 if (syncpt_id < host1x_syncpt_nb_pts(host))
 515                         set_bit(syncpt_id, waitchk_mask);
 516         }
 517
 518         /* get current syncpt values for waitchk */
 519         for_each_set_bit(i, waitchk_mask, host1x_syncpt_nb_pts(host))
 520                 host1x_syncpt_load(host->syncpt + i);
 521
 522         /* pin memory */
 523         err = pin_job(job);
 524         if (!err)
 525                 goto out;
 526
 527         /* patch gathers */
 528         for (i = 0; i < job->num_gathers; i++) {
 529                 struct host1x_job_gather *g = &job->gathers[i];
 530
 531                 /* process each gather mem only once */
 532                 if (g->handled)
 533                         continue;
 534
 535                 g->base = job->gather_addr_phys[i];
 536
 537                 for (j = 0; j < job->num_gathers; j++)
 538                         if (job->gathers[j].bo == g->bo)
 539                                 job->gathers[j].handled = true;
 540
 541                 err = do_relocs(job, g->bo);
 542                 if (err)
 543                         break;
 544
 545                 err = do_waitchks(job, host, g->bo);
 546                 if (err)
 547                         break;
 548         }
 549
 550         if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) && !err) {
 551                 err = copy_gathers(job, dev);
 552                 if (err) {
 553                         host1x_job_unpin(job);
 554                         return err;
 555                 }
 556         }
 557
 558 out:
 559         wmb();
 560
 561         return err;
 562 }
 563
 564 void host1x_job_unpin(struct host1x_job *job)
 565 {
 566         unsigned int i;
 567
 568         for (i = 0; i < job->num_unpins; i++) {
 569                 struct host1x_job_unpin_data *unpin = &job->unpins[i];
 570                 host1x_bo_unpin(unpin->bo, unpin->sgt);
 571                 host1x_bo_put(unpin->bo);
 572         }
 573         job->num_unpins = 0;
 574
 575         if (job->gather_copy_size)
 576                 dma_free_writecombine(job->channel->dev, job->gather_copy_size,
 577                                       job->gather_copy_mapped,
 578                                       job->gather_copy);
 579 }
 580
 581 /*
 582  * Debug routine used to dump job entries
 583  */
 584 void host1x_job_dump(struct device *dev, struct host1x_job *job)
 585 {
 586         dev_dbg(dev, "    SYNCPT_ID   %d\n", job->syncpt_id);
 587         dev_dbg(dev, "    SYNCPT_VAL  %d\n", job->syncpt_end);
 588         dev_dbg(dev, "    FIRST_GET   0x%x\n", job->first_get);
 589         dev_dbg(dev, "    TIMEOUT     %d\n", job->timeout);
 590         dev_dbg(dev, "    NUM_SLOTS   %d\n", job->num_slots);
 591         dev_dbg(dev, "    NUM_HANDLES %d\n", job->num_unpins);
 592 }