Pileus Git - ~andy/linux/blob - drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/shmem_fs.h>
  35 #include <linux/slab.h>
  36 #include <linux/swap.h>
  37 #include <linux/pci.h>
  38
  39 static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
  40 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  41 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  42 static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  43                                                     unsigned alignment,
  44                                                     bool map_and_fenceable);
  45 static int i915_gem_phys_pwrite(struct drm_device *dev,
  46                                 struct drm_i915_gem_object *obj,
  47                                 struct drm_i915_gem_pwrite *args,
  48                                 struct drm_file *file);
  49
  50 static void i915_gem_write_fence(struct drm_device *dev, int reg,
  51                                  struct drm_i915_gem_object *obj);
  52 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  53                                          struct drm_i915_fence_reg *fence,
  54                                          bool enable);
  55
  56 static int i915_gem_inactive_shrink(struct shrinker *shrinker,
  57                                     struct shrink_control *sc);
  58 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
  59
  60 static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
  61 {
  62         if (obj->tiling_mode)
  63                 i915_gem_release_mmap(obj);
  64
  65         /* As we do not have an associated fence register, we will force
  66          * a tiling change if we ever need to acquire one.
  67          */
  68         obj->fence_dirty = false;
  69         obj->fence_reg = I915_FENCE_REG_NONE;
  70 }
  71
  72 /* some bookkeeping */
  73 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  74                                   size_t size)
  75 {
  76         dev_priv->mm.object_count++;
  77         dev_priv->mm.object_memory += size;
  78 }
  79
  80 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  81                                      size_t size)
  82 {
  83         dev_priv->mm.object_count--;
  84         dev_priv->mm.object_memory -= size;
  85 }
  86
  87 static int
  88 i915_gem_wait_for_error(struct drm_device *dev)
  89 {
  90         struct drm_i915_private *dev_priv = dev->dev_private;
  91         struct completion *x = &dev_priv->error_completion;
  92         unsigned long flags;
  93         int ret;
  94
  95         if (!atomic_read(&dev_priv->mm.wedged))
  96                 return 0;
  97
  98         ret = wait_for_completion_interruptible(x);
  99         if (ret)
 100                 return ret;
 101
 102         if (atomic_read(&dev_priv->mm.wedged)) {
 103                 /* GPU is hung, bump the completion count to account for
 104                  * the token we just consumed so that we never hit zero and
 105                  * end up waiting upon a subsequent completion event that
 106                  * will never happen.
 107                  */
 108                 spin_lock_irqsave(&x->wait.lock, flags);
 109                 x->done++;
 110                 spin_unlock_irqrestore(&x->wait.lock, flags);
 111         }
 112         return 0;
 113 }
 114
 115 int i915_mutex_lock_interruptible(struct drm_device *dev)
 116 {
 117         int ret;
 118
 119         ret = i915_gem_wait_for_error(dev);
 120         if (ret)
 121                 return ret;
 122
 123         ret = mutex_lock_interruptible(&dev->struct_mutex);
 124         if (ret)
 125                 return ret;
 126
 127         WARN_ON(i915_verify_lists(dev));
 128         return 0;
 129 }
 130
 131 static inline bool
 132 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
 133 {
 134         return !obj->active;
 135 }
 136
 137 int
 138 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 139                     struct drm_file *file)
 140 {
 141         struct drm_i915_gem_init *args = data;
 142
 143         if (drm_core_check_feature(dev, DRIVER_MODESET))
 144                 return -ENODEV;
 145
 146         if (args->gtt_start >= args->gtt_end ||
 147             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
 148                 return -EINVAL;
 149
 150         /* GEM with user mode setting was never supported on ilk and later. */
 151         if (INTEL_INFO(dev)->gen >= 5)
 152                 return -ENODEV;
 153
 154         mutex_lock(&dev->struct_mutex);
 155         i915_gem_init_global_gtt(dev, args->gtt_start,
 156                                  args->gtt_end, args->gtt_end);
 157         mutex_unlock(&dev->struct_mutex);
 158
 159         return 0;
 160 }
 161
 162 int
 163 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 164                             struct drm_file *file)
 165 {
 166         struct drm_i915_private *dev_priv = dev->dev_private;
 167         struct drm_i915_gem_get_aperture *args = data;
 168         struct drm_i915_gem_object *obj;
 169         size_t pinned;
 170
 171         pinned = 0;
 172         mutex_lock(&dev->struct_mutex);
 173         list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list)
 174                 if (obj->pin_count)
 175                         pinned += obj->gtt_space->size;
 176         mutex_unlock(&dev->struct_mutex);
 177
 178         args->aper_size = dev_priv->mm.gtt_total;
 179         args->aper_available_size = args->aper_size - pinned;
 180
 181         return 0;
 182 }
 183
 184 static int
 185 i915_gem_create(struct drm_file *file,
 186                 struct drm_device *dev,
 187                 uint64_t size,
 188                 uint32_t *handle_p)
 189 {
 190         struct drm_i915_gem_object *obj;
 191         int ret;
 192         u32 handle;
 193
 194         size = roundup(size, PAGE_SIZE);
 195         if (size == 0)
 196                 return -EINVAL;
 197
 198         /* Allocate the new object */
 199         obj = i915_gem_alloc_object(dev, size);
 200         if (obj == NULL)
 201                 return -ENOMEM;
 202
 203         ret = drm_gem_handle_create(file, &obj->base, &handle);
 204         if (ret) {
 205                 drm_gem_object_release(&obj->base);
 206                 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
 207                 kfree(obj);
 208                 return ret;
 209         }
 210
 211         /* drop reference from allocate - handle holds it now */
 212         drm_gem_object_unreference(&obj->base);
 213         trace_i915_gem_object_create(obj);
 214
 215         *handle_p = handle;
 216         return 0;
 217 }
 218
 219 int
 220 i915_gem_dumb_create(struct drm_file *file,
 221                      struct drm_device *dev,
 222                      struct drm_mode_create_dumb *args)
 223 {
 224         /* have to work out size/pitch and return them */
 225         args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
 226         args->size = args->pitch * args->height;
 227         return i915_gem_create(file, dev,
 228                                args->size, &args->handle);
 229 }
 230
 231 int i915_gem_dumb_destroy(struct drm_file *file,
 232                           struct drm_device *dev,
 233                           uint32_t handle)
 234 {
 235         return drm_gem_handle_delete(file, handle);
 236 }
 237
 238 /**
 239  * Creates a new mm object and returns a handle to it.
 240  */
 241 int
 242 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 243                       struct drm_file *file)
 244 {
 245         struct drm_i915_gem_create *args = data;
 246
 247         return i915_gem_create(file, dev,
 248                                args->size, &args->handle);
 249 }
 250
 251 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
 252 {
 253         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
 254
 255         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 256                 obj->tiling_mode != I915_TILING_NONE;
 257 }
 258
 259 static inline int
 260 __copy_to_user_swizzled(char __user *cpu_vaddr,
 261                         const char *gpu_vaddr, int gpu_offset,
 262                         int length)
 263 {
 264         int ret, cpu_offset = 0;
 265
 266         while (length > 0) {
 267                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 268                 int this_length = min(cacheline_end - gpu_offset, length);
 269                 int swizzled_gpu_offset = gpu_offset ^ 64;
 270
 271                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
 272                                      gpu_vaddr + swizzled_gpu_offset,
 273                                      this_length);
 274                 if (ret)
 275                         return ret + length;
 276
 277                 cpu_offset += this_length;
 278                 gpu_offset += this_length;
 279                 length -= this_length;
 280         }
 281
 282         return 0;
 283 }
 284
 285 static inline int
 286 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
 287                           const char __user *cpu_vaddr,
 288                           int length)
 289 {
 290         int ret, cpu_offset = 0;
 291
 292         while (length > 0) {
 293                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 294                 int this_length = min(cacheline_end - gpu_offset, length);
 295                 int swizzled_gpu_offset = gpu_offset ^ 64;
 296
 297                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
 298                                        cpu_vaddr + cpu_offset,
 299                                        this_length);
 300                 if (ret)
 301                         return ret + length;
 302
 303                 cpu_offset += this_length;
 304                 gpu_offset += this_length;
 305                 length -= this_length;
 306         }
 307
 308         return 0;
 309 }
 310
 311 /* Per-page copy function for the shmem pread fastpath.
 312  * Flushes invalid cachelines before reading the target if
 313  * needs_clflush is set. */
 314 static int
 315 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
 316                  char __user *user_data,
 317                  bool page_do_bit17_swizzling, bool needs_clflush)
 318 {
 319         char *vaddr;
 320         int ret;
 321
 322         if (unlikely(page_do_bit17_swizzling))
 323                 return -EINVAL;
 324
 325         vaddr = kmap_atomic(page);
 326         if (needs_clflush)
 327                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 328                                        page_length);
 329         ret = __copy_to_user_inatomic(user_data,
 330                                       vaddr + shmem_page_offset,
 331                                       page_length);
 332         kunmap_atomic(vaddr);
 333
 334         return ret;
 335 }
 336
 337 static void
 338 shmem_clflush_swizzled_range(char *addr, unsigned long length,
 339                              bool swizzled)
 340 {
 341         if (unlikely(swizzled)) {
 342                 unsigned long start = (unsigned long) addr;
 343                 unsigned long end = (unsigned long) addr + length;
 344
 345                 /* For swizzling simply ensure that we always flush both
 346                  * channels. Lame, but simple and it works. Swizzled
 347                  * pwrite/pread is far from a hotpath - current userspace
 348                  * doesn't use it at all. */
 349                 start = round_down(start, 128);
 350                 end = round_up(end, 128);
 351
 352                 drm_clflush_virt_range((void *)start, end - start);
 353         } else {
 354                 drm_clflush_virt_range(addr, length);
 355         }
 356
 357 }
 358
 359 /* Only difference to the fast-path function is that this can handle bit17
 360  * and uses non-atomic copy and kmap functions. */
 361 static int
 362 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
 363                  char __user *user_data,
 364                  bool page_do_bit17_swizzling, bool needs_clflush)
 365 {
 366         char *vaddr;
 367         int ret;
 368
 369         vaddr = kmap(page);
 370         if (needs_clflush)
 371                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 372                                              page_length,
 373                                              page_do_bit17_swizzling);
 374
 375         if (page_do_bit17_swizzling)
 376                 ret = __copy_to_user_swizzled(user_data,
 377                                               vaddr, shmem_page_offset,
 378                                               page_length);
 379         else
 380                 ret = __copy_to_user(user_data,
 381                                      vaddr + shmem_page_offset,
 382                                      page_length);
 383         kunmap(page);
 384
 385         return ret;
 386 }
 387
 388 static int
 389 i915_gem_shmem_pread(struct drm_device *dev,
 390                      struct drm_i915_gem_object *obj,
 391                      struct drm_i915_gem_pread *args,
 392                      struct drm_file *file)
 393 {
 394         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
 395         char __user *user_data;
 396         ssize_t remain;
 397         loff_t offset;
 398         int shmem_page_offset, page_length, ret = 0;
 399         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 400         int hit_slowpath = 0;
 401         int prefaulted = 0;
 402         int needs_clflush = 0;
 403         int release_page;
 404
 405         user_data = (char __user *) (uintptr_t) args->data_ptr;
 406         remain = args->size;
 407
 408         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 409
 410         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
 411                 /* If we're not in the cpu read domain, set ourself into the gtt
 412                  * read domain and manually flush cachelines (if required). This
 413                  * optimizes for the case when the gpu will dirty the data
 414                  * anyway again before the next pread happens. */
 415                 if (obj->cache_level == I915_CACHE_NONE)
 416                         needs_clflush = 1;
 417                 ret = i915_gem_object_set_to_gtt_domain(obj, false);
 418                 if (ret)
 419                         return ret;
 420         }
 421
 422         offset = args->offset;
 423
 424         while (remain > 0) {
 425                 struct page *page;
 426
 427                 /* Operation in this page
 428                  *
 429                  * shmem_page_offset = offset within page in shmem file
 430                  * page_length = bytes to copy for this page
 431                  */
 432                 shmem_page_offset = offset_in_page(offset);
 433                 page_length = remain;
 434                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 435                         page_length = PAGE_SIZE - shmem_page_offset;
 436
 437                 if (obj->pages) {
 438                         page = obj->pages[offset >> PAGE_SHIFT];
 439                         release_page = 0;
 440                 } else {
 441                         page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
 442                         if (IS_ERR(page)) {
 443                                 ret = PTR_ERR(page);
 444                                 goto out;
 445                         }
 446                         release_page = 1;
 447                 }
 448
 449                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 450                         (page_to_phys(page) & (1 << 17)) != 0;
 451
 452                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
 453                                        user_data, page_do_bit17_swizzling,
 454                                        needs_clflush);
 455                 if (ret == 0)
 456                         goto next_page;
 457
 458                 hit_slowpath = 1;
 459                 page_cache_get(page);
 460                 mutex_unlock(&dev->struct_mutex);
 461
 462                 if (!prefaulted) {
 463                         ret = fault_in_multipages_writeable(user_data, remain);
 464                         /* Userspace is tricking us, but we've already clobbered
 465                          * its pages with the prefault and promised to write the
 466                          * data up to the first fault. Hence ignore any errors
 467                          * and just continue. */
 468                         (void)ret;
 469                         prefaulted = 1;
 470                 }
 471
 472                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
 473                                        user_data, page_do_bit17_swizzling,
 474                                        needs_clflush);
 475
 476                 mutex_lock(&dev->struct_mutex);
 477                 page_cache_release(page);
 478 next_page:
 479                 mark_page_accessed(page);
 480                 if (release_page)
 481                         page_cache_release(page);
 482
 483                 if (ret) {
 484                         ret = -EFAULT;
 485                         goto out;
 486                 }
 487
 488                 remain -= page_length;
 489                 user_data += page_length;
 490                 offset += page_length;
 491         }
 492
 493 out:
 494         if (hit_slowpath) {
 495                 /* Fixup: Kill any reinstated backing storage pages */
 496                 if (obj->madv == __I915_MADV_PURGED)
 497                         i915_gem_object_truncate(obj);
 498         }
 499
 500         return ret;
 501 }
 502
 503 /**
 504  * Reads data from the object referenced by handle.
 505  *
 506  * On error, the contents of *data are undefined.
 507  */
 508 int
 509 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 510                      struct drm_file *file)
 511 {
 512         struct drm_i915_gem_pread *args = data;
 513         struct drm_i915_gem_object *obj;
 514         int ret = 0;
 515
 516         if (args->size == 0)
 517                 return 0;
 518
 519         if (!access_ok(VERIFY_WRITE,
 520                        (char __user *)(uintptr_t)args->data_ptr,
 521                        args->size))
 522                 return -EFAULT;
 523
 524         ret = i915_mutex_lock_interruptible(dev);
 525         if (ret)
 526                 return ret;
 527
 528         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 529         if (&obj->base == NULL) {
 530                 ret = -ENOENT;
 531                 goto unlock;
 532         }
 533
 534         /* Bounds check source.  */
 535         if (args->offset > obj->base.size ||
 536             args->size > obj->base.size - args->offset) {
 537                 ret = -EINVAL;
 538                 goto out;
 539         }
 540
 541         trace_i915_gem_object_pread(obj, args->offset, args->size);
 542
 543         ret = i915_gem_shmem_pread(dev, obj, args, file);
 544
 545 out:
 546         drm_gem_object_unreference(&obj->base);
 547 unlock:
 548         mutex_unlock(&dev->struct_mutex);
 549         return ret;
 550 }
 551
 552 /* This is the fast write path which cannot handle
 553  * page faults in the source data
 554  */
 555
 556 static inline int
 557 fast_user_write(struct io_mapping *mapping,
 558                 loff_t page_base, int page_offset,
 559                 char __user *user_data,
 560                 int length)
 561 {
 562         void __iomem *vaddr_atomic;
 563         void *vaddr;
 564         unsigned long unwritten;
 565
 566         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 567         /* We can use the cpu mem copy function because this is X86. */
 568         vaddr = (void __force*)vaddr_atomic + page_offset;
 569         unwritten = __copy_from_user_inatomic_nocache(vaddr,
 570                                                       user_data, length);
 571         io_mapping_unmap_atomic(vaddr_atomic);
 572         return unwritten;
 573 }
 574
 575 /**
 576  * This is the fast pwrite path, where we copy the data directly from the
 577  * user into the GTT, uncached.
 578  */
 579 static int
 580 i915_gem_gtt_pwrite_fast(struct drm_device *dev,
 581                          struct drm_i915_gem_object *obj,
 582                          struct drm_i915_gem_pwrite *args,
 583                          struct drm_file *file)
 584 {
 585         drm_i915_private_t *dev_priv = dev->dev_private;
 586         ssize_t remain;
 587         loff_t offset, page_base;
 588         char __user *user_data;
 589         int page_offset, page_length, ret;
 590
 591         ret = i915_gem_object_pin(obj, 0, true);
 592         if (ret)
 593                 goto out;
 594
 595         ret = i915_gem_object_set_to_gtt_domain(obj, true);
 596         if (ret)
 597                 goto out_unpin;
 598
 599         ret = i915_gem_object_put_fence(obj);
 600         if (ret)
 601                 goto out_unpin;
 602
 603         user_data = (char __user *) (uintptr_t) args->data_ptr;
 604         remain = args->size;
 605
 606         offset = obj->gtt_offset + args->offset;
 607
 608         while (remain > 0) {
 609                 /* Operation in this page
 610                  *
 611                  * page_base = page offset within aperture
 612                  * page_offset = offset within page
 613                  * page_length = bytes to copy for this page
 614                  */
 615                 page_base = offset & PAGE_MASK;
 616                 page_offset = offset_in_page(offset);
 617                 page_length = remain;
 618                 if ((page_offset + remain) > PAGE_SIZE)
 619                         page_length = PAGE_SIZE - page_offset;
 620
 621                 /* If we get a fault while copying data, then (presumably) our
 622                  * source page isn't available.  Return the error and we'll
 623                  * retry in the slow path.
 624                  */
 625                 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
 626                                     page_offset, user_data, page_length)) {
 627                         ret = -EFAULT;
 628                         goto out_unpin;
 629                 }
 630
 631                 remain -= page_length;
 632                 user_data += page_length;
 633                 offset += page_length;
 634         }
 635
 636 out_unpin:
 637         i915_gem_object_unpin(obj);
 638 out:
 639         return ret;
 640 }
 641
 642 /* Per-page copy function for the shmem pwrite fastpath.
 643  * Flushes invalid cachelines before writing to the target if
 644  * needs_clflush_before is set and flushes out any written cachelines after
 645  * writing if needs_clflush is set. */
 646 static int
 647 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
 648                   char __user *user_data,
 649                   bool page_do_bit17_swizzling,
 650                   bool needs_clflush_before,
 651                   bool needs_clflush_after)
 652 {
 653         char *vaddr;
 654         int ret;
 655
 656         if (unlikely(page_do_bit17_swizzling))
 657                 return -EINVAL;
 658
 659         vaddr = kmap_atomic(page);
 660         if (needs_clflush_before)
 661                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 662                                        page_length);
 663         ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
 664                                                 user_data,
 665                                                 page_length);
 666         if (needs_clflush_after)
 667                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 668                                        page_length);
 669         kunmap_atomic(vaddr);
 670
 671         return ret;
 672 }
 673
 674 /* Only difference to the fast-path function is that this can handle bit17
 675  * and uses non-atomic copy and kmap functions. */
 676 static int
 677 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
 678                   char __user *user_data,
 679                   bool page_do_bit17_swizzling,
 680                   bool needs_clflush_before,
 681                   bool needs_clflush_after)
 682 {
 683         char *vaddr;
 684         int ret;
 685
 686         vaddr = kmap(page);
 687         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
 688                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 689                                              page_length,
 690                                              page_do_bit17_swizzling);
 691         if (page_do_bit17_swizzling)
 692                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
 693                                                 user_data,
 694                                                 page_length);
 695         else
 696                 ret = __copy_from_user(vaddr + shmem_page_offset,
 697                                        user_data,
 698                                        page_length);
 699         if (needs_clflush_after)
 700                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 701                                              page_length,
 702                                              page_do_bit17_swizzling);
 703         kunmap(page);
 704
 705         return ret;
 706 }
 707
 708 static int
 709 i915_gem_shmem_pwrite(struct drm_device *dev,
 710                       struct drm_i915_gem_object *obj,
 711                       struct drm_i915_gem_pwrite *args,
 712                       struct drm_file *file)
 713 {
 714         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
 715         ssize_t remain;
 716         loff_t offset;
 717         char __user *user_data;
 718         int shmem_page_offset, page_length, ret = 0;
 719         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 720         int hit_slowpath = 0;
 721         int needs_clflush_after = 0;
 722         int needs_clflush_before = 0;
 723         int release_page;
 724
 725         user_data = (char __user *) (uintptr_t) args->data_ptr;
 726         remain = args->size;
 727
 728         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 729
 730         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 731                 /* If we're not in the cpu write domain, set ourself into the gtt
 732                  * write domain and manually flush cachelines (if required). This
 733                  * optimizes for the case when the gpu will use the data
 734                  * right away and we therefore have to clflush anyway. */
 735                 if (obj->cache_level == I915_CACHE_NONE)
 736                         needs_clflush_after = 1;
 737                 ret = i915_gem_object_set_to_gtt_domain(obj, true);
 738                 if (ret)
 739                         return ret;
 740         }
 741         /* Same trick applies for invalidate partially written cachelines before
 742          * writing.  */
 743         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
 744             && obj->cache_level == I915_CACHE_NONE)
 745                 needs_clflush_before = 1;
 746
 747         offset = args->offset;
 748         obj->dirty = 1;
 749
 750         while (remain > 0) {
 751                 struct page *page;
 752                 int partial_cacheline_write;
 753
 754                 /* Operation in this page
 755                  *
 756                  * shmem_page_offset = offset within page in shmem file
 757                  * page_length = bytes to copy for this page
 758                  */
 759                 shmem_page_offset = offset_in_page(offset);
 760
 761                 page_length = remain;
 762                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 763                         page_length = PAGE_SIZE - shmem_page_offset;
 764
 765                 /* If we don't overwrite a cacheline completely we need to be
 766                  * careful to have up-to-date data by first clflushing. Don't
 767                  * overcomplicate things and flush the entire patch. */
 768                 partial_cacheline_write = needs_clflush_before &&
 769                         ((shmem_page_offset | page_length)
 770                                 & (boot_cpu_data.x86_clflush_size - 1));
 771
 772                 if (obj->pages) {
 773                         page = obj->pages[offset >> PAGE_SHIFT];
 774                         release_page = 0;
 775                 } else {
 776                         page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
 777                         if (IS_ERR(page)) {
 778                                 ret = PTR_ERR(page);
 779                                 goto out;
 780                         }
 781                         release_page = 1;
 782                 }
 783
 784                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 785                         (page_to_phys(page) & (1 << 17)) != 0;
 786
 787                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
 788                                         user_data, page_do_bit17_swizzling,
 789                                         partial_cacheline_write,
 790                                         needs_clflush_after);
 791                 if (ret == 0)
 792                         goto next_page;
 793
 794                 hit_slowpath = 1;
 795                 page_cache_get(page);
 796                 mutex_unlock(&dev->struct_mutex);
 797
 798                 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
 799                                         user_data, page_do_bit17_swizzling,
 800                                         partial_cacheline_write,
 801                                         needs_clflush_after);
 802
 803                 mutex_lock(&dev->struct_mutex);
 804                 page_cache_release(page);
 805 next_page:
 806                 set_page_dirty(page);
 807                 mark_page_accessed(page);
 808                 if (release_page)
 809                         page_cache_release(page);
 810
 811                 if (ret) {
 812                         ret = -EFAULT;
 813                         goto out;
 814                 }
 815
 816                 remain -= page_length;
 817                 user_data += page_length;
 818                 offset += page_length;
 819         }
 820
 821 out:
 822         if (hit_slowpath) {
 823                 /* Fixup: Kill any reinstated backing storage pages */
 824                 if (obj->madv == __I915_MADV_PURGED)
 825                         i915_gem_object_truncate(obj);
 826                 /* and flush dirty cachelines in case the object isn't in the cpu write
 827                  * domain anymore. */
 828                 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 829                         i915_gem_clflush_object(obj);
 830                         intel_gtt_chipset_flush();
 831                 }
 832         }
 833
 834         if (needs_clflush_after)
 835                 intel_gtt_chipset_flush();
 836
 837         return ret;
 838 }
 839
 840 /**
 841  * Writes data to the object referenced by handle.
 842  *
 843  * On error, the contents of the buffer that were to be modified are undefined.
 844  */
 845 int
 846 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 847                       struct drm_file *file)
 848 {
 849         struct drm_i915_gem_pwrite *args = data;
 850         struct drm_i915_gem_object *obj;
 851         int ret;
 852
 853         if (args->size == 0)
 854                 return 0;
 855
 856         if (!access_ok(VERIFY_READ,
 857                        (char __user *)(uintptr_t)args->data_ptr,
 858                        args->size))
 859                 return -EFAULT;
 860
 861         ret = fault_in_multipages_readable((char __user *)(uintptr_t)args->data_ptr,
 862                                            args->size);
 863         if (ret)
 864                 return -EFAULT;
 865
 866         ret = i915_mutex_lock_interruptible(dev);
 867         if (ret)
 868                 return ret;
 869
 870         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 871         if (&obj->base == NULL) {
 872                 ret = -ENOENT;
 873                 goto unlock;
 874         }
 875
 876         /* Bounds check destination. */
 877         if (args->offset > obj->base.size ||
 878             args->size > obj->base.size - args->offset) {
 879                 ret = -EINVAL;
 880                 goto out;
 881         }
 882
 883         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
 884
 885         ret = -EFAULT;
 886         /* We can only do the GTT pwrite on untiled buffers, as otherwise
 887          * it would end up going through the fenced access, and we'll get
 888          * different detiling behavior between reading and writing.
 889          * pread/pwrite currently are reading and writing from the CPU
 890          * perspective, requiring manual detiling by the client.
 891          */
 892         if (obj->phys_obj) {
 893                 ret = i915_gem_phys_pwrite(dev, obj, args, file);
 894                 goto out;
 895         }
 896
 897         if (obj->gtt_space &&
 898             obj->cache_level == I915_CACHE_NONE &&
 899             obj->tiling_mode == I915_TILING_NONE &&
 900             obj->map_and_fenceable &&
 901             obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 902                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
 903                 /* Note that the gtt paths might fail with non-page-backed user
 904                  * pointers (e.g. gtt mappings when moving data between
 905                  * textures). Fallback to the shmem path in that case. */
 906         }
 907
 908         if (ret == -EFAULT)
 909                 ret = i915_gem_shmem_pwrite(dev, obj, args, file);
 910
 911 out:
 912         drm_gem_object_unreference(&obj->base);
 913 unlock:
 914         mutex_unlock(&dev->struct_mutex);
 915         return ret;
 916 }
 917
 918 /**
 919  * Called when user space prepares to use an object with the CPU, either
 920  * through the mmap ioctl's mapping or a GTT mapping.
 921  */
 922 int
 923 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 924                           struct drm_file *file)
 925 {
 926         struct drm_i915_gem_set_domain *args = data;
 927         struct drm_i915_gem_object *obj;
 928         uint32_t read_domains = args->read_domains;
 929         uint32_t write_domain = args->write_domain;
 930         int ret;
 931
 932         /* Only handle setting domains to types used by the CPU. */
 933         if (write_domain & I915_GEM_GPU_DOMAINS)
 934                 return -EINVAL;
 935
 936         if (read_domains & I915_GEM_GPU_DOMAINS)
 937                 return -EINVAL;
 938
 939         /* Having something in the write domain implies it's in the read
 940          * domain, and only that read domain.  Enforce that in the request.
 941          */
 942         if (write_domain != 0 && read_domains != write_domain)
 943                 return -EINVAL;
 944
 945         ret = i915_mutex_lock_interruptible(dev);
 946         if (ret)
 947                 return ret;
 948
 949         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 950         if (&obj->base == NULL) {
 951                 ret = -ENOENT;
 952                 goto unlock;
 953         }
 954
 955         if (read_domains & I915_GEM_DOMAIN_GTT) {
 956                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
 957
 958                 /* Silently promote "you're not bound, there was nothing to do"
 959                  * to success, since the client was just asking us to
 960                  * make sure everything was done.
 961                  */
 962                 if (ret == -EINVAL)
 963                         ret = 0;
 964         } else {
 965                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
 966         }
 967
 968         drm_gem_object_unreference(&obj->base);
 969 unlock:
 970         mutex_unlock(&dev->struct_mutex);
 971         return ret;
 972 }
 973
 974 /**
 975  * Called when user space has done writes to this buffer
 976  */
 977 int
 978 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 979                          struct drm_file *file)
 980 {
 981         struct drm_i915_gem_sw_finish *args = data;
 982         struct drm_i915_gem_object *obj;
 983         int ret = 0;
 984
 985         ret = i915_mutex_lock_interruptible(dev);
 986         if (ret)
 987                 return ret;
 988
 989         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 990         if (&obj->base == NULL) {
 991                 ret = -ENOENT;
 992                 goto unlock;
 993         }
 994
 995         /* Pinned buffers may be scanout, so flush the cache */
 996         if (obj->pin_count)
 997                 i915_gem_object_flush_cpu_write_domain(obj);
 998
 999         drm_gem_object_unreference(&obj->base);
1000 unlock:
1001         mutex_unlock(&dev->struct_mutex);
1002         return ret;
1003 }
1004
1005 /**
1006  * Maps the contents of an object, returning the address it is mapped
1007  * into.
1008  *
1009  * While the mapping holds a reference on the contents of the object, it doesn't
1010  * imply a ref on the object itself.
1011  */
1012 int
1013 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1014                     struct drm_file *file)
1015 {
1016         struct drm_i915_gem_mmap *args = data;
1017         struct drm_gem_object *obj;
1018         unsigned long addr;
1019
1020         obj = drm_gem_object_lookup(dev, file, args->handle);
1021         if (obj == NULL)
1022                 return -ENOENT;
1023
1024         addr = vm_mmap(obj->filp, 0, args->size,
1025                        PROT_READ | PROT_WRITE, MAP_SHARED,
1026                        args->offset);
1027         drm_gem_object_unreference_unlocked(obj);
1028         if (IS_ERR((void *)addr))
1029                 return addr;
1030
1031         args->addr_ptr = (uint64_t) addr;
1032
1033         return 0;
1034 }
1035
1036 /**
1037  * i915_gem_fault - fault a page into the GTT
1038  * vma: VMA in question
1039  * vmf: fault info
1040  *
1041  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1042  * from userspace.  The fault handler takes care of binding the object to
1043  * the GTT (if needed), allocating and programming a fence register (again,
1044  * only if needed based on whether the old reg is still valid or the object
1045  * is tiled) and inserting a new PTE into the faulting process.
1046  *
1047  * Note that the faulting process may involve evicting existing objects
1048  * from the GTT and/or fence registers to make room.  So performance may
1049  * suffer if the GTT working set is large or there are few fence registers
1050  * left.
1051  */
1052 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1053 {
1054         struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1055         struct drm_device *dev = obj->base.dev;
1056         drm_i915_private_t *dev_priv = dev->dev_private;
1057         pgoff_t page_offset;
1058         unsigned long pfn;
1059         int ret = 0;
1060         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1061
1062         /* We don't use vmf->pgoff since that has the fake offset */
1063         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1064                 PAGE_SHIFT;
1065
1066         ret = i915_mutex_lock_interruptible(dev);
1067         if (ret)
1068                 goto out;
1069
1070         trace_i915_gem_object_fault(obj, page_offset, true, write);
1071
1072         /* Now bind it into the GTT if needed */
1073         if (!obj->map_and_fenceable) {
1074                 ret = i915_gem_object_unbind(obj);
1075                 if (ret)
1076                         goto unlock;
1077         }
1078         if (!obj->gtt_space) {
1079                 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
1080                 if (ret)
1081                         goto unlock;
1082
1083                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1084                 if (ret)
1085                         goto unlock;
1086         }
1087
1088         if (!obj->has_global_gtt_mapping)
1089                 i915_gem_gtt_bind_object(obj, obj->cache_level);
1090
1091         ret = i915_gem_object_get_fence(obj);
1092         if (ret)
1093                 goto unlock;
1094
1095         if (i915_gem_object_is_inactive(obj))
1096                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1097
1098         obj->fault_mappable = true;
1099
1100         pfn = ((dev->agp->base + obj->gtt_offset) >> PAGE_SHIFT) +
1101                 page_offset;
1102
1103         /* Finally, remap it using the new GTT offset */
1104         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1105 unlock:
1106         mutex_unlock(&dev->struct_mutex);
1107 out:
1108         switch (ret) {
1109         case -EIO:
1110         case -EAGAIN:
1111                 /* Give the error handler a chance to run and move the
1112                  * objects off the GPU active list. Next time we service the
1113                  * fault, we should be able to transition the page into the
1114                  * GTT without touching the GPU (and so avoid further
1115                  * EIO/EGAIN). If the GPU is wedged, then there is no issue
1116                  * with coherency, just lost writes.
1117                  */
1118                 set_need_resched();
1119         case 0:
1120         case -ERESTARTSYS:
1121         case -EINTR:
1122                 return VM_FAULT_NOPAGE;
1123         case -ENOMEM:
1124                 return VM_FAULT_OOM;
1125         default:
1126                 return VM_FAULT_SIGBUS;
1127         }
1128 }
1129
1130 /**
1131  * i915_gem_release_mmap - remove physical page mappings
1132  * @obj: obj in question
1133  *
1134  * Preserve the reservation of the mmapping with the DRM core code, but
1135  * relinquish ownership of the pages back to the system.
1136  *
1137  * It is vital that we remove the page mapping if we have mapped a tiled
1138  * object through the GTT and then lose the fence register due to
1139  * resource pressure. Similarly if the object has been moved out of the
1140  * aperture, than pages mapped into userspace must be revoked. Removing the
1141  * mapping will then trigger a page fault on the next user access, allowing
1142  * fixup by i915_gem_fault().
1143  */
1144 void
1145 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1146 {
1147         if (!obj->fault_mappable)
1148                 return;
1149
1150         if (obj->base.dev->dev_mapping)
1151                 unmap_mapping_range(obj->base.dev->dev_mapping,
1152                                     (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
1153                                     obj->base.size, 1);
1154
1155         obj->fault_mappable = false;
1156 }
1157
1158 static uint32_t
1159 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1160 {
1161         uint32_t gtt_size;
1162
1163         if (INTEL_INFO(dev)->gen >= 4 ||
1164             tiling_mode == I915_TILING_NONE)
1165                 return size;
1166
1167         /* Previous chips need a power-of-two fence region when tiling */
1168         if (INTEL_INFO(dev)->gen == 3)
1169                 gtt_size = 1024*1024;
1170         else
1171                 gtt_size = 512*1024;
1172
1173         while (gtt_size < size)
1174                 gtt_size <<= 1;
1175
1176         return gtt_size;
1177 }
1178
1179 /**
1180  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1181  * @obj: object to check
1182  *
1183  * Return the required GTT alignment for an object, taking into account
1184  * potential fence register mapping.
1185  */
1186 static uint32_t
1187 i915_gem_get_gtt_alignment(struct drm_device *dev,
1188                            uint32_t size,
1189                            int tiling_mode)
1190 {
1191         /*
1192          * Minimum alignment is 4k (GTT page size), but might be greater
1193          * if a fence register is needed for the object.
1194          */
1195         if (INTEL_INFO(dev)->gen >= 4 ||
1196             tiling_mode == I915_TILING_NONE)
1197                 return 4096;
1198
1199         /*
1200          * Previous chips need to be aligned to the size of the smallest
1201          * fence register that can contain the object.
1202          */
1203         return i915_gem_get_gtt_size(dev, size, tiling_mode);
1204 }
1205
1206 /**
1207  * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
1208  *                                       unfenced object
1209  * @dev: the device
1210  * @size: size of the object
1211  * @tiling_mode: tiling mode of the object
1212  *
1213  * Return the required GTT alignment for an object, only taking into account
1214  * unfenced tiled surface requirements.
1215  */
1216 uint32_t
1217 i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
1218                                     uint32_t size,
1219                                     int tiling_mode)
1220 {
1221         /*
1222          * Minimum alignment is 4k (GTT page size) for sane hw.
1223          */
1224         if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
1225             tiling_mode == I915_TILING_NONE)
1226                 return 4096;
1227
1228         /* Previous hardware however needs to be aligned to a power-of-two
1229          * tile height. The simplest method for determining this is to reuse
1230          * the power-of-tile object size.
1231          */
1232         return i915_gem_get_gtt_size(dev, size, tiling_mode);
1233 }
1234
1235 int
1236 i915_gem_mmap_gtt(struct drm_file *file,
1237                   struct drm_device *dev,
1238                   uint32_t handle,
1239                   uint64_t *offset)
1240 {
1241         struct drm_i915_private *dev_priv = dev->dev_private;
1242         struct drm_i915_gem_object *obj;
1243         int ret;
1244
1245         ret = i915_mutex_lock_interruptible(dev);
1246         if (ret)
1247                 return ret;
1248
1249         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1250         if (&obj->base == NULL) {
1251                 ret = -ENOENT;
1252                 goto unlock;
1253         }
1254
1255         if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
1256                 ret = -E2BIG;
1257                 goto out;
1258         }
1259
1260         if (obj->madv != I915_MADV_WILLNEED) {
1261                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1262                 ret = -EINVAL;
1263                 goto out;
1264         }
1265
1266         if (!obj->base.map_list.map) {
1267                 ret = drm_gem_create_mmap_offset(&obj->base);
1268                 if (ret)
1269                         goto out;
1270         }
1271
1272         *offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
1273
1274 out:
1275         drm_gem_object_unreference(&obj->base);
1276 unlock:
1277         mutex_unlock(&dev->struct_mutex);
1278         return ret;
1279 }
1280
1281 /**
1282  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1283  * @dev: DRM device
1284  * @data: GTT mapping ioctl data
1285  * @file: GEM object info
1286  *
1287  * Simply returns the fake offset to userspace so it can mmap it.
1288  * The mmap call will end up in drm_gem_mmap(), which will set things
1289  * up so we can get faults in the handler above.
1290  *
1291  * The fault handler will take care of binding the object into the GTT
1292  * (since it may have been evicted to make room for something), allocating
1293  * a fence register, and mapping the appropriate aperture address into
1294  * userspace.
1295  */
1296 int
1297 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1298                         struct drm_file *file)
1299 {
1300         struct drm_i915_gem_mmap_gtt *args = data;
1301
1302         return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
1303 }
1304
1305
1306 static int
1307 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
1308                               gfp_t gfpmask)
1309 {
1310         int page_count, i;
1311         struct address_space *mapping;
1312         struct inode *inode;
1313         struct page *page;
1314
1315         /* Get the list of pages out of our struct file.  They'll be pinned
1316          * at this point until we release them.
1317          */
1318         page_count = obj->base.size / PAGE_SIZE;
1319         BUG_ON(obj->pages != NULL);
1320         obj->pages = drm_malloc_ab(page_count, sizeof(struct page *));
1321         if (obj->pages == NULL)
1322                 return -ENOMEM;
1323
1324         inode = obj->base.filp->f_path.dentry->d_inode;
1325         mapping = inode->i_mapping;
1326         gfpmask |= mapping_gfp_mask(mapping);
1327
1328         for (i = 0; i < page_count; i++) {
1329                 page = shmem_read_mapping_page_gfp(mapping, i, gfpmask);
1330                 if (IS_ERR(page))
1331                         goto err_pages;
1332
1333                 obj->pages[i] = page;
1334         }
1335
1336         if (i915_gem_object_needs_bit17_swizzle(obj))
1337                 i915_gem_object_do_bit_17_swizzle(obj);
1338
1339         return 0;
1340
1341 err_pages:
1342         while (i--)
1343                 page_cache_release(obj->pages[i]);
1344
1345         drm_free_large(obj->pages);
1346         obj->pages = NULL;
1347         return PTR_ERR(page);
1348 }
1349
1350 static void
1351 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1352 {
1353         int page_count = obj->base.size / PAGE_SIZE;
1354         int i;
1355
1356         BUG_ON(obj->madv == __I915_MADV_PURGED);
1357
1358         if (i915_gem_object_needs_bit17_swizzle(obj))
1359                 i915_gem_object_save_bit_17_swizzle(obj);
1360
1361         if (obj->madv == I915_MADV_DONTNEED)
1362                 obj->dirty = 0;
1363
1364         for (i = 0; i < page_count; i++) {
1365                 if (obj->dirty)
1366                         set_page_dirty(obj->pages[i]);
1367
1368                 if (obj->madv == I915_MADV_WILLNEED)
1369                         mark_page_accessed(obj->pages[i]);
1370
1371                 page_cache_release(obj->pages[i]);
1372         }
1373         obj->dirty = 0;
1374
1375         drm_free_large(obj->pages);
1376         obj->pages = NULL;
1377 }
1378
1379 void
1380 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1381                                struct intel_ring_buffer *ring,
1382                                u32 seqno)
1383 {
1384         struct drm_device *dev = obj->base.dev;
1385         struct drm_i915_private *dev_priv = dev->dev_private;
1386
1387         BUG_ON(ring == NULL);
1388         obj->ring = ring;
1389
1390         /* Add a reference if we're newly entering the active list. */
1391         if (!obj->active) {
1392                 drm_gem_object_reference(&obj->base);
1393                 obj->active = 1;
1394         }
1395
1396         /* Move from whatever list we were on to the tail of execution. */
1397         list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
1398         list_move_tail(&obj->ring_list, &ring->active_list);
1399
1400         obj->last_rendering_seqno = seqno;
1401
1402         if (obj->fenced_gpu_access) {
1403                 obj->last_fenced_seqno = seqno;
1404
1405                 /* Bump MRU to take account of the delayed flush */
1406                 if (obj->fence_reg != I915_FENCE_REG_NONE) {
1407                         struct drm_i915_fence_reg *reg;
1408
1409                         reg = &dev_priv->fence_regs[obj->fence_reg];
1410                         list_move_tail(&reg->lru_list,
1411                                        &dev_priv->mm.fence_list);
1412                 }
1413         }
1414 }
1415
1416 static void
1417 i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
1418 {
1419         list_del_init(&obj->ring_list);
1420         obj->last_rendering_seqno = 0;
1421         obj->last_fenced_seqno = 0;
1422 }
1423
1424 static void
1425 i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
1426 {
1427         struct drm_device *dev = obj->base.dev;
1428         drm_i915_private_t *dev_priv = dev->dev_private;
1429
1430         BUG_ON(!obj->active);
1431         list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
1432
1433         i915_gem_object_move_off_active(obj);
1434 }
1435
1436 static void
1437 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1438 {
1439         struct drm_device *dev = obj->base.dev;
1440         struct drm_i915_private *dev_priv = dev->dev_private;
1441
1442         list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1443
1444         BUG_ON(!list_empty(&obj->gpu_write_list));
1445         BUG_ON(!obj->active);
1446         obj->ring = NULL;
1447
1448         i915_gem_object_move_off_active(obj);
1449         obj->fenced_gpu_access = false;
1450
1451         obj->active = 0;
1452         obj->pending_gpu_write = false;
1453         drm_gem_object_unreference(&obj->base);
1454
1455         WARN_ON(i915_verify_lists(dev));
1456 }
1457
1458 /* Immediately discard the backing storage */
1459 static void
1460 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1461 {
1462         struct inode *inode;
1463
1464         /* Our goal here is to return as much of the memory as
1465          * is possible back to the system as we are called from OOM.
1466          * To do this we must instruct the shmfs to drop all of its
1467          * backing pages, *now*.
1468          */
1469         inode = obj->base.filp->f_path.dentry->d_inode;
1470         shmem_truncate_range(inode, 0, (loff_t)-1);
1471
1472         if (obj->base.map_list.map)
1473                 drm_gem_free_mmap_offset(&obj->base);
1474
1475         obj->madv = __I915_MADV_PURGED;
1476 }
1477
1478 static inline int
1479 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
1480 {
1481         return obj->madv == I915_MADV_DONTNEED;
1482 }
1483
1484 static void
1485 i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
1486                                uint32_t flush_domains)
1487 {
1488         struct drm_i915_gem_object *obj, *next;
1489
1490         list_for_each_entry_safe(obj, next,
1491                                  &ring->gpu_write_list,
1492                                  gpu_write_list) {
1493                 if (obj->base.write_domain & flush_domains) {
1494                         uint32_t old_write_domain = obj->base.write_domain;
1495
1496                         obj->base.write_domain = 0;
1497                         list_del_init(&obj->gpu_write_list);
1498                         i915_gem_object_move_to_active(obj, ring,
1499                                                        i915_gem_next_request_seqno(ring));
1500
1501                         trace_i915_gem_object_change_domain(obj,
1502                                                             obj->base.read_domains,
1503                                                             old_write_domain);
1504                 }
1505         }
1506 }
1507
1508 static u32
1509 i915_gem_get_seqno(struct drm_device *dev)
1510 {
1511         drm_i915_private_t *dev_priv = dev->dev_private;
1512         u32 seqno = dev_priv->next_seqno;
1513
1514         /* reserve 0 for non-seqno */
1515         if (++dev_priv->next_seqno == 0)
1516                 dev_priv->next_seqno = 1;
1517
1518         return seqno;
1519 }
1520
1521 u32
1522 i915_gem_next_request_seqno(struct intel_ring_buffer *ring)
1523 {
1524         if (ring->outstanding_lazy_request == 0)
1525                 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev);
1526
1527         return ring->outstanding_lazy_request;
1528 }
1529
1530 int
1531 i915_add_request(struct intel_ring_buffer *ring,
1532                  struct drm_file *file,
1533                  struct drm_i915_gem_request *request)
1534 {
1535         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1536         uint32_t seqno;
1537         u32 request_ring_position;
1538         int was_empty;
1539         int ret;
1540
1541         BUG_ON(request == NULL);
1542         seqno = i915_gem_next_request_seqno(ring);
1543
1544         /* Record the position of the start of the request so that
1545          * should we detect the updated seqno part-way through the
1546          * GPU processing the request, we never over-estimate the
1547          * position of the head.
1548          */
1549         request_ring_position = intel_ring_get_tail(ring);
1550
1551         ret = ring->add_request(ring, &seqno);
1552         if (ret)
1553             return ret;
1554
1555         trace_i915_gem_request_add(ring, seqno);
1556
1557         request->seqno = seqno;
1558         request->ring = ring;
1559         request->tail = request_ring_position;
1560         request->emitted_jiffies = jiffies;
1561         was_empty = list_empty(&ring->request_list);
1562         list_add_tail(&request->list, &ring->request_list);
1563
1564         if (file) {
1565                 struct drm_i915_file_private *file_priv = file->driver_priv;
1566
1567                 spin_lock(&file_priv->mm.lock);
1568                 request->file_priv = file_priv;
1569                 list_add_tail(&request->client_list,
1570                               &file_priv->mm.request_list);
1571                 spin_unlock(&file_priv->mm.lock);
1572         }
1573
1574         ring->outstanding_lazy_request = 0;
1575
1576         if (!dev_priv->mm.suspended) {
1577                 if (i915_enable_hangcheck) {
1578                         mod_timer(&dev_priv->hangcheck_timer,
1579                                   jiffies +
1580                                   msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
1581                 }
1582                 if (was_empty)
1583                         queue_delayed_work(dev_priv->wq,
1584                                            &dev_priv->mm.retire_work, HZ);
1585         }
1586         return 0;
1587 }
1588
1589 static inline void
1590 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1591 {
1592         struct drm_i915_file_private *file_priv = request->file_priv;
1593
1594         if (!file_priv)
1595                 return;
1596
1597         spin_lock(&file_priv->mm.lock);
1598         if (request->file_priv) {
1599                 list_del(&request->client_list);
1600                 request->file_priv = NULL;
1601         }
1602         spin_unlock(&file_priv->mm.lock);
1603 }
1604
1605 static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1606                                       struct intel_ring_buffer *ring)
1607 {
1608         while (!list_empty(&ring->request_list)) {
1609                 struct drm_i915_gem_request *request;
1610
1611                 request = list_first_entry(&ring->request_list,
1612                                            struct drm_i915_gem_request,
1613                                            list);
1614
1615                 list_del(&request->list);
1616                 i915_gem_request_remove_from_client(request);
1617                 kfree(request);
1618         }
1619
1620         while (!list_empty(&ring->active_list)) {
1621                 struct drm_i915_gem_object *obj;
1622
1623                 obj = list_first_entry(&ring->active_list,
1624                                        struct drm_i915_gem_object,
1625                                        ring_list);
1626
1627                 obj->base.write_domain = 0;
1628                 list_del_init(&obj->gpu_write_list);
1629                 i915_gem_object_move_to_inactive(obj);
1630         }
1631 }
1632
1633 static void i915_gem_reset_fences(struct drm_device *dev)
1634 {
1635         struct drm_i915_private *dev_priv = dev->dev_private;
1636         int i;
1637
1638         for (i = 0; i < dev_priv->num_fence_regs; i++) {
1639                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
1640
1641                 i915_gem_write_fence(dev, i, NULL);
1642
1643                 if (reg->obj)
1644                         i915_gem_object_fence_lost(reg->obj);
1645
1646                 reg->pin_count = 0;
1647                 reg->obj = NULL;
1648                 INIT_LIST_HEAD(&reg->lru_list);
1649         }
1650
1651         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
1652 }
1653
1654 void i915_gem_reset(struct drm_device *dev)
1655 {
1656         struct drm_i915_private *dev_priv = dev->dev_private;
1657         struct drm_i915_gem_object *obj;
1658         struct intel_ring_buffer *ring;
1659         int i;
1660
1661         for_each_ring(ring, dev_priv, i)
1662                 i915_gem_reset_ring_lists(dev_priv, ring);
1663
1664         /* Remove anything from the flushing lists. The GPU cache is likely
1665          * to be lost on reset along with the data, so simply move the
1666          * lost bo to the inactive list.
1667          */
1668         while (!list_empty(&dev_priv->mm.flushing_list)) {
1669                 obj = list_first_entry(&dev_priv->mm.flushing_list,
1670                                       struct drm_i915_gem_object,
1671                                       mm_list);
1672
1673                 obj->base.write_domain = 0;
1674                 list_del_init(&obj->gpu_write_list);
1675                 i915_gem_object_move_to_inactive(obj);
1676         }
1677
1678         /* Move everything out of the GPU domains to ensure we do any
1679          * necessary invalidation upon reuse.
1680          */
1681         list_for_each_entry(obj,
1682                             &dev_priv->mm.inactive_list,
1683                             mm_list)
1684         {
1685                 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1686         }
1687
1688         /* The fence registers are invalidated so clear them out */
1689         i915_gem_reset_fences(dev);
1690 }
1691
1692 /**
1693  * This function clears the request list as sequence numbers are passed.
1694  */
1695 void
1696 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
1697 {
1698         uint32_t seqno;
1699         int i;
1700
1701         if (list_empty(&ring->request_list))
1702                 return;
1703
1704         WARN_ON(i915_verify_lists(ring->dev));
1705
1706         seqno = ring->get_seqno(ring);
1707
1708         for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
1709                 if (seqno >= ring->sync_seqno[i])
1710                         ring->sync_seqno[i] = 0;
1711
1712         while (!list_empty(&ring->request_list)) {
1713                 struct drm_i915_gem_request *request;
1714
1715                 request = list_first_entry(&ring->request_list,
1716                                            struct drm_i915_gem_request,
1717                                            list);
1718
1719                 if (!i915_seqno_passed(seqno, request->seqno))
1720                         break;
1721
1722                 trace_i915_gem_request_retire(ring, request->seqno);
1723                 /* We know the GPU must have read the request to have
1724                  * sent us the seqno + interrupt, so use the position
1725                  * of tail of the request to update the last known position
1726                  * of the GPU head.
1727                  */
1728                 ring->last_retired_head = request->tail;
1729
1730                 list_del(&request->list);
1731                 i915_gem_request_remove_from_client(request);
1732                 kfree(request);
1733         }
1734
1735         /* Move any buffers on the active list that are no longer referenced
1736          * by the ringbuffer to the flushing/inactive lists as appropriate.
1737          */
1738         while (!list_empty(&ring->active_list)) {
1739                 struct drm_i915_gem_object *obj;
1740
1741                 obj = list_first_entry(&ring->active_list,
1742                                       struct drm_i915_gem_object,
1743                                       ring_list);
1744
1745                 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
1746                         break;
1747
1748                 if (obj->base.write_domain != 0)
1749                         i915_gem_object_move_to_flushing(obj);
1750                 else
1751                         i915_gem_object_move_to_inactive(obj);
1752         }
1753
1754         if (unlikely(ring->trace_irq_seqno &&
1755                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
1756                 ring->irq_put(ring);
1757                 ring->trace_irq_seqno = 0;
1758         }
1759
1760         WARN_ON(i915_verify_lists(ring->dev));
1761 }
1762
1763 void
1764 i915_gem_retire_requests(struct drm_device *dev)
1765 {
1766         drm_i915_private_t *dev_priv = dev->dev_private;
1767         struct intel_ring_buffer *ring;
1768         int i;
1769
1770         for_each_ring(ring, dev_priv, i)
1771                 i915_gem_retire_requests_ring(ring);
1772 }
1773
1774 static void
1775 i915_gem_retire_work_handler(struct work_struct *work)
1776 {
1777         drm_i915_private_t *dev_priv;
1778         struct drm_device *dev;
1779         struct intel_ring_buffer *ring;
1780         bool idle;
1781         int i;
1782
1783         dev_priv = container_of(work, drm_i915_private_t,
1784                                 mm.retire_work.work);
1785         dev = dev_priv->dev;
1786
1787         /* Come back later if the device is busy... */
1788         if (!mutex_trylock(&dev->struct_mutex)) {
1789                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1790                 return;
1791         }
1792
1793         i915_gem_retire_requests(dev);
1794
1795         /* Send a periodic flush down the ring so we don't hold onto GEM
1796          * objects indefinitely.
1797          */
1798         idle = true;
1799         for_each_ring(ring, dev_priv, i) {
1800                 if (!list_empty(&ring->gpu_write_list)) {
1801                         struct drm_i915_gem_request *request;
1802                         int ret;
1803
1804                         ret = i915_gem_flush_ring(ring,
1805                                                   0, I915_GEM_GPU_DOMAINS);
1806                         request = kzalloc(sizeof(*request), GFP_KERNEL);
1807                         if (ret || request == NULL ||
1808                             i915_add_request(ring, NULL, request))
1809                             kfree(request);
1810                 }
1811
1812                 idle &= list_empty(&ring->request_list);
1813         }
1814
1815         if (!dev_priv->mm.suspended && !idle)
1816                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1817
1818         mutex_unlock(&dev->struct_mutex);
1819 }
1820
1821 static int
1822 i915_gem_check_wedge(struct drm_i915_private *dev_priv)
1823 {
1824         BUG_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex));
1825
1826         if (atomic_read(&dev_priv->mm.wedged)) {
1827                 struct completion *x = &dev_priv->error_completion;
1828                 bool recovery_complete;
1829                 unsigned long flags;
1830
1831                 /* Give the error handler a chance to run. */
1832                 spin_lock_irqsave(&x->wait.lock, flags);
1833                 recovery_complete = x->done > 0;
1834                 spin_unlock_irqrestore(&x->wait.lock, flags);
1835
1836                 return recovery_complete ? -EIO : -EAGAIN;
1837         }
1838
1839         return 0;
1840 }
1841
1842 /*
1843  * Compare seqno against outstanding lazy request. Emit a request if they are
1844  * equal.
1845  */
1846 static int
1847 i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
1848 {
1849         int ret = 0;
1850
1851         BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
1852
1853         if (seqno == ring->outstanding_lazy_request) {
1854                 struct drm_i915_gem_request *request;
1855
1856                 request = kzalloc(sizeof(*request), GFP_KERNEL);
1857                 if (request == NULL)
1858                         return -ENOMEM;
1859
1860                 ret = i915_add_request(ring, NULL, request);
1861                 if (ret) {
1862                         kfree(request);
1863                         return ret;
1864                 }
1865
1866                 BUG_ON(seqno != request->seqno);
1867         }
1868
1869         return ret;
1870 }
1871
1872 static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
1873                         bool interruptible)
1874 {
1875         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1876         int ret = 0;
1877
1878         if (i915_seqno_passed(ring->get_seqno(ring), seqno))
1879                 return 0;
1880
1881         trace_i915_gem_request_wait_begin(ring, seqno);
1882         if (WARN_ON(!ring->irq_get(ring)))
1883                 return -ENODEV;
1884
1885 #define EXIT_COND \
1886         (i915_seqno_passed(ring->get_seqno(ring), seqno) || \
1887         atomic_read(&dev_priv->mm.wedged))
1888
1889         if (interruptible)
1890                 ret = wait_event_interruptible(ring->irq_queue,
1891                                                EXIT_COND);
1892         else
1893                 wait_event(ring->irq_queue, EXIT_COND);
1894
1895         ring->irq_put(ring);
1896         trace_i915_gem_request_wait_end(ring, seqno);
1897 #undef EXIT_COND
1898
1899         return ret;
1900 }
1901
1902 /**
1903  * Waits for a sequence number to be signaled, and cleans up the
1904  * request and object lists appropriately for that event.
1905  */
1906 int
1907 i915_wait_request(struct intel_ring_buffer *ring,
1908                   uint32_t seqno)
1909 {
1910         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1911         int ret = 0;
1912
1913         BUG_ON(seqno == 0);
1914
1915         ret = i915_gem_check_wedge(dev_priv);
1916         if (ret)
1917                 return ret;
1918
1919         ret = i915_gem_check_olr(ring, seqno);
1920         if (ret)
1921                 return ret;
1922
1923         ret = __wait_seqno(ring, seqno, dev_priv->mm.interruptible);
1924         if (atomic_read(&dev_priv->mm.wedged))
1925                 ret = -EAGAIN;
1926
1927         return ret;
1928 }
1929
1930 /**
1931  * Ensures that all rendering to the object has completed and the object is
1932  * safe to unbind from the GTT or access from the CPU.
1933  */
1934 int
1935 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
1936 {
1937         int ret;
1938
1939         /* This function only exists to support waiting for existing rendering,
1940          * not for emitting required flushes.
1941          */
1942         BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
1943
1944         /* If there is rendering queued on the buffer being evicted, wait for
1945          * it.
1946          */
1947         if (obj->active) {
1948                 ret = i915_wait_request(obj->ring, obj->last_rendering_seqno);
1949                 if (ret)
1950                         return ret;
1951                 i915_gem_retire_requests_ring(obj->ring);
1952         }
1953
1954         return 0;
1955 }
1956
1957 /**
1958  * i915_gem_object_sync - sync an object to a ring.
1959  *
1960  * @obj: object which may be in use on another ring.
1961  * @to: ring we wish to use the object on. May be NULL.
1962  *
1963  * This code is meant to abstract object synchronization with the GPU.
1964  * Calling with NULL implies synchronizing the object with the CPU
1965  * rather than a particular GPU ring.
1966  *
1967  * Returns 0 if successful, else propagates up the lower layer error.
1968  */
1969 int
1970 i915_gem_object_sync(struct drm_i915_gem_object *obj,
1971                      struct intel_ring_buffer *to)
1972 {
1973         struct intel_ring_buffer *from = obj->ring;
1974         u32 seqno;
1975         int ret, idx;
1976
1977         if (from == NULL || to == from)
1978                 return 0;
1979
1980         if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
1981                 return i915_gem_object_wait_rendering(obj);
1982
1983         idx = intel_ring_sync_index(from, to);
1984
1985         seqno = obj->last_rendering_seqno;
1986         if (seqno <= from->sync_seqno[idx])
1987                 return 0;
1988
1989         ret = i915_gem_check_olr(obj->ring, seqno);
1990         if (ret)
1991                 return ret;
1992
1993         ret = to->sync_to(to, from, seqno);
1994         if (!ret)
1995                 from->sync_seqno[idx] = seqno;
1996
1997         return ret;
1998 }
1999
2000 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
2001 {
2002         u32 old_write_domain, old_read_domains;
2003
2004         /* Act a barrier for all accesses through the GTT */
2005         mb();
2006
2007         /* Force a pagefault for domain tracking on next user access */
2008         i915_gem_release_mmap(obj);
2009
2010         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
2011                 return;
2012
2013         old_read_domains = obj->base.read_domains;
2014         old_write_domain = obj->base.write_domain;
2015
2016         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
2017         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2018
2019         trace_i915_gem_object_change_domain(obj,
2020                                             old_read_domains,
2021                                             old_write_domain);
2022 }
2023
2024 /**
2025  * Unbinds an object from the GTT aperture.
2026  */
2027 int
2028 i915_gem_object_unbind(struct drm_i915_gem_object *obj)
2029 {
2030         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2031         int ret = 0;
2032
2033         if (obj->gtt_space == NULL)
2034                 return 0;
2035
2036         if (obj->pin_count != 0) {
2037                 DRM_ERROR("Attempting to unbind pinned buffer\n");
2038                 return -EINVAL;
2039         }
2040
2041         ret = i915_gem_object_finish_gpu(obj);
2042         if (ret)
2043                 return ret;
2044         /* Continue on if we fail due to EIO, the GPU is hung so we
2045          * should be safe and we need to cleanup or else we might
2046          * cause memory corruption through use-after-free.
2047          */
2048
2049         i915_gem_object_finish_gtt(obj);
2050
2051         /* Move the object to the CPU domain to ensure that
2052          * any possible CPU writes while it's not in the GTT
2053          * are flushed when we go to remap it.
2054          */
2055         if (ret == 0)
2056                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2057         if (ret == -ERESTARTSYS)
2058                 return ret;
2059         if (ret) {
2060                 /* In the event of a disaster, abandon all caches and
2061                  * hope for the best.
2062                  */
2063                 i915_gem_clflush_object(obj);
2064                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2065         }
2066
2067         /* release the fence reg _after_ flushing */
2068         ret = i915_gem_object_put_fence(obj);
2069         if (ret)
2070                 return ret;
2071
2072         trace_i915_gem_object_unbind(obj);
2073
2074         if (obj->has_global_gtt_mapping)
2075                 i915_gem_gtt_unbind_object(obj);
2076         if (obj->has_aliasing_ppgtt_mapping) {
2077                 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
2078                 obj->has_aliasing_ppgtt_mapping = 0;
2079         }
2080         i915_gem_gtt_finish_object(obj);
2081
2082         i915_gem_object_put_pages_gtt(obj);
2083
2084         list_del_init(&obj->gtt_list);
2085         list_del_init(&obj->mm_list);
2086         /* Avoid an unnecessary call to unbind on rebind. */
2087         obj->map_and_fenceable = true;
2088
2089         drm_mm_put_block(obj->gtt_space);
2090         obj->gtt_space = NULL;
2091         obj->gtt_offset = 0;
2092
2093         if (i915_gem_object_is_purgeable(obj))
2094                 i915_gem_object_truncate(obj);
2095
2096         return ret;
2097 }
2098
2099 int
2100 i915_gem_flush_ring(struct intel_ring_buffer *ring,
2101                     uint32_t invalidate_domains,
2102                     uint32_t flush_domains)
2103 {
2104         int ret;
2105
2106         if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
2107                 return 0;
2108
2109         trace_i915_gem_ring_flush(ring, invalidate_domains, flush_domains);
2110
2111         ret = ring->flush(ring, invalidate_domains, flush_domains);
2112         if (ret)
2113                 return ret;
2114
2115         if (flush_domains & I915_GEM_GPU_DOMAINS)
2116                 i915_gem_process_flushing_list(ring, flush_domains);
2117
2118         return 0;
2119 }
2120
2121 static int i915_ring_idle(struct intel_ring_buffer *ring)
2122 {
2123         int ret;
2124
2125         if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
2126                 return 0;
2127
2128         if (!list_empty(&ring->gpu_write_list)) {
2129                 ret = i915_gem_flush_ring(ring,
2130                                     I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2131                 if (ret)
2132                         return ret;
2133         }
2134
2135         return i915_wait_request(ring, i915_gem_next_request_seqno(ring));
2136 }
2137
2138 int i915_gpu_idle(struct drm_device *dev)
2139 {
2140         drm_i915_private_t *dev_priv = dev->dev_private;
2141         struct intel_ring_buffer *ring;
2142         int ret, i;
2143
2144         /* Flush everything onto the inactive list. */
2145         for_each_ring(ring, dev_priv, i) {
2146                 ret = i915_ring_idle(ring);
2147                 if (ret)
2148                         return ret;
2149
2150                 /* Is the device fubar? */
2151                 if (WARN_ON(!list_empty(&ring->gpu_write_list)))
2152                         return -EBUSY;
2153         }
2154
2155         return 0;
2156 }
2157
2158 static void sandybridge_write_fence_reg(struct drm_device *dev, int reg,
2159                                         struct drm_i915_gem_object *obj)
2160 {
2161         drm_i915_private_t *dev_priv = dev->dev_private;
2162         uint64_t val;
2163
2164         if (obj) {
2165                 u32 size = obj->gtt_space->size;
2166
2167                 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2168                                  0xfffff000) << 32;
2169                 val |= obj->gtt_offset & 0xfffff000;
2170                 val |= (uint64_t)((obj->stride / 128) - 1) <<
2171                         SANDYBRIDGE_FENCE_PITCH_SHIFT;
2172
2173                 if (obj->tiling_mode == I915_TILING_Y)
2174                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2175                 val |= I965_FENCE_REG_VALID;
2176         } else
2177                 val = 0;
2178
2179         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
2180         POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
2181 }
2182
2183 static void i965_write_fence_reg(struct drm_device *dev, int reg,
2184                                  struct drm_i915_gem_object *obj)
2185 {
2186         drm_i915_private_t *dev_priv = dev->dev_private;
2187         uint64_t val;
2188
2189         if (obj) {
2190                 u32 size = obj->gtt_space->size;
2191
2192                 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2193                                  0xfffff000) << 32;
2194                 val |= obj->gtt_offset & 0xfffff000;
2195                 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2196                 if (obj->tiling_mode == I915_TILING_Y)
2197                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2198                 val |= I965_FENCE_REG_VALID;
2199         } else
2200                 val = 0;
2201
2202         I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
2203         POSTING_READ(FENCE_REG_965_0 + reg * 8);
2204 }
2205
2206 static void i915_write_fence_reg(struct drm_device *dev, int reg,
2207                                  struct drm_i915_gem_object *obj)
2208 {
2209         drm_i915_private_t *dev_priv = dev->dev_private;
2210         u32 val;
2211
2212         if (obj) {
2213                 u32 size = obj->gtt_space->size;
2214                 int pitch_val;
2215                 int tile_width;
2216
2217                 WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
2218                      (size & -size) != size ||
2219                      (obj->gtt_offset & (size - 1)),
2220                      "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2221                      obj->gtt_offset, obj->map_and_fenceable, size);
2222
2223                 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
2224                         tile_width = 128;
2225                 else
2226                         tile_width = 512;
2227
2228                 /* Note: pitch better be a power of two tile widths */
2229                 pitch_val = obj->stride / tile_width;
2230                 pitch_val = ffs(pitch_val) - 1;
2231
2232                 val = obj->gtt_offset;
2233                 if (obj->tiling_mode == I915_TILING_Y)
2234                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2235                 val |= I915_FENCE_SIZE_BITS(size);
2236                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2237                 val |= I830_FENCE_REG_VALID;
2238         } else
2239                 val = 0;
2240
2241         if (reg < 8)
2242                 reg = FENCE_REG_830_0 + reg * 4;
2243         else
2244                 reg = FENCE_REG_945_8 + (reg - 8) * 4;
2245
2246         I915_WRITE(reg, val);
2247         POSTING_READ(reg);
2248 }
2249
2250 static void i830_write_fence_reg(struct drm_device *dev, int reg,
2251                                 struct drm_i915_gem_object *obj)
2252 {
2253         drm_i915_private_t *dev_priv = dev->dev_private;
2254         uint32_t val;
2255
2256         if (obj) {
2257                 u32 size = obj->gtt_space->size;
2258                 uint32_t pitch_val;
2259
2260                 WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
2261                      (size & -size) != size ||
2262                      (obj->gtt_offset & (size - 1)),
2263                      "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
2264                      obj->gtt_offset, size);
2265
2266                 pitch_val = obj->stride / 128;
2267                 pitch_val = ffs(pitch_val) - 1;
2268
2269                 val = obj->gtt_offset;
2270                 if (obj->tiling_mode == I915_TILING_Y)
2271                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2272                 val |= I830_FENCE_SIZE_BITS(size);
2273                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2274                 val |= I830_FENCE_REG_VALID;
2275         } else
2276                 val = 0;
2277
2278         I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
2279         POSTING_READ(FENCE_REG_830_0 + reg * 4);
2280 }
2281
2282 static void i915_gem_write_fence(struct drm_device *dev, int reg,
2283                                  struct drm_i915_gem_object *obj)
2284 {
2285         switch (INTEL_INFO(dev)->gen) {
2286         case 7:
2287         case 6: sandybridge_write_fence_reg(dev, reg, obj); break;
2288         case 5:
2289         case 4: i965_write_fence_reg(dev, reg, obj); break;
2290         case 3: i915_write_fence_reg(dev, reg, obj); break;
2291         case 2: i830_write_fence_reg(dev, reg, obj); break;
2292         default: break;
2293         }
2294 }
2295
2296 static inline int fence_number(struct drm_i915_private *dev_priv,
2297                                struct drm_i915_fence_reg *fence)
2298 {
2299         return fence - dev_priv->fence_regs;
2300 }
2301
2302 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
2303                                          struct drm_i915_fence_reg *fence,
2304                                          bool enable)
2305 {
2306         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2307         int reg = fence_number(dev_priv, fence);
2308
2309         i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
2310
2311         if (enable) {
2312                 obj->fence_reg = reg;
2313                 fence->obj = obj;
2314                 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
2315         } else {
2316                 obj->fence_reg = I915_FENCE_REG_NONE;
2317                 fence->obj = NULL;
2318                 list_del_init(&fence->lru_list);
2319         }
2320 }
2321
2322 static int
2323 i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
2324 {
2325         int ret;
2326
2327         if (obj->fenced_gpu_access) {
2328                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2329                         ret = i915_gem_flush_ring(obj->ring,
2330                                                   0, obj->base.write_domain);
2331                         if (ret)
2332                                 return ret;
2333                 }
2334
2335                 obj->fenced_gpu_access = false;
2336         }
2337
2338         if (obj->last_fenced_seqno) {
2339                 ret = i915_wait_request(obj->ring, obj->last_fenced_seqno);
2340                 if (ret)
2341                         return ret;
2342
2343                 obj->last_fenced_seqno = 0;
2344         }
2345
2346         /* Ensure that all CPU reads are completed before installing a fence
2347          * and all writes before removing the fence.
2348          */
2349         if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
2350                 mb();
2351
2352         return 0;
2353 }
2354
2355 int
2356 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2357 {
2358         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2359         int ret;
2360
2361         ret = i915_gem_object_flush_fence(obj);
2362         if (ret)
2363                 return ret;
2364
2365         if (obj->fence_reg == I915_FENCE_REG_NONE)
2366                 return 0;
2367
2368         i915_gem_object_update_fence(obj,
2369                                      &dev_priv->fence_regs[obj->fence_reg],
2370                                      false);
2371         i915_gem_object_fence_lost(obj);
2372
2373         return 0;
2374 }
2375
2376 static struct drm_i915_fence_reg *
2377 i915_find_fence_reg(struct drm_device *dev)
2378 {
2379         struct drm_i915_private *dev_priv = dev->dev_private;
2380         struct drm_i915_fence_reg *reg, *avail;
2381         int i;
2382
2383         /* First try to find a free reg */
2384         avail = NULL;
2385         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2386                 reg = &dev_priv->fence_regs[i];
2387                 if (!reg->obj)
2388                         return reg;
2389
2390                 if (!reg->pin_count)
2391                         avail = reg;
2392         }
2393
2394         if (avail == NULL)
2395                 return NULL;
2396
2397         /* None available, try to steal one or wait for a user to finish */
2398         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
2399                 if (reg->pin_count)
2400                         continue;
2401
2402                 return reg;
2403         }
2404
2405         return NULL;
2406 }
2407
2408 /**
2409  * i915_gem_object_get_fence - set up fencing for an object
2410  * @obj: object to map through a fence reg
2411  *
2412  * When mapping objects through the GTT, userspace wants to be able to write
2413  * to them without having to worry about swizzling if the object is tiled.
2414  * This function walks the fence regs looking for a free one for @obj,
2415  * stealing one if it can't find any.
2416  *
2417  * It then sets up the reg based on the object's properties: address, pitch
2418  * and tiling format.
2419  *
2420  * For an untiled surface, this removes any existing fence.
2421  */
2422 int
2423 i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
2424 {
2425         struct drm_device *dev = obj->base.dev;
2426         struct drm_i915_private *dev_priv = dev->dev_private;
2427         bool enable = obj->tiling_mode != I915_TILING_NONE;
2428         struct drm_i915_fence_reg *reg;
2429         int ret;
2430
2431         /* Have we updated the tiling parameters upon the object and so
2432          * will need to serialise the write to the associated fence register?
2433          */
2434         if (obj->fence_dirty) {
2435                 ret = i915_gem_object_flush_fence(obj);
2436                 if (ret)
2437                         return ret;
2438         }
2439
2440         /* Just update our place in the LRU if our fence is getting reused. */
2441         if (obj->fence_reg != I915_FENCE_REG_NONE) {
2442                 reg = &dev_priv->fence_regs[obj->fence_reg];
2443                 if (!obj->fence_dirty) {
2444                         list_move_tail(&reg->lru_list,
2445                                        &dev_priv->mm.fence_list);
2446                         return 0;
2447                 }
2448         } else if (enable) {
2449                 reg = i915_find_fence_reg(dev);
2450                 if (reg == NULL)
2451                         return -EDEADLK;
2452
2453                 if (reg->obj) {
2454                         struct drm_i915_gem_object *old = reg->obj;
2455
2456                         ret = i915_gem_object_flush_fence(old);
2457                         if (ret)
2458                                 return ret;
2459
2460                         i915_gem_object_fence_lost(old);
2461                 }
2462         } else
2463                 return 0;
2464
2465         i915_gem_object_update_fence(obj, reg, enable);
2466         obj->fence_dirty = false;
2467
2468         return 0;
2469 }
2470
2471 /**
2472  * Finds free space in the GTT aperture and binds the object there.
2473  */
2474 static int
2475 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
2476                             unsigned alignment,
2477                             bool map_and_fenceable)
2478 {
2479         struct drm_device *dev = obj->base.dev;
2480         drm_i915_private_t *dev_priv = dev->dev_private;
2481         struct drm_mm_node *free_space;
2482         gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
2483         u32 size, fence_size, fence_alignment, unfenced_alignment;
2484         bool mappable, fenceable;
2485         int ret;
2486
2487         if (obj->madv != I915_MADV_WILLNEED) {
2488                 DRM_ERROR("Attempting to bind a purgeable object\n");
2489                 return -EINVAL;
2490         }
2491
2492         fence_size = i915_gem_get_gtt_size(dev,
2493                                            obj->base.size,
2494                                            obj->tiling_mode);
2495         fence_alignment = i915_gem_get_gtt_alignment(dev,
2496                                                      obj->base.size,
2497                                                      obj->tiling_mode);
2498         unfenced_alignment =
2499                 i915_gem_get_unfenced_gtt_alignment(dev,
2500                                                     obj->base.size,
2501                                                     obj->tiling_mode);
2502
2503         if (alignment == 0)
2504                 alignment = map_and_fenceable ? fence_alignment :
2505                                                 unfenced_alignment;
2506         if (map_and_fenceable && alignment & (fence_alignment - 1)) {
2507                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2508                 return -EINVAL;
2509         }
2510
2511         size = map_and_fenceable ? fence_size : obj->base.size;
2512
2513         /* If the object is bigger than the entire aperture, reject it early
2514          * before evicting everything in a vain attempt to find space.
2515          */
2516         if (obj->base.size >
2517             (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
2518                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2519                 return -E2BIG;
2520         }
2521
2522  search_free:
2523         if (map_and_fenceable)
2524                 free_space =
2525                         drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
2526                                                     size, alignment, 0,
2527                                                     dev_priv->mm.gtt_mappable_end,
2528                                                     0);
2529         else
2530                 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2531                                                 size, alignment, 0);
2532
2533         if (free_space != NULL) {
2534                 if (map_and_fenceable)
2535                         obj->gtt_space =
2536                                 drm_mm_get_block_range_generic(free_space,
2537                                                                size, alignment, 0,
2538                                                                dev_priv->mm.gtt_mappable_end,
2539                                                                0);
2540                 else
2541                         obj->gtt_space =
2542                                 drm_mm_get_block(free_space, size, alignment);
2543         }
2544         if (obj->gtt_space == NULL) {
2545                 /* If the gtt is empty and we're still having trouble
2546                  * fitting our object in, we're out of memory.
2547                  */
2548                 ret = i915_gem_evict_something(dev, size, alignment,
2549                                                map_and_fenceable);
2550                 if (ret)
2551                         return ret;
2552
2553                 goto search_free;
2554         }
2555
2556         ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
2557         if (ret) {
2558                 drm_mm_put_block(obj->gtt_space);
2559                 obj->gtt_space = NULL;
2560
2561                 if (ret == -ENOMEM) {
2562                         /* first try to reclaim some memory by clearing the GTT */
2563                         ret = i915_gem_evict_everything(dev, false);
2564                         if (ret) {
2565                                 /* now try to shrink everyone else */
2566                                 if (gfpmask) {
2567                                         gfpmask = 0;
2568                                         goto search_free;
2569                                 }
2570
2571                                 return -ENOMEM;
2572                         }
2573
2574                         goto search_free;
2575                 }
2576
2577                 return ret;
2578         }
2579
2580         ret = i915_gem_gtt_prepare_object(obj);
2581         if (ret) {
2582                 i915_gem_object_put_pages_gtt(obj);
2583                 drm_mm_put_block(obj->gtt_space);
2584                 obj->gtt_space = NULL;
2585
2586                 if (i915_gem_evict_everything(dev, false))
2587                         return ret;
2588
2589                 goto search_free;
2590         }
2591
2592         if (!dev_priv->mm.aliasing_ppgtt)
2593                 i915_gem_gtt_bind_object(obj, obj->cache_level);
2594
2595         list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
2596         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2597
2598         /* Assert that the object is not currently in any GPU domain. As it
2599          * wasn't in the GTT, there shouldn't be any way it could have been in
2600          * a GPU cache
2601          */
2602         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2603         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2604
2605         obj->gtt_offset = obj->gtt_space->start;
2606
2607         fenceable =
2608                 obj->gtt_space->size == fence_size &&
2609                 (obj->gtt_space->start & (fence_alignment - 1)) == 0;
2610
2611         mappable =
2612                 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
2613
2614         obj->map_and_fenceable = mappable && fenceable;
2615
2616         trace_i915_gem_object_bind(obj, map_and_fenceable);
2617         return 0;
2618 }
2619
2620 void
2621 i915_gem_clflush_object(struct drm_i915_gem_object *obj)
2622 {
2623         /* If we don't have a page list set up, then we're not pinned
2624          * to GPU, and we can ignore the cache flush because it'll happen
2625          * again at bind time.
2626          */
2627         if (obj->pages == NULL)
2628                 return;
2629
2630         /* If the GPU is snooping the contents of the CPU cache,
2631          * we do not need to manually clear the CPU cache lines.  However,
2632          * the caches are only snooped when the render cache is
2633          * flushed/invalidated.  As we always have to emit invalidations
2634          * and flushes when moving into and out of the RENDER domain, correct
2635          * snooping behaviour occurs naturally as the result of our domain
2636          * tracking.
2637          */
2638         if (obj->cache_level != I915_CACHE_NONE)
2639                 return;
2640
2641         trace_i915_gem_object_clflush(obj);
2642
2643         drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
2644 }
2645
2646 /** Flushes any GPU write domain for the object if it's dirty. */
2647 static int
2648 i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
2649 {
2650         if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
2651                 return 0;
2652
2653         /* Queue the GPU write cache flushing we need. */
2654         return i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
2655 }
2656
2657 /** Flushes the GTT write domain for the object if it's dirty. */
2658 static void
2659 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
2660 {
2661         uint32_t old_write_domain;
2662
2663         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
2664                 return;
2665
2666         /* No actual flushing is required for the GTT write domain.  Writes
2667          * to it immediately go to main memory as far as we know, so there's
2668          * no chipset flush.  It also doesn't land in render cache.
2669          *
2670          * However, we do have to enforce the order so that all writes through
2671          * the GTT land before any writes to the device, such as updates to
2672          * the GATT itself.
2673          */
2674         wmb();
2675
2676         old_write_domain = obj->base.write_domain;
2677         obj->base.write_domain = 0;
2678
2679         trace_i915_gem_object_change_domain(obj,
2680                                             obj->base.read_domains,
2681                                             old_write_domain);
2682 }
2683
2684 /** Flushes the CPU write domain for the object if it's dirty. */
2685 static void
2686 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
2687 {
2688         uint32_t old_write_domain;
2689
2690         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
2691                 return;
2692
2693         i915_gem_clflush_object(obj);
2694         intel_gtt_chipset_flush();
2695         old_write_domain = obj->base.write_domain;
2696         obj->base.write_domain = 0;
2697
2698         trace_i915_gem_object_change_domain(obj,
2699                                             obj->base.read_domains,
2700                                             old_write_domain);
2701 }
2702
2703 /**
2704  * Moves a single object to the GTT read, and possibly write domain.
2705  *
2706  * This function returns when the move is complete, including waiting on
2707  * flushes to occur.
2708  */
2709 int
2710 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
2711 {
2712         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2713         uint32_t old_write_domain, old_read_domains;
2714         int ret;
2715
2716         /* Not valid to be called on unbound objects. */
2717         if (obj->gtt_space == NULL)
2718                 return -EINVAL;
2719
2720         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
2721                 return 0;
2722
2723         ret = i915_gem_object_flush_gpu_write_domain(obj);
2724         if (ret)
2725                 return ret;
2726
2727         if (obj->pending_gpu_write || write) {
2728                 ret = i915_gem_object_wait_rendering(obj);
2729                 if (ret)
2730                         return ret;
2731         }
2732
2733         i915_gem_object_flush_cpu_write_domain(obj);
2734
2735         old_write_domain = obj->base.write_domain;
2736         old_read_domains = obj->base.read_domains;
2737
2738         /* It should now be out of any other write domains, and we can update
2739          * the domain values for our changes.
2740          */
2741         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2742         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2743         if (write) {
2744                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
2745                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
2746                 obj->dirty = 1;
2747         }
2748
2749         trace_i915_gem_object_change_domain(obj,
2750                                             old_read_domains,
2751                                             old_write_domain);
2752
2753         /* And bump the LRU for this access */
2754         if (i915_gem_object_is_inactive(obj))
2755                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2756
2757         return 0;
2758 }
2759
2760 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
2761                                     enum i915_cache_level cache_level)
2762 {
2763         struct drm_device *dev = obj->base.dev;
2764         drm_i915_private_t *dev_priv = dev->dev_private;
2765         int ret;
2766
2767         if (obj->cache_level == cache_level)
2768                 return 0;
2769
2770         if (obj->pin_count) {
2771                 DRM_DEBUG("can not change the cache level of pinned objects\n");
2772                 return -EBUSY;
2773         }
2774
2775         if (obj->gtt_space) {
2776                 ret = i915_gem_object_finish_gpu(obj);
2777                 if (ret)
2778                         return ret;
2779
2780                 i915_gem_object_finish_gtt(obj);
2781
2782                 /* Before SandyBridge, you could not use tiling or fence
2783                  * registers with snooped memory, so relinquish any fences
2784                  * currently pointing to our region in the aperture.
2785                  */
2786                 if (INTEL_INFO(obj->base.dev)->gen < 6) {
2787                         ret = i915_gem_object_put_fence(obj);
2788                         if (ret)
2789                                 return ret;
2790                 }
2791
2792                 if (obj->has_global_gtt_mapping)
2793                         i915_gem_gtt_bind_object(obj, cache_level);
2794                 if (obj->has_aliasing_ppgtt_mapping)
2795                         i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
2796                                                obj, cache_level);
2797         }
2798
2799         if (cache_level == I915_CACHE_NONE) {
2800                 u32 old_read_domains, old_write_domain;
2801
2802                 /* If we're coming from LLC cached, then we haven't
2803                  * actually been tracking whether the data is in the
2804                  * CPU cache or not, since we only allow one bit set
2805                  * in obj->write_domain and have been skipping the clflushes.
2806                  * Just set it to the CPU cache for now.
2807                  */
2808                 WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
2809                 WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
2810
2811                 old_read_domains = obj->base.read_domains;
2812                 old_write_domain = obj->base.write_domain;
2813
2814                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2815                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2816
2817                 trace_i915_gem_object_change_domain(obj,
2818                                                     old_read_domains,
2819                                                     old_write_domain);
2820         }
2821
2822         obj->cache_level = cache_level;
2823         return 0;
2824 }
2825
2826 /*
2827  * Prepare buffer for display plane (scanout, cursors, etc).
2828  * Can be called from an uninterruptible phase (modesetting) and allows
2829  * any flushes to be pipelined (for pageflips).
2830  */
2831 int
2832 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
2833                                      u32 alignment,
2834                                      struct intel_ring_buffer *pipelined)
2835 {
2836         u32 old_read_domains, old_write_domain;
2837         int ret;
2838
2839         ret = i915_gem_object_flush_gpu_write_domain(obj);
2840         if (ret)
2841                 return ret;
2842
2843         if (pipelined != obj->ring) {
2844                 ret = i915_gem_object_sync(obj, pipelined);
2845                 if (ret)
2846                         return ret;
2847         }
2848
2849         /* The display engine is not coherent with the LLC cache on gen6.  As
2850          * a result, we make sure that the pinning that is about to occur is
2851          * done with uncached PTEs. This is lowest common denominator for all
2852          * chipsets.
2853          *
2854          * However for gen6+, we could do better by using the GFDT bit instead
2855          * of uncaching, which would allow us to flush all the LLC-cached data
2856          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
2857          */
2858         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
2859         if (ret)
2860                 return ret;
2861
2862         /* As the user may map the buffer once pinned in the display plane
2863          * (e.g. libkms for the bootup splash), we have to ensure that we
2864          * always use map_and_fenceable for all scanout buffers.
2865          */
2866         ret = i915_gem_object_pin(obj, alignment, true);
2867         if (ret)
2868                 return ret;
2869
2870         i915_gem_object_flush_cpu_write_domain(obj);
2871
2872         old_write_domain = obj->base.write_domain;
2873         old_read_domains = obj->base.read_domains;
2874
2875         /* It should now be out of any other write domains, and we can update
2876          * the domain values for our changes.
2877          */
2878         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2879         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2880
2881         trace_i915_gem_object_change_domain(obj,
2882                                             old_read_domains,
2883                                             old_write_domain);
2884
2885         return 0;
2886 }
2887
2888 int
2889 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
2890 {
2891         int ret;
2892
2893         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
2894                 return 0;
2895
2896         if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2897                 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
2898                 if (ret)
2899                         return ret;
2900         }
2901
2902         ret = i915_gem_object_wait_rendering(obj);
2903         if (ret)
2904                 return ret;
2905
2906         /* Ensure that we invalidate the GPU's caches and TLBs. */
2907         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
2908         return 0;
2909 }
2910
2911 /**
2912  * Moves a single object to the CPU read, and possibly write domain.
2913  *
2914  * This function returns when the move is complete, including waiting on
2915  * flushes to occur.
2916  */
2917 int
2918 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
2919 {
2920         uint32_t old_write_domain, old_read_domains;
2921         int ret;
2922
2923         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
2924                 return 0;
2925
2926         ret = i915_gem_object_flush_gpu_write_domain(obj);
2927         if (ret)
2928                 return ret;
2929
2930         if (write || obj->pending_gpu_write) {
2931                 ret = i915_gem_object_wait_rendering(obj);
2932                 if (ret)
2933                         return ret;
2934         }
2935
2936         i915_gem_object_flush_gtt_write_domain(obj);
2937
2938         old_write_domain = obj->base.write_domain;
2939         old_read_domains = obj->base.read_domains;
2940
2941         /* Flush the CPU cache if it's still invalid. */
2942         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2943                 i915_gem_clflush_object(obj);
2944
2945                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
2946         }
2947
2948         /* It should now be out of any other write domains, and we can update
2949          * the domain values for our changes.
2950          */
2951         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2952
2953         /* If we're writing through the CPU, then the GPU read domains will
2954          * need to be invalidated at next use.
2955          */
2956         if (write) {
2957                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2958                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2959         }
2960
2961         trace_i915_gem_object_change_domain(obj,
2962                                             old_read_domains,
2963                                             old_write_domain);
2964
2965         return 0;
2966 }
2967
2968 /* Throttle our rendering by waiting until the ring has completed our requests
2969  * emitted over 20 msec ago.
2970  *
2971  * Note that if we were to use the current jiffies each time around the loop,
2972  * we wouldn't escape the function with any frames outstanding if the time to
2973  * render a frame was over 20ms.
2974  *
2975  * This should get us reasonable parallelism between CPU and GPU but also
2976  * relatively low latency when blocking on a particular request to finish.
2977  */
2978 static int
2979 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
2980 {
2981         struct drm_i915_private *dev_priv = dev->dev_private;
2982         struct drm_i915_file_private *file_priv = file->driver_priv;
2983         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
2984         struct drm_i915_gem_request *request;
2985         struct intel_ring_buffer *ring = NULL;
2986         u32 seqno = 0;
2987         int ret;
2988
2989         if (atomic_read(&dev_priv->mm.wedged))
2990                 return -EIO;
2991
2992         spin_lock(&file_priv->mm.lock);
2993         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
2994                 if (time_after_eq(request->emitted_jiffies, recent_enough))
2995                         break;
2996
2997                 ring = request->ring;
2998                 seqno = request->seqno;
2999         }
3000         spin_unlock(&file_priv->mm.lock);
3001
3002         if (seqno == 0)
3003                 return 0;
3004
3005         ret = __wait_seqno(ring, seqno, true);
3006         if (ret == 0)
3007                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3008
3009         return ret;
3010 }
3011
3012 int
3013 i915_gem_object_pin(struct drm_i915_gem_object *obj,
3014                     uint32_t alignment,
3015                     bool map_and_fenceable)
3016 {
3017         int ret;
3018
3019         BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
3020
3021         if (obj->gtt_space != NULL) {
3022                 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
3023                     (map_and_fenceable && !obj->map_and_fenceable)) {
3024                         WARN(obj->pin_count,
3025                              "bo is already pinned with incorrect alignment:"
3026                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
3027                              " obj->map_and_fenceable=%d\n",
3028                              obj->gtt_offset, alignment,
3029                              map_and_fenceable,
3030                              obj->map_and_fenceable);
3031                         ret = i915_gem_object_unbind(obj);
3032                         if (ret)
3033                                 return ret;
3034                 }
3035         }
3036
3037         if (obj->gtt_space == NULL) {
3038                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
3039                                                   map_and_fenceable);
3040                 if (ret)
3041                         return ret;
3042         }
3043
3044         if (!obj->has_global_gtt_mapping && map_and_fenceable)
3045                 i915_gem_gtt_bind_object(obj, obj->cache_level);
3046
3047         obj->pin_count++;
3048         obj->pin_mappable |= map_and_fenceable;
3049
3050         return 0;
3051 }
3052
3053 void
3054 i915_gem_object_unpin(struct drm_i915_gem_object *obj)
3055 {
3056         BUG_ON(obj->pin_count == 0);
3057         BUG_ON(obj->gtt_space == NULL);
3058
3059         if (--obj->pin_count == 0)
3060                 obj->pin_mappable = false;
3061 }
3062
3063 int
3064 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
3065                    struct drm_file *file)
3066 {
3067         struct drm_i915_gem_pin *args = data;
3068         struct drm_i915_gem_object *obj;
3069         int ret;
3070
3071         ret = i915_mutex_lock_interruptible(dev);
3072         if (ret)
3073                 return ret;
3074
3075         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3076         if (&obj->base == NULL) {
3077                 ret = -ENOENT;
3078                 goto unlock;
3079         }
3080
3081         if (obj->madv != I915_MADV_WILLNEED) {
3082                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
3083                 ret = -EINVAL;
3084                 goto out;
3085         }
3086
3087         if (obj->pin_filp != NULL && obj->pin_filp != file) {
3088                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
3089                           args->handle);
3090                 ret = -EINVAL;
3091                 goto out;
3092         }
3093
3094         obj->user_pin_count++;
3095         obj->pin_filp = file;
3096         if (obj->user_pin_count == 1) {
3097                 ret = i915_gem_object_pin(obj, args->alignment, true);
3098                 if (ret)
3099                         goto out;
3100         }
3101
3102         /* XXX - flush the CPU caches for pinned objects
3103          * as the X server doesn't manage domains yet
3104          */
3105         i915_gem_object_flush_cpu_write_domain(obj);
3106         args->offset = obj->gtt_offset;
3107 out:
3108         drm_gem_object_unreference(&obj->base);
3109 unlock:
3110         mutex_unlock(&dev->struct_mutex);
3111         return ret;
3112 }
3113
3114 int
3115 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
3116                      struct drm_file *file)
3117 {
3118         struct drm_i915_gem_pin *args = data;
3119         struct drm_i915_gem_object *obj;
3120         int ret;
3121
3122         ret = i915_mutex_lock_interruptible(dev);
3123         if (ret)
3124                 return ret;
3125
3126         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3127         if (&obj->base == NULL) {
3128                 ret = -ENOENT;
3129                 goto unlock;
3130         }
3131
3132         if (obj->pin_filp != file) {
3133                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
3134                           args->handle);
3135                 ret = -EINVAL;
3136                 goto out;
3137         }
3138         obj->user_pin_count--;
3139         if (obj->user_pin_count == 0) {
3140                 obj->pin_filp = NULL;
3141                 i915_gem_object_unpin(obj);
3142         }
3143
3144 out:
3145         drm_gem_object_unreference(&obj->base);
3146 unlock:
3147         mutex_unlock(&dev->struct_mutex);
3148         return ret;
3149 }
3150
3151 int
3152 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3153                     struct drm_file *file)
3154 {
3155         struct drm_i915_gem_busy *args = data;
3156         struct drm_i915_gem_object *obj;
3157         int ret;
3158
3159         ret = i915_mutex_lock_interruptible(dev);
3160         if (ret)
3161                 return ret;
3162
3163         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3164         if (&obj->base == NULL) {
3165                 ret = -ENOENT;
3166                 goto unlock;
3167         }
3168
3169         /* Count all active objects as busy, even if they are currently not used
3170          * by the gpu. Users of this interface expect objects to eventually
3171          * become non-busy without any further actions, therefore emit any
3172          * necessary flushes here.
3173          */
3174         args->busy = obj->active;
3175         if (args->busy) {
3176                 /* Unconditionally flush objects, even when the gpu still uses this
3177                  * object. Userspace calling this function indicates that it wants to
3178                  * use this buffer rather sooner than later, so issuing the required
3179                  * flush earlier is beneficial.
3180                  */
3181                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
3182                         ret = i915_gem_flush_ring(obj->ring,
3183                                                   0, obj->base.write_domain);
3184                 } else {
3185                         ret = i915_gem_check_olr(obj->ring,
3186                                                  obj->last_rendering_seqno);
3187                 }
3188
3189                 /* Update the active list for the hardware's current position.
3190                  * Otherwise this only updates on a delayed timer or when irqs
3191                  * are actually unmasked, and our working set ends up being
3192                  * larger than required.
3193                  */
3194                 i915_gem_retire_requests_ring(obj->ring);
3195
3196                 args->busy = obj->active;
3197         }
3198
3199         drm_gem_object_unreference(&obj->base);
3200 unlock:
3201         mutex_unlock(&dev->struct_mutex);
3202         return ret;
3203 }
3204
3205 int
3206 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
3207                         struct drm_file *file_priv)
3208 {
3209         return i915_gem_ring_throttle(dev, file_priv);
3210 }
3211
3212 int
3213 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
3214                        struct drm_file *file_priv)
3215 {
3216         struct drm_i915_gem_madvise *args = data;
3217         struct drm_i915_gem_object *obj;
3218         int ret;
3219
3220         switch (args->madv) {
3221         case I915_MADV_DONTNEED:
3222         case I915_MADV_WILLNEED:
3223             break;
3224         default:
3225             return -EINVAL;
3226         }
3227
3228         ret = i915_mutex_lock_interruptible(dev);
3229         if (ret)
3230                 return ret;
3231
3232         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
3233         if (&obj->base == NULL) {
3234                 ret = -ENOENT;
3235                 goto unlock;
3236         }
3237
3238         if (obj->pin_count) {
3239                 ret = -EINVAL;
3240                 goto out;
3241         }
3242
3243         if (obj->madv != __I915_MADV_PURGED)
3244                 obj->madv = args->madv;
3245
3246         /* if the object is no longer bound, discard its backing storage */
3247         if (i915_gem_object_is_purgeable(obj) &&
3248             obj->gtt_space == NULL)
3249                 i915_gem_object_truncate(obj);
3250
3251         args->retained = obj->madv != __I915_MADV_PURGED;
3252
3253 out:
3254         drm_gem_object_unreference(&obj->base);
3255 unlock:
3256         mutex_unlock(&dev->struct_mutex);
3257         return ret;
3258 }
3259
3260 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
3261                                                   size_t size)
3262 {
3263         struct drm_i915_private *dev_priv = dev->dev_private;
3264         struct drm_i915_gem_object *obj;
3265         struct address_space *mapping;
3266         u32 mask;
3267
3268         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
3269         if (obj == NULL)
3270                 return NULL;
3271
3272         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
3273                 kfree(obj);
3274                 return NULL;
3275         }
3276
3277         mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
3278         if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
3279                 /* 965gm cannot relocate objects above 4GiB. */
3280                 mask &= ~__GFP_HIGHMEM;
3281                 mask |= __GFP_DMA32;
3282         }
3283
3284         mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
3285         mapping_set_gfp_mask(mapping, mask);
3286
3287         i915_gem_info_add_obj(dev_priv, size);
3288
3289         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3290         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3291
3292         if (HAS_LLC(dev)) {
3293                 /* On some devices, we can have the GPU use the LLC (the CPU
3294                  * cache) for about a 10% performance improvement
3295                  * compared to uncached.  Graphics requests other than
3296                  * display scanout are coherent with the CPU in
3297                  * accessing this cache.  This means in this mode we
3298                  * don't need to clflush on the CPU side, and on the
3299                  * GPU side we only need to flush internal caches to
3300                  * get data visible to the CPU.
3301                  *
3302                  * However, we maintain the display planes as UC, and so
3303                  * need to rebind when first used as such.
3304                  */
3305                 obj->cache_level = I915_CACHE_LLC;
3306         } else
3307                 obj->cache_level = I915_CACHE_NONE;
3308
3309         obj->base.driver_private = NULL;
3310         obj->fence_reg = I915_FENCE_REG_NONE;
3311         INIT_LIST_HEAD(&obj->mm_list);
3312         INIT_LIST_HEAD(&obj->gtt_list);
3313         INIT_LIST_HEAD(&obj->ring_list);
3314         INIT_LIST_HEAD(&obj->exec_list);
3315         INIT_LIST_HEAD(&obj->gpu_write_list);
3316         obj->madv = I915_MADV_WILLNEED;
3317         /* Avoid an unnecessary call to unbind on the first bind. */
3318         obj->map_and_fenceable = true;
3319
3320         return obj;
3321 }
3322
3323 int i915_gem_init_object(struct drm_gem_object *obj)
3324 {
3325         BUG();
3326
3327         return 0;
3328 }
3329
3330 void i915_gem_free_object(struct drm_gem_object *gem_obj)
3331 {
3332         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
3333         struct drm_device *dev = obj->base.dev;
3334         drm_i915_private_t *dev_priv = dev->dev_private;
3335
3336         trace_i915_gem_object_destroy(obj);
3337
3338         if (obj->phys_obj)
3339                 i915_gem_detach_phys_object(dev, obj);
3340
3341         obj->pin_count = 0;
3342         if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
3343                 bool was_interruptible;
3344
3345                 was_interruptible = dev_priv->mm.interruptible;
3346                 dev_priv->mm.interruptible = false;
3347
3348                 WARN_ON(i915_gem_object_unbind(obj));
3349
3350                 dev_priv->mm.interruptible = was_interruptible;
3351         }
3352
3353         if (obj->base.map_list.map)
3354                 drm_gem_free_mmap_offset(&obj->base);
3355
3356         drm_gem_object_release(&obj->base);
3357         i915_gem_info_remove_obj(dev_priv, obj->base.size);
3358
3359         kfree(obj->bit_17);
3360         kfree(obj);
3361 }
3362
3363 int
3364 i915_gem_idle(struct drm_device *dev)
3365 {
3366         drm_i915_private_t *dev_priv = dev->dev_private;
3367         int ret;
3368
3369         mutex_lock(&dev->struct_mutex);
3370
3371         if (dev_priv->mm.suspended) {
3372                 mutex_unlock(&dev->struct_mutex);
3373                 return 0;
3374         }
3375
3376         ret = i915_gpu_idle(dev);
3377         if (ret) {
3378                 mutex_unlock(&dev->struct_mutex);
3379                 return ret;
3380         }
3381         i915_gem_retire_requests(dev);
3382
3383         /* Under UMS, be paranoid and evict. */
3384         if (!drm_core_check_feature(dev, DRIVER_MODESET))
3385                 i915_gem_evict_everything(dev, false);
3386
3387         i915_gem_reset_fences(dev);
3388
3389         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
3390          * We need to replace this with a semaphore, or something.
3391          * And not confound mm.suspended!
3392          */
3393         dev_priv->mm.suspended = 1;
3394         del_timer_sync(&dev_priv->hangcheck_timer);
3395
3396         i915_kernel_lost_context(dev);
3397         i915_gem_cleanup_ringbuffer(dev);
3398
3399         mutex_unlock(&dev->struct_mutex);
3400
3401         /* Cancel the retire work handler, which should be idle now. */
3402         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3403
3404         return 0;
3405 }
3406
3407 void i915_gem_init_swizzling(struct drm_device *dev)
3408 {
3409         drm_i915_private_t *dev_priv = dev->dev_private;
3410
3411         if (INTEL_INFO(dev)->gen < 5 ||
3412             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
3413                 return;
3414
3415         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
3416                                  DISP_TILE_SURFACE_SWIZZLING);
3417
3418         if (IS_GEN5(dev))
3419                 return;
3420
3421         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
3422         if (IS_GEN6(dev))
3423                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
3424         else
3425                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
3426 }
3427
3428 void i915_gem_init_ppgtt(struct drm_device *dev)
3429 {
3430         drm_i915_private_t *dev_priv = dev->dev_private;
3431         uint32_t pd_offset;
3432         struct intel_ring_buffer *ring;
3433         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
3434         uint32_t __iomem *pd_addr;
3435         uint32_t pd_entry;
3436         int i;
3437
3438         if (!dev_priv->mm.aliasing_ppgtt)
3439                 return;
3440
3441
3442         pd_addr = dev_priv->mm.gtt->gtt + ppgtt->pd_offset/sizeof(uint32_t);
3443         for (i = 0; i < ppgtt->num_pd_entries; i++) {
3444                 dma_addr_t pt_addr;
3445
3446                 if (dev_priv->mm.gtt->needs_dmar)
3447                         pt_addr = ppgtt->pt_dma_addr[i];
3448                 else
3449                         pt_addr = page_to_phys(ppgtt->pt_pages[i]);
3450
3451                 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
3452                 pd_entry |= GEN6_PDE_VALID;
3453
3454                 writel(pd_entry, pd_addr + i);
3455         }
3456         readl(pd_addr);
3457
3458         pd_offset = ppgtt->pd_offset;
3459         pd_offset /= 64; /* in cachelines, */
3460         pd_offset <<= 16;
3461
3462         if (INTEL_INFO(dev)->gen == 6) {
3463                 uint32_t ecochk, gab_ctl, ecobits;
3464
3465                 ecobits = I915_READ(GAC_ECO_BITS);
3466                 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
3467
3468                 gab_ctl = I915_READ(GAB_CTL);
3469                 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
3470
3471                 ecochk = I915_READ(GAM_ECOCHK);
3472                 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
3473                                        ECOCHK_PPGTT_CACHE64B);
3474                 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
3475         } else if (INTEL_INFO(dev)->gen >= 7) {
3476                 I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B);
3477                 /* GFX_MODE is per-ring on gen7+ */
3478         }
3479
3480         for_each_ring(ring, dev_priv, i) {
3481                 if (INTEL_INFO(dev)->gen >= 7)
3482                         I915_WRITE(RING_MODE_GEN7(ring),
3483                                    _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
3484
3485                 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
3486                 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
3487         }
3488 }
3489
3490 int
3491 i915_gem_init_hw(struct drm_device *dev)
3492 {
3493         drm_i915_private_t *dev_priv = dev->dev_private;
3494         int ret;
3495
3496         i915_gem_init_swizzling(dev);
3497
3498         ret = intel_init_render_ring_buffer(dev);
3499         if (ret)
3500                 return ret;
3501
3502         if (HAS_BSD(dev)) {
3503                 ret = intel_init_bsd_ring_buffer(dev);
3504                 if (ret)
3505                         goto cleanup_render_ring;
3506         }
3507
3508         if (HAS_BLT(dev)) {
3509                 ret = intel_init_blt_ring_buffer(dev);
3510                 if (ret)
3511                         goto cleanup_bsd_ring;
3512         }
3513
3514         dev_priv->next_seqno = 1;
3515
3516         i915_gem_init_ppgtt(dev);
3517
3518         return 0;
3519
3520 cleanup_bsd_ring:
3521         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
3522 cleanup_render_ring:
3523         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
3524         return ret;
3525 }
3526
3527 static bool
3528 intel_enable_ppgtt(struct drm_device *dev)
3529 {
3530         if (i915_enable_ppgtt >= 0)
3531                 return i915_enable_ppgtt;
3532
3533 #ifdef CONFIG_INTEL_IOMMU
3534         /* Disable ppgtt on SNB if VT-d is on. */
3535         if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
3536                 return false;
3537 #endif
3538
3539         return true;
3540 }
3541
3542 int i915_gem_init(struct drm_device *dev)
3543 {
3544         struct drm_i915_private *dev_priv = dev->dev_private;
3545         unsigned long gtt_size, mappable_size;
3546         int ret;
3547
3548         gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
3549         mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
3550
3551         mutex_lock(&dev->struct_mutex);
3552         if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
3553                 /* PPGTT pdes are stolen from global gtt ptes, so shrink the
3554                  * aperture accordingly when using aliasing ppgtt. */
3555                 gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
3556
3557                 i915_gem_init_global_gtt(dev, 0, mappable_size, gtt_size);
3558
3559                 ret = i915_gem_init_aliasing_ppgtt(dev);
3560                 if (ret) {
3561                         mutex_unlock(&dev->struct_mutex);
3562                         return ret;
3563                 }
3564         } else {
3565                 /* Let GEM Manage all of the aperture.
3566                  *
3567                  * However, leave one page at the end still bound to the scratch
3568                  * page.  There are a number of places where the hardware
3569                  * apparently prefetches past the end of the object, and we've
3570                  * seen multiple hangs with the GPU head pointer stuck in a
3571                  * batchbuffer bound at the last page of the aperture.  One page
3572                  * should be enough to keep any prefetching inside of the
3573                  * aperture.
3574                  */
3575                 i915_gem_init_global_gtt(dev, 0, mappable_size,
3576                                          gtt_size);
3577         }
3578
3579         ret = i915_gem_init_hw(dev);
3580         mutex_unlock(&dev->struct_mutex);
3581         if (ret) {
3582                 i915_gem_cleanup_aliasing_ppgtt(dev);
3583                 return ret;
3584         }
3585
3586         /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */
3587         if (!drm_core_check_feature(dev, DRIVER_MODESET))
3588                 dev_priv->dri1.allow_batchbuffer = 1;
3589         return 0;
3590 }
3591
3592 void
3593 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
3594 {
3595         drm_i915_private_t *dev_priv = dev->dev_private;
3596         struct intel_ring_buffer *ring;
3597         int i;
3598
3599         for_each_ring(ring, dev_priv, i)
3600                 intel_cleanup_ring_buffer(ring);
3601 }
3602
3603 int
3604 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
3605                        struct drm_file *file_priv)
3606 {
3607         drm_i915_private_t *dev_priv = dev->dev_private;
3608         int ret;
3609
3610         if (drm_core_check_feature(dev, DRIVER_MODESET))
3611                 return 0;
3612
3613         if (atomic_read(&dev_priv->mm.wedged)) {
3614                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
3615                 atomic_set(&dev_priv->mm.wedged, 0);
3616         }
3617
3618         mutex_lock(&dev->struct_mutex);
3619         dev_priv->mm.suspended = 0;
3620
3621         ret = i915_gem_init_hw(dev);
3622         if (ret != 0) {
3623                 mutex_unlock(&dev->struct_mutex);
3624                 return ret;
3625         }
3626
3627         BUG_ON(!list_empty(&dev_priv->mm.active_list));
3628         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
3629         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
3630         mutex_unlock(&dev->struct_mutex);
3631
3632         ret = drm_irq_install(dev);
3633         if (ret)
3634                 goto cleanup_ringbuffer;
3635
3636         return 0;
3637
3638 cleanup_ringbuffer:
3639         mutex_lock(&dev->struct_mutex);
3640         i915_gem_cleanup_ringbuffer(dev);
3641         dev_priv->mm.suspended = 1;
3642         mutex_unlock(&dev->struct_mutex);
3643
3644         return ret;
3645 }
3646
3647 int
3648 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
3649                        struct drm_file *file_priv)
3650 {
3651         if (drm_core_check_feature(dev, DRIVER_MODESET))
3652                 return 0;
3653
3654         drm_irq_uninstall(dev);
3655         return i915_gem_idle(dev);
3656 }
3657
3658 void
3659 i915_gem_lastclose(struct drm_device *dev)
3660 {
3661         int ret;
3662
3663         if (drm_core_check_feature(dev, DRIVER_MODESET))
3664                 return;
3665
3666         ret = i915_gem_idle(dev);
3667         if (ret)
3668                 DRM_ERROR("failed to idle hardware: %d\n", ret);
3669 }
3670
3671 static void
3672 init_ring_lists(struct intel_ring_buffer *ring)
3673 {
3674         INIT_LIST_HEAD(&ring->active_list);
3675         INIT_LIST_HEAD(&ring->request_list);
3676         INIT_LIST_HEAD(&ring->gpu_write_list);
3677 }
3678
3679 void
3680 i915_gem_load(struct drm_device *dev)
3681 {
3682         int i;
3683         drm_i915_private_t *dev_priv = dev->dev_private;
3684
3685         INIT_LIST_HEAD(&dev_priv->mm.active_list);
3686         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
3687         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
3688         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
3689         INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
3690         for (i = 0; i < I915_NUM_RINGS; i++)
3691                 init_ring_lists(&dev_priv->ring[i]);
3692         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
3693                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
3694         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
3695                           i915_gem_retire_work_handler);
3696         init_completion(&dev_priv->error_completion);
3697
3698         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
3699         if (IS_GEN3(dev)) {
3700                 I915_WRITE(MI_ARB_STATE,
3701                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
3702         }
3703
3704         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
3705
3706         /* Old X drivers will take 0-2 for front, back, depth buffers */
3707         if (!drm_core_check_feature(dev, DRIVER_MODESET))
3708                 dev_priv->fence_reg_start = 3;
3709
3710         if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
3711                 dev_priv->num_fence_regs = 16;
3712         else
3713                 dev_priv->num_fence_regs = 8;
3714
3715         /* Initialize fence registers to zero */
3716         i915_gem_reset_fences(dev);
3717
3718         i915_gem_detect_bit_6_swizzle(dev);
3719         init_waitqueue_head(&dev_priv->pending_flip_queue);
3720
3721         dev_priv->mm.interruptible = true;
3722
3723         dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
3724         dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
3725         register_shrinker(&dev_priv->mm.inactive_shrinker);
3726 }
3727
3728 /*
3729  * Create a physically contiguous memory object for this object
3730  * e.g. for cursor + overlay regs
3731  */
3732 static int i915_gem_init_phys_object(struct drm_device *dev,
3733                                      int id, int size, int align)
3734 {
3735         drm_i915_private_t *dev_priv = dev->dev_private;
3736         struct drm_i915_gem_phys_object *phys_obj;
3737         int ret;
3738
3739         if (dev_priv->mm.phys_objs[id - 1] || !size)
3740                 return 0;
3741
3742         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
3743         if (!phys_obj)
3744                 return -ENOMEM;
3745
3746         phys_obj->id = id;
3747
3748         phys_obj->handle = drm_pci_alloc(dev, size, align);
3749         if (!phys_obj->handle) {
3750                 ret = -ENOMEM;
3751                 goto kfree_obj;
3752         }
3753 #ifdef CONFIG_X86
3754         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
3755 #endif
3756
3757         dev_priv->mm.phys_objs[id - 1] = phys_obj;
3758
3759         return 0;
3760 kfree_obj:
3761         kfree(phys_obj);
3762         return ret;
3763 }
3764
3765 static void i915_gem_free_phys_object(struct drm_device *dev, int id)
3766 {
3767         drm_i915_private_t *dev_priv = dev->dev_private;
3768         struct drm_i915_gem_phys_object *phys_obj;
3769
3770         if (!dev_priv->mm.phys_objs[id - 1])
3771                 return;
3772
3773         phys_obj = dev_priv->mm.phys_objs[id - 1];
3774         if (phys_obj->cur_obj) {
3775                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
3776         }
3777
3778 #ifdef CONFIG_X86
3779         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
3780 #endif
3781         drm_pci_free(dev, phys_obj->handle);
3782         kfree(phys_obj);
3783         dev_priv->mm.phys_objs[id - 1] = NULL;
3784 }
3785
3786 void i915_gem_free_all_phys_object(struct drm_device *dev)
3787 {
3788         int i;
3789
3790         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
3791                 i915_gem_free_phys_object(dev, i);
3792 }
3793
3794 void i915_gem_detach_phys_object(struct drm_device *dev,
3795                                  struct drm_i915_gem_object *obj)
3796 {
3797         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
3798         char *vaddr;
3799         int i;
3800         int page_count;
3801
3802         if (!obj->phys_obj)
3803                 return;
3804         vaddr = obj->phys_obj->handle->vaddr;
3805
3806         page_count = obj->base.size / PAGE_SIZE;
3807         for (i = 0; i < page_count; i++) {
3808                 struct page *page = shmem_read_mapping_page(mapping, i);
3809                 if (!IS_ERR(page)) {
3810                         char *dst = kmap_atomic(page);
3811                         memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
3812                         kunmap_atomic(dst);
3813
3814                         drm_clflush_pages(&page, 1);
3815
3816                         set_page_dirty(page);
3817                         mark_page_accessed(page);
3818                         page_cache_release(page);
3819                 }
3820         }
3821         intel_gtt_chipset_flush();
3822
3823         obj->phys_obj->cur_obj = NULL;
3824         obj->phys_obj = NULL;
3825 }
3826
3827 int
3828 i915_gem_attach_phys_object(struct drm_device *dev,
3829                             struct drm_i915_gem_object *obj,
3830                             int id,
3831                             int align)
3832 {
3833         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
3834         drm_i915_private_t *dev_priv = dev->dev_private;
3835         int ret = 0;
3836         int page_count;
3837         int i;
3838
3839         if (id > I915_MAX_PHYS_OBJECT)
3840                 return -EINVAL;
3841
3842         if (obj->phys_obj) {
3843                 if (obj->phys_obj->id == id)
3844                         return 0;
3845                 i915_gem_detach_phys_object(dev, obj);
3846         }
3847
3848         /* create a new object */
3849         if (!dev_priv->mm.phys_objs[id - 1]) {
3850                 ret = i915_gem_init_phys_object(dev, id,
3851                                                 obj->base.size, align);
3852                 if (ret) {
3853                         DRM_ERROR("failed to init phys object %d size: %zu\n",
3854                                   id, obj->base.size);
3855                         return ret;
3856                 }
3857         }
3858
3859         /* bind to the object */
3860         obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
3861         obj->phys_obj->cur_obj = obj;
3862
3863         page_count = obj->base.size / PAGE_SIZE;
3864
3865         for (i = 0; i < page_count; i++) {
3866                 struct page *page;
3867                 char *dst, *src;
3868
3869                 page = shmem_read_mapping_page(mapping, i);
3870                 if (IS_ERR(page))
3871                         return PTR_ERR(page);
3872
3873                 src = kmap_atomic(page);
3874                 dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
3875                 memcpy(dst, src, PAGE_SIZE);
3876                 kunmap_atomic(src);
3877
3878                 mark_page_accessed(page);
3879                 page_cache_release(page);
3880         }
3881
3882         return 0;
3883 }
3884
3885 static int
3886 i915_gem_phys_pwrite(struct drm_device *dev,
3887                      struct drm_i915_gem_object *obj,
3888                      struct drm_i915_gem_pwrite *args,
3889                      struct drm_file *file_priv)
3890 {
3891         void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
3892         char __user *user_data = (char __user *) (uintptr_t) args->data_ptr;
3893
3894         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
3895                 unsigned long unwritten;
3896
3897                 /* The physical object once assigned is fixed for the lifetime
3898                  * of the obj, so we can safely drop the lock and continue
3899                  * to access vaddr.
3900                  */
3901                 mutex_unlock(&dev->struct_mutex);
3902                 unwritten = copy_from_user(vaddr, user_data, args->size);
3903                 mutex_lock(&dev->struct_mutex);
3904                 if (unwritten)
3905                         return -EFAULT;
3906         }
3907
3908         intel_gtt_chipset_flush();
3909         return 0;
3910 }
3911
3912 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
3913 {
3914         struct drm_i915_file_private *file_priv = file->driver_priv;
3915
3916         /* Clean up our request list when the client is going away, so that
3917          * later retire_requests won't dereference our soon-to-be-gone
3918          * file_priv.
3919          */
3920         spin_lock(&file_priv->mm.lock);
3921         while (!list_empty(&file_priv->mm.request_list)) {
3922                 struct drm_i915_gem_request *request;
3923
3924                 request = list_first_entry(&file_priv->mm.request_list,
3925                                            struct drm_i915_gem_request,
3926                                            client_list);
3927                 list_del(&request->client_list);
3928                 request->file_priv = NULL;
3929         }
3930         spin_unlock(&file_priv->mm.lock);
3931 }
3932
3933 static int
3934 i915_gpu_is_active(struct drm_device *dev)
3935 {
3936         drm_i915_private_t *dev_priv = dev->dev_private;
3937         int lists_empty;
3938
3939         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
3940                       list_empty(&dev_priv->mm.active_list);
3941
3942         return !lists_empty;
3943 }
3944
3945 static int
3946 i915_gem_inactive_shrink(struct shrinker *shrinker, struct shrink_control *sc)
3947 {
3948         struct drm_i915_private *dev_priv =
3949                 container_of(shrinker,
3950                              struct drm_i915_private,
3951                              mm.inactive_shrinker);
3952         struct drm_device *dev = dev_priv->dev;
3953         struct drm_i915_gem_object *obj, *next;
3954         int nr_to_scan = sc->nr_to_scan;
3955         int cnt;
3956
3957         if (!mutex_trylock(&dev->struct_mutex))
3958                 return 0;
3959
3960         /* "fast-path" to count number of available objects */
3961         if (nr_to_scan == 0) {
3962                 cnt = 0;
3963                 list_for_each_entry(obj,
3964                                     &dev_priv->mm.inactive_list,
3965                                     mm_list)
3966                         cnt++;
3967                 mutex_unlock(&dev->struct_mutex);
3968                 return cnt / 100 * sysctl_vfs_cache_pressure;
3969         }
3970
3971 rescan:
3972         /* first scan for clean buffers */
3973         i915_gem_retire_requests(dev);
3974
3975         list_for_each_entry_safe(obj, next,
3976                                  &dev_priv->mm.inactive_list,
3977                                  mm_list) {
3978                 if (i915_gem_object_is_purgeable(obj)) {
3979                         if (i915_gem_object_unbind(obj) == 0 &&
3980                             --nr_to_scan == 0)
3981                                 break;
3982                 }
3983         }
3984
3985         /* second pass, evict/count anything still on the inactive list */
3986         cnt = 0;
3987         list_for_each_entry_safe(obj, next,
3988                                  &dev_priv->mm.inactive_list,
3989                                  mm_list) {
3990                 if (nr_to_scan &&
3991                     i915_gem_object_unbind(obj) == 0)
3992                         nr_to_scan--;
3993                 else
3994                         cnt++;
3995         }
3996
3997         if (nr_to_scan && i915_gpu_is_active(dev)) {
3998                 /*
3999                  * We are desperate for pages, so as a last resort, wait
4000                  * for the GPU to finish and discard whatever we can.
4001                  * This has a dramatic impact to reduce the number of
4002                  * OOM-killer events whilst running the GPU aggressively.
4003                  */
4004                 if (i915_gpu_idle(dev) == 0)
4005                         goto rescan;
4006         }
4007         mutex_unlock(&dev->struct_mutex);
4008         return cnt / 100 * sysctl_vfs_cache_pressure;
4009 }