Pileus Git - ~andy/linux/blob - drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/shmem_fs.h>
  35 #include <linux/slab.h>
  36 #include <linux/swap.h>
  37 #include <linux/pci.h>
  38
  39 static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
  40 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  41 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  42 static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  43                                                     unsigned alignment,
  44                                                     bool map_and_fenceable);
  45 static int i915_gem_phys_pwrite(struct drm_device *dev,
  46                                 struct drm_i915_gem_object *obj,
  47                                 struct drm_i915_gem_pwrite *args,
  48                                 struct drm_file *file);
  49
  50 static void i915_gem_write_fence(struct drm_device *dev, int reg,
  51                                  struct drm_i915_gem_object *obj);
  52 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  53                                          struct drm_i915_fence_reg *fence,
  54                                          bool enable);
  55
  56 static int i915_gem_inactive_shrink(struct shrinker *shrinker,
  57                                     struct shrink_control *sc);
  58 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
  59
  60 static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
  61 {
  62         if (obj->tiling_mode)
  63                 i915_gem_release_mmap(obj);
  64
  65         /* As we do not have an associated fence register, we will force
  66          * a tiling change if we ever need to acquire one.
  67          */
  68         obj->fence_dirty = false;
  69         obj->fence_reg = I915_FENCE_REG_NONE;
  70 }
  71
  72 /* some bookkeeping */
  73 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  74                                   size_t size)
  75 {
  76         dev_priv->mm.object_count++;
  77         dev_priv->mm.object_memory += size;
  78 }
  79
  80 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  81                                      size_t size)
  82 {
  83         dev_priv->mm.object_count--;
  84         dev_priv->mm.object_memory -= size;
  85 }
  86
  87 static int
  88 i915_gem_wait_for_error(struct drm_device *dev)
  89 {
  90         struct drm_i915_private *dev_priv = dev->dev_private;
  91         struct completion *x = &dev_priv->error_completion;
  92         unsigned long flags;
  93         int ret;
  94
  95         if (!atomic_read(&dev_priv->mm.wedged))
  96                 return 0;
  97
  98         ret = wait_for_completion_interruptible(x);
  99         if (ret)
 100                 return ret;
 101
 102         if (atomic_read(&dev_priv->mm.wedged)) {
 103                 /* GPU is hung, bump the completion count to account for
 104                  * the token we just consumed so that we never hit zero and
 105                  * end up waiting upon a subsequent completion event that
 106                  * will never happen.
 107                  */
 108                 spin_lock_irqsave(&x->wait.lock, flags);
 109                 x->done++;
 110                 spin_unlock_irqrestore(&x->wait.lock, flags);
 111         }
 112         return 0;
 113 }
 114
 115 int i915_mutex_lock_interruptible(struct drm_device *dev)
 116 {
 117         int ret;
 118
 119         ret = i915_gem_wait_for_error(dev);
 120         if (ret)
 121                 return ret;
 122
 123         ret = mutex_lock_interruptible(&dev->struct_mutex);
 124         if (ret)
 125                 return ret;
 126
 127         WARN_ON(i915_verify_lists(dev));
 128         return 0;
 129 }
 130
 131 static inline bool
 132 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
 133 {
 134         return !obj->active;
 135 }
 136
 137 int
 138 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 139                     struct drm_file *file)
 140 {
 141         struct drm_i915_gem_init *args = data;
 142
 143         if (drm_core_check_feature(dev, DRIVER_MODESET))
 144                 return -ENODEV;
 145
 146         if (args->gtt_start >= args->gtt_end ||
 147             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
 148                 return -EINVAL;
 149
 150         /* GEM with user mode setting was never supported on ilk and later. */
 151         if (INTEL_INFO(dev)->gen >= 5)
 152                 return -ENODEV;
 153
 154         mutex_lock(&dev->struct_mutex);
 155         i915_gem_init_global_gtt(dev, args->gtt_start,
 156                                  args->gtt_end, args->gtt_end);
 157         mutex_unlock(&dev->struct_mutex);
 158
 159         return 0;
 160 }
 161
 162 int
 163 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 164                             struct drm_file *file)
 165 {
 166         struct drm_i915_private *dev_priv = dev->dev_private;
 167         struct drm_i915_gem_get_aperture *args = data;
 168         struct drm_i915_gem_object *obj;
 169         size_t pinned;
 170
 171         pinned = 0;
 172         mutex_lock(&dev->struct_mutex);
 173         list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list)
 174                 if (obj->pin_count)
 175                         pinned += obj->gtt_space->size;
 176         mutex_unlock(&dev->struct_mutex);
 177
 178         args->aper_size = dev_priv->mm.gtt_total;
 179         args->aper_available_size = args->aper_size - pinned;
 180
 181         return 0;
 182 }
 183
 184 static int
 185 i915_gem_create(struct drm_file *file,
 186                 struct drm_device *dev,
 187                 uint64_t size,
 188                 uint32_t *handle_p)
 189 {
 190         struct drm_i915_gem_object *obj;
 191         int ret;
 192         u32 handle;
 193
 194         size = roundup(size, PAGE_SIZE);
 195         if (size == 0)
 196                 return -EINVAL;
 197
 198         /* Allocate the new object */
 199         obj = i915_gem_alloc_object(dev, size);
 200         if (obj == NULL)
 201                 return -ENOMEM;
 202
 203         ret = drm_gem_handle_create(file, &obj->base, &handle);
 204         if (ret) {
 205                 drm_gem_object_release(&obj->base);
 206                 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
 207                 kfree(obj);
 208                 return ret;
 209         }
 210
 211         /* drop reference from allocate - handle holds it now */
 212         drm_gem_object_unreference(&obj->base);
 213         trace_i915_gem_object_create(obj);
 214
 215         *handle_p = handle;
 216         return 0;
 217 }
 218
 219 int
 220 i915_gem_dumb_create(struct drm_file *file,
 221                      struct drm_device *dev,
 222                      struct drm_mode_create_dumb *args)
 223 {
 224         /* have to work out size/pitch and return them */
 225         args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
 226         args->size = args->pitch * args->height;
 227         return i915_gem_create(file, dev,
 228                                args->size, &args->handle);
 229 }
 230
 231 int i915_gem_dumb_destroy(struct drm_file *file,
 232                           struct drm_device *dev,
 233                           uint32_t handle)
 234 {
 235         return drm_gem_handle_delete(file, handle);
 236 }
 237
 238 /**
 239  * Creates a new mm object and returns a handle to it.
 240  */
 241 int
 242 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 243                       struct drm_file *file)
 244 {
 245         struct drm_i915_gem_create *args = data;
 246
 247         return i915_gem_create(file, dev,
 248                                args->size, &args->handle);
 249 }
 250
 251 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
 252 {
 253         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
 254
 255         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 256                 obj->tiling_mode != I915_TILING_NONE;
 257 }
 258
 259 static inline int
 260 __copy_to_user_swizzled(char __user *cpu_vaddr,
 261                         const char *gpu_vaddr, int gpu_offset,
 262                         int length)
 263 {
 264         int ret, cpu_offset = 0;
 265
 266         while (length > 0) {
 267                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 268                 int this_length = min(cacheline_end - gpu_offset, length);
 269                 int swizzled_gpu_offset = gpu_offset ^ 64;
 270
 271                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
 272                                      gpu_vaddr + swizzled_gpu_offset,
 273                                      this_length);
 274                 if (ret)
 275                         return ret + length;
 276
 277                 cpu_offset += this_length;
 278                 gpu_offset += this_length;
 279                 length -= this_length;
 280         }
 281
 282         return 0;
 283 }
 284
 285 static inline int
 286 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
 287                           const char __user *cpu_vaddr,
 288                           int length)
 289 {
 290         int ret, cpu_offset = 0;
 291
 292         while (length > 0) {
 293                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 294                 int this_length = min(cacheline_end - gpu_offset, length);
 295                 int swizzled_gpu_offset = gpu_offset ^ 64;
 296
 297                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
 298                                        cpu_vaddr + cpu_offset,
 299                                        this_length);
 300                 if (ret)
 301                         return ret + length;
 302
 303                 cpu_offset += this_length;
 304                 gpu_offset += this_length;
 305                 length -= this_length;
 306         }
 307
 308         return 0;
 309 }
 310
 311 /* Per-page copy function for the shmem pread fastpath.
 312  * Flushes invalid cachelines before reading the target if
 313  * needs_clflush is set. */
 314 static int
 315 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
 316                  char __user *user_data,
 317                  bool page_do_bit17_swizzling, bool needs_clflush)
 318 {
 319         char *vaddr;
 320         int ret;
 321
 322         if (unlikely(page_do_bit17_swizzling))
 323                 return -EINVAL;
 324
 325         vaddr = kmap_atomic(page);
 326         if (needs_clflush)
 327                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 328                                        page_length);
 329         ret = __copy_to_user_inatomic(user_data,
 330                                       vaddr + shmem_page_offset,
 331                                       page_length);
 332         kunmap_atomic(vaddr);
 333
 334         return ret;
 335 }
 336
 337 static void
 338 shmem_clflush_swizzled_range(char *addr, unsigned long length,
 339                              bool swizzled)
 340 {
 341         if (unlikely(swizzled)) {
 342                 unsigned long start = (unsigned long) addr;
 343                 unsigned long end = (unsigned long) addr + length;
 344
 345                 /* For swizzling simply ensure that we always flush both
 346                  * channels. Lame, but simple and it works. Swizzled
 347                  * pwrite/pread is far from a hotpath - current userspace
 348                  * doesn't use it at all. */
 349                 start = round_down(start, 128);
 350                 end = round_up(end, 128);
 351
 352                 drm_clflush_virt_range((void *)start, end - start);
 353         } else {
 354                 drm_clflush_virt_range(addr, length);
 355         }
 356
 357 }
 358
 359 /* Only difference to the fast-path function is that this can handle bit17
 360  * and uses non-atomic copy and kmap functions. */
 361 static int
 362 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
 363                  char __user *user_data,
 364                  bool page_do_bit17_swizzling, bool needs_clflush)
 365 {
 366         char *vaddr;
 367         int ret;
 368
 369         vaddr = kmap(page);
 370         if (needs_clflush)
 371                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 372                                              page_length,
 373                                              page_do_bit17_swizzling);
 374
 375         if (page_do_bit17_swizzling)
 376                 ret = __copy_to_user_swizzled(user_data,
 377                                               vaddr, shmem_page_offset,
 378                                               page_length);
 379         else
 380                 ret = __copy_to_user(user_data,
 381                                      vaddr + shmem_page_offset,
 382                                      page_length);
 383         kunmap(page);
 384
 385         return ret;
 386 }
 387
 388 static int
 389 i915_gem_shmem_pread(struct drm_device *dev,
 390                      struct drm_i915_gem_object *obj,
 391                      struct drm_i915_gem_pread *args,
 392                      struct drm_file *file)
 393 {
 394         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
 395         char __user *user_data;
 396         ssize_t remain;
 397         loff_t offset;
 398         int shmem_page_offset, page_length, ret = 0;
 399         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 400         int hit_slowpath = 0;
 401         int prefaulted = 0;
 402         int needs_clflush = 0;
 403         int release_page;
 404
 405         user_data = (char __user *) (uintptr_t) args->data_ptr;
 406         remain = args->size;
 407
 408         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 409
 410         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
 411                 /* If we're not in the cpu read domain, set ourself into the gtt
 412                  * read domain and manually flush cachelines (if required). This
 413                  * optimizes for the case when the gpu will dirty the data
 414                  * anyway again before the next pread happens. */
 415                 if (obj->cache_level == I915_CACHE_NONE)
 416                         needs_clflush = 1;
 417                 ret = i915_gem_object_set_to_gtt_domain(obj, false);
 418                 if (ret)
 419                         return ret;
 420         }
 421
 422         offset = args->offset;
 423
 424         while (remain > 0) {
 425                 struct page *page;
 426
 427                 /* Operation in this page
 428                  *
 429                  * shmem_page_offset = offset within page in shmem file
 430                  * page_length = bytes to copy for this page
 431                  */
 432                 shmem_page_offset = offset_in_page(offset);
 433                 page_length = remain;
 434                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 435                         page_length = PAGE_SIZE - shmem_page_offset;
 436
 437                 if (obj->pages) {
 438                         page = obj->pages[offset >> PAGE_SHIFT];
 439                         release_page = 0;
 440                 } else {
 441                         page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
 442                         if (IS_ERR(page)) {
 443                                 ret = PTR_ERR(page);
 444                                 goto out;
 445                         }
 446                         release_page = 1;
 447                 }
 448
 449                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 450                         (page_to_phys(page) & (1 << 17)) != 0;
 451
 452                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
 453                                        user_data, page_do_bit17_swizzling,
 454                                        needs_clflush);
 455                 if (ret == 0)
 456                         goto next_page;
 457
 458                 hit_slowpath = 1;
 459                 page_cache_get(page);
 460                 mutex_unlock(&dev->struct_mutex);
 461
 462                 if (!prefaulted) {
 463                         ret = fault_in_multipages_writeable(user_data, remain);
 464                         /* Userspace is tricking us, but we've already clobbered
 465                          * its pages with the prefault and promised to write the
 466                          * data up to the first fault. Hence ignore any errors
 467                          * and just continue. */
 468                         (void)ret;
 469                         prefaulted = 1;
 470                 }
 471
 472                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
 473                                        user_data, page_do_bit17_swizzling,
 474                                        needs_clflush);
 475
 476                 mutex_lock(&dev->struct_mutex);
 477                 page_cache_release(page);
 478 next_page:
 479                 mark_page_accessed(page);
 480                 if (release_page)
 481                         page_cache_release(page);
 482
 483                 if (ret) {
 484                         ret = -EFAULT;
 485                         goto out;
 486                 }
 487
 488                 remain -= page_length;
 489                 user_data += page_length;
 490                 offset += page_length;
 491         }
 492
 493 out:
 494         if (hit_slowpath) {
 495                 /* Fixup: Kill any reinstated backing storage pages */
 496                 if (obj->madv == __I915_MADV_PURGED)
 497                         i915_gem_object_truncate(obj);
 498         }
 499
 500         return ret;
 501 }
 502
 503 /**
 504  * Reads data from the object referenced by handle.
 505  *
 506  * On error, the contents of *data are undefined.
 507  */
 508 int
 509 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 510                      struct drm_file *file)
 511 {
 512         struct drm_i915_gem_pread *args = data;
 513         struct drm_i915_gem_object *obj;
 514         int ret = 0;
 515
 516         if (args->size == 0)
 517                 return 0;
 518
 519         if (!access_ok(VERIFY_WRITE,
 520                        (char __user *)(uintptr_t)args->data_ptr,
 521                        args->size))
 522                 return -EFAULT;
 523
 524         ret = i915_mutex_lock_interruptible(dev);
 525         if (ret)
 526                 return ret;
 527
 528         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 529         if (&obj->base == NULL) {
 530                 ret = -ENOENT;
 531                 goto unlock;
 532         }
 533
 534         /* Bounds check source.  */
 535         if (args->offset > obj->base.size ||
 536             args->size > obj->base.size - args->offset) {
 537                 ret = -EINVAL;
 538                 goto out;
 539         }
 540
 541         trace_i915_gem_object_pread(obj, args->offset, args->size);
 542
 543         ret = i915_gem_shmem_pread(dev, obj, args, file);
 544
 545 out:
 546         drm_gem_object_unreference(&obj->base);
 547 unlock:
 548         mutex_unlock(&dev->struct_mutex);
 549         return ret;
 550 }
 551
 552 /* This is the fast write path which cannot handle
 553  * page faults in the source data
 554  */
 555
 556 static inline int
 557 fast_user_write(struct io_mapping *mapping,
 558                 loff_t page_base, int page_offset,
 559                 char __user *user_data,
 560                 int length)
 561 {
 562         void __iomem *vaddr_atomic;
 563         void *vaddr;
 564         unsigned long unwritten;
 565
 566         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 567         /* We can use the cpu mem copy function because this is X86. */
 568         vaddr = (void __force*)vaddr_atomic + page_offset;
 569         unwritten = __copy_from_user_inatomic_nocache(vaddr,
 570                                                       user_data, length);
 571         io_mapping_unmap_atomic(vaddr_atomic);
 572         return unwritten;
 573 }
 574
 575 /**
 576  * This is the fast pwrite path, where we copy the data directly from the
 577  * user into the GTT, uncached.
 578  */
 579 static int
 580 i915_gem_gtt_pwrite_fast(struct drm_device *dev,
 581                          struct drm_i915_gem_object *obj,
 582                          struct drm_i915_gem_pwrite *args,
 583                          struct drm_file *file)
 584 {
 585         drm_i915_private_t *dev_priv = dev->dev_private;
 586         ssize_t remain;
 587         loff_t offset, page_base;
 588         char __user *user_data;
 589         int page_offset, page_length, ret;
 590
 591         ret = i915_gem_object_pin(obj, 0, true);
 592         if (ret)
 593                 goto out;
 594
 595         ret = i915_gem_object_set_to_gtt_domain(obj, true);
 596         if (ret)
 597                 goto out_unpin;
 598
 599         ret = i915_gem_object_put_fence(obj);
 600         if (ret)
 601                 goto out_unpin;
 602
 603         user_data = (char __user *) (uintptr_t) args->data_ptr;
 604         remain = args->size;
 605
 606         offset = obj->gtt_offset + args->offset;
 607
 608         while (remain > 0) {
 609                 /* Operation in this page
 610                  *
 611                  * page_base = page offset within aperture
 612                  * page_offset = offset within page
 613                  * page_length = bytes to copy for this page
 614                  */
 615                 page_base = offset & PAGE_MASK;
 616                 page_offset = offset_in_page(offset);
 617                 page_length = remain;
 618                 if ((page_offset + remain) > PAGE_SIZE)
 619                         page_length = PAGE_SIZE - page_offset;
 620
 621                 /* If we get a fault while copying data, then (presumably) our
 622                  * source page isn't available.  Return the error and we'll
 623                  * retry in the slow path.
 624                  */
 625                 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
 626                                     page_offset, user_data, page_length)) {
 627                         ret = -EFAULT;
 628                         goto out_unpin;
 629                 }
 630
 631                 remain -= page_length;
 632                 user_data += page_length;
 633                 offset += page_length;
 634         }
 635
 636 out_unpin:
 637         i915_gem_object_unpin(obj);
 638 out:
 639         return ret;
 640 }
 641
 642 /* Per-page copy function for the shmem pwrite fastpath.
 643  * Flushes invalid cachelines before writing to the target if
 644  * needs_clflush_before is set and flushes out any written cachelines after
 645  * writing if needs_clflush is set. */
 646 static int
 647 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
 648                   char __user *user_data,
 649                   bool page_do_bit17_swizzling,
 650                   bool needs_clflush_before,
 651                   bool needs_clflush_after)
 652 {
 653         char *vaddr;
 654         int ret;
 655
 656         if (unlikely(page_do_bit17_swizzling))
 657                 return -EINVAL;
 658
 659         vaddr = kmap_atomic(page);
 660         if (needs_clflush_before)
 661                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 662                                        page_length);
 663         ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
 664                                                 user_data,
 665                                                 page_length);
 666         if (needs_clflush_after)
 667                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 668                                        page_length);
 669         kunmap_atomic(vaddr);
 670
 671         return ret;
 672 }
 673
 674 /* Only difference to the fast-path function is that this can handle bit17
 675  * and uses non-atomic copy and kmap functions. */
 676 static int
 677 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
 678                   char __user *user_data,
 679                   bool page_do_bit17_swizzling,
 680                   bool needs_clflush_before,
 681                   bool needs_clflush_after)
 682 {
 683         char *vaddr;
 684         int ret;
 685
 686         vaddr = kmap(page);
 687         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
 688                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 689                                              page_length,
 690                                              page_do_bit17_swizzling);
 691         if (page_do_bit17_swizzling)
 692                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
 693                                                 user_data,
 694                                                 page_length);
 695         else
 696                 ret = __copy_from_user(vaddr + shmem_page_offset,
 697                                        user_data,
 698                                        page_length);
 699         if (needs_clflush_after)
 700                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 701                                              page_length,
 702                                              page_do_bit17_swizzling);
 703         kunmap(page);
 704
 705         return ret;
 706 }
 707
 708 static int
 709 i915_gem_shmem_pwrite(struct drm_device *dev,
 710                       struct drm_i915_gem_object *obj,
 711                       struct drm_i915_gem_pwrite *args,
 712                       struct drm_file *file)
 713 {
 714         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
 715         ssize_t remain;
 716         loff_t offset;
 717         char __user *user_data;
 718         int shmem_page_offset, page_length, ret = 0;
 719         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 720         int hit_slowpath = 0;
 721         int needs_clflush_after = 0;
 722         int needs_clflush_before = 0;
 723         int release_page;
 724
 725         user_data = (char __user *) (uintptr_t) args->data_ptr;
 726         remain = args->size;
 727
 728         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 729
 730         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 731                 /* If we're not in the cpu write domain, set ourself into the gtt
 732                  * write domain and manually flush cachelines (if required). This
 733                  * optimizes for the case when the gpu will use the data
 734                  * right away and we therefore have to clflush anyway. */
 735                 if (obj->cache_level == I915_CACHE_NONE)
 736                         needs_clflush_after = 1;
 737                 ret = i915_gem_object_set_to_gtt_domain(obj, true);
 738                 if (ret)
 739                         return ret;
 740         }
 741         /* Same trick applies for invalidate partially written cachelines before
 742          * writing.  */
 743         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
 744             && obj->cache_level == I915_CACHE_NONE)
 745                 needs_clflush_before = 1;
 746
 747         offset = args->offset;
 748         obj->dirty = 1;
 749
 750         while (remain > 0) {
 751                 struct page *page;
 752                 int partial_cacheline_write;
 753
 754                 /* Operation in this page
 755                  *
 756                  * shmem_page_offset = offset within page in shmem file
 757                  * page_length = bytes to copy for this page
 758                  */
 759                 shmem_page_offset = offset_in_page(offset);
 760
 761                 page_length = remain;
 762                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 763                         page_length = PAGE_SIZE - shmem_page_offset;
 764
 765                 /* If we don't overwrite a cacheline completely we need to be
 766                  * careful to have up-to-date data by first clflushing. Don't
 767                  * overcomplicate things and flush the entire patch. */
 768                 partial_cacheline_write = needs_clflush_before &&
 769                         ((shmem_page_offset | page_length)
 770                                 & (boot_cpu_data.x86_clflush_size - 1));
 771
 772                 if (obj->pages) {
 773                         page = obj->pages[offset >> PAGE_SHIFT];
 774                         release_page = 0;
 775                 } else {
 776                         page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
 777                         if (IS_ERR(page)) {
 778                                 ret = PTR_ERR(page);
 779                                 goto out;
 780                         }
 781                         release_page = 1;
 782                 }
 783
 784                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 785                         (page_to_phys(page) & (1 << 17)) != 0;
 786
 787                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
 788                                         user_data, page_do_bit17_swizzling,
 789                                         partial_cacheline_write,
 790                                         needs_clflush_after);
 791                 if (ret == 0)
 792                         goto next_page;
 793
 794                 hit_slowpath = 1;
 795                 page_cache_get(page);
 796                 mutex_unlock(&dev->struct_mutex);
 797
 798                 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
 799                                         user_data, page_do_bit17_swizzling,
 800                                         partial_cacheline_write,
 801                                         needs_clflush_after);
 802
 803                 mutex_lock(&dev->struct_mutex);
 804                 page_cache_release(page);
 805 next_page:
 806                 set_page_dirty(page);
 807                 mark_page_accessed(page);
 808                 if (release_page)
 809                         page_cache_release(page);
 810
 811                 if (ret) {
 812                         ret = -EFAULT;
 813                         goto out;
 814                 }
 815
 816                 remain -= page_length;
 817                 user_data += page_length;
 818                 offset += page_length;
 819         }
 820
 821 out:
 822         if (hit_slowpath) {
 823                 /* Fixup: Kill any reinstated backing storage pages */
 824                 if (obj->madv == __I915_MADV_PURGED)
 825                         i915_gem_object_truncate(obj);
 826                 /* and flush dirty cachelines in case the object isn't in the cpu write
 827                  * domain anymore. */
 828                 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 829                         i915_gem_clflush_object(obj);
 830                         intel_gtt_chipset_flush();
 831                 }
 832         }
 833
 834         if (needs_clflush_after)
 835                 intel_gtt_chipset_flush();
 836
 837         return ret;
 838 }
 839
 840 /**
 841  * Writes data to the object referenced by handle.
 842  *
 843  * On error, the contents of the buffer that were to be modified are undefined.
 844  */
 845 int
 846 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 847                       struct drm_file *file)
 848 {
 849         struct drm_i915_gem_pwrite *args = data;
 850         struct drm_i915_gem_object *obj;
 851         int ret;
 852
 853         if (args->size == 0)
 854                 return 0;
 855
 856         if (!access_ok(VERIFY_READ,
 857                        (char __user *)(uintptr_t)args->data_ptr,
 858                        args->size))
 859                 return -EFAULT;
 860
 861         ret = fault_in_multipages_readable((char __user *)(uintptr_t)args->data_ptr,
 862                                            args->size);
 863         if (ret)
 864                 return -EFAULT;
 865
 866         ret = i915_mutex_lock_interruptible(dev);
 867         if (ret)
 868                 return ret;
 869
 870         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 871         if (&obj->base == NULL) {
 872                 ret = -ENOENT;
 873                 goto unlock;
 874         }
 875
 876         /* Bounds check destination. */
 877         if (args->offset > obj->base.size ||
 878             args->size > obj->base.size - args->offset) {
 879                 ret = -EINVAL;
 880                 goto out;
 881         }
 882
 883         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
 884
 885         ret = -EFAULT;
 886         /* We can only do the GTT pwrite on untiled buffers, as otherwise
 887          * it would end up going through the fenced access, and we'll get
 888          * different detiling behavior between reading and writing.
 889          * pread/pwrite currently are reading and writing from the CPU
 890          * perspective, requiring manual detiling by the client.
 891          */
 892         if (obj->phys_obj) {
 893                 ret = i915_gem_phys_pwrite(dev, obj, args, file);
 894                 goto out;
 895         }
 896
 897         if (obj->gtt_space &&
 898             obj->cache_level == I915_CACHE_NONE &&
 899             obj->tiling_mode == I915_TILING_NONE &&
 900             obj->map_and_fenceable &&
 901             obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 902                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
 903                 /* Note that the gtt paths might fail with non-page-backed user
 904                  * pointers (e.g. gtt mappings when moving data between
 905                  * textures). Fallback to the shmem path in that case. */
 906         }
 907
 908         if (ret == -EFAULT)
 909                 ret = i915_gem_shmem_pwrite(dev, obj, args, file);
 910
 911 out:
 912         drm_gem_object_unreference(&obj->base);
 913 unlock:
 914         mutex_unlock(&dev->struct_mutex);
 915         return ret;
 916 }
 917
 918 /**
 919  * Called when user space prepares to use an object with the CPU, either
 920  * through the mmap ioctl's mapping or a GTT mapping.
 921  */
 922 int
 923 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 924                           struct drm_file *file)
 925 {
 926         struct drm_i915_gem_set_domain *args = data;
 927         struct drm_i915_gem_object *obj;
 928         uint32_t read_domains = args->read_domains;
 929         uint32_t write_domain = args->write_domain;
 930         int ret;
 931
 932         /* Only handle setting domains to types used by the CPU. */
 933         if (write_domain & I915_GEM_GPU_DOMAINS)
 934                 return -EINVAL;
 935
 936         if (read_domains & I915_GEM_GPU_DOMAINS)
 937                 return -EINVAL;
 938
 939         /* Having something in the write domain implies it's in the read
 940          * domain, and only that read domain.  Enforce that in the request.
 941          */
 942         if (write_domain != 0 && read_domains != write_domain)
 943                 return -EINVAL;
 944
 945         ret = i915_mutex_lock_interruptible(dev);
 946         if (ret)
 947                 return ret;
 948
 949         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 950         if (&obj->base == NULL) {
 951                 ret = -ENOENT;
 952                 goto unlock;
 953         }
 954
 955         if (read_domains & I915_GEM_DOMAIN_GTT) {
 956                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
 957
 958                 /* Silently promote "you're not bound, there was nothing to do"
 959                  * to success, since the client was just asking us to
 960                  * make sure everything was done.
 961                  */
 962                 if (ret == -EINVAL)
 963                         ret = 0;
 964         } else {
 965                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
 966         }
 967
 968         drm_gem_object_unreference(&obj->base);
 969 unlock:
 970         mutex_unlock(&dev->struct_mutex);
 971         return ret;
 972 }
 973
 974 /**
 975  * Called when user space has done writes to this buffer
 976  */
 977 int
 978 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 979                          struct drm_file *file)
 980 {
 981         struct drm_i915_gem_sw_finish *args = data;
 982         struct drm_i915_gem_object *obj;
 983         int ret = 0;
 984
 985         ret = i915_mutex_lock_interruptible(dev);
 986         if (ret)
 987                 return ret;
 988
 989         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 990         if (&obj->base == NULL) {
 991                 ret = -ENOENT;
 992                 goto unlock;
 993         }
 994
 995         /* Pinned buffers may be scanout, so flush the cache */
 996         if (obj->pin_count)
 997                 i915_gem_object_flush_cpu_write_domain(obj);
 998
 999         drm_gem_object_unreference(&obj->base);
1000 unlock:
1001         mutex_unlock(&dev->struct_mutex);
1002         return ret;
1003 }
1004
1005 /**
1006  * Maps the contents of an object, returning the address it is mapped
1007  * into.
1008  *
1009  * While the mapping holds a reference on the contents of the object, it doesn't
1010  * imply a ref on the object itself.
1011  */
1012 int
1013 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1014                     struct drm_file *file)
1015 {
1016         struct drm_i915_gem_mmap *args = data;
1017         struct drm_gem_object *obj;
1018         unsigned long addr;
1019
1020         obj = drm_gem_object_lookup(dev, file, args->handle);
1021         if (obj == NULL)
1022                 return -ENOENT;
1023
1024         addr = vm_mmap(obj->filp, 0, args->size,
1025                        PROT_READ | PROT_WRITE, MAP_SHARED,
1026                        args->offset);
1027         drm_gem_object_unreference_unlocked(obj);
1028         if (IS_ERR((void *)addr))
1029                 return addr;
1030
1031         args->addr_ptr = (uint64_t) addr;
1032
1033         return 0;
1034 }
1035
1036 /**
1037  * i915_gem_fault - fault a page into the GTT
1038  * vma: VMA in question
1039  * vmf: fault info
1040  *
1041  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1042  * from userspace.  The fault handler takes care of binding the object to
1043  * the GTT (if needed), allocating and programming a fence register (again,
1044  * only if needed based on whether the old reg is still valid or the object
1045  * is tiled) and inserting a new PTE into the faulting process.
1046  *
1047  * Note that the faulting process may involve evicting existing objects
1048  * from the GTT and/or fence registers to make room.  So performance may
1049  * suffer if the GTT working set is large or there are few fence registers
1050  * left.
1051  */
1052 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1053 {
1054         struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1055         struct drm_device *dev = obj->base.dev;
1056         drm_i915_private_t *dev_priv = dev->dev_private;
1057         pgoff_t page_offset;
1058         unsigned long pfn;
1059         int ret = 0;
1060         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1061
1062         /* We don't use vmf->pgoff since that has the fake offset */
1063         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1064                 PAGE_SHIFT;
1065
1066         ret = i915_mutex_lock_interruptible(dev);
1067         if (ret)
1068                 goto out;
1069
1070         trace_i915_gem_object_fault(obj, page_offset, true, write);
1071
1072         /* Now bind it into the GTT if needed */
1073         if (!obj->map_and_fenceable) {
1074                 ret = i915_gem_object_unbind(obj);
1075                 if (ret)
1076                         goto unlock;
1077         }
1078         if (!obj->gtt_space) {
1079                 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
1080                 if (ret)
1081                         goto unlock;
1082
1083                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1084                 if (ret)
1085                         goto unlock;
1086         }
1087
1088         if (!obj->has_global_gtt_mapping)
1089                 i915_gem_gtt_bind_object(obj, obj->cache_level);
1090
1091         ret = i915_gem_object_get_fence(obj);
1092         if (ret)
1093                 goto unlock;
1094
1095         if (i915_gem_object_is_inactive(obj))
1096                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1097
1098         obj->fault_mappable = true;
1099
1100         pfn = ((dev->agp->base + obj->gtt_offset) >> PAGE_SHIFT) +
1101                 page_offset;
1102
1103         /* Finally, remap it using the new GTT offset */
1104         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1105 unlock:
1106         mutex_unlock(&dev->struct_mutex);
1107 out:
1108         switch (ret) {
1109         case -EIO:
1110         case -EAGAIN:
1111                 /* Give the error handler a chance to run and move the
1112                  * objects off the GPU active list. Next time we service the
1113                  * fault, we should be able to transition the page into the
1114                  * GTT without touching the GPU (and so avoid further
1115                  * EIO/EGAIN). If the GPU is wedged, then there is no issue
1116                  * with coherency, just lost writes.
1117                  */
1118                 set_need_resched();
1119         case 0:
1120         case -ERESTARTSYS:
1121         case -EINTR:
1122                 return VM_FAULT_NOPAGE;
1123         case -ENOMEM:
1124                 return VM_FAULT_OOM;
1125         default:
1126                 return VM_FAULT_SIGBUS;
1127         }
1128 }
1129
1130 /**
1131  * i915_gem_release_mmap - remove physical page mappings
1132  * @obj: obj in question
1133  *
1134  * Preserve the reservation of the mmapping with the DRM core code, but
1135  * relinquish ownership of the pages back to the system.
1136  *
1137  * It is vital that we remove the page mapping if we have mapped a tiled
1138  * object through the GTT and then lose the fence register due to
1139  * resource pressure. Similarly if the object has been moved out of the
1140  * aperture, than pages mapped into userspace must be revoked. Removing the
1141  * mapping will then trigger a page fault on the next user access, allowing
1142  * fixup by i915_gem_fault().
1143  */
1144 void
1145 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1146 {
1147         if (!obj->fault_mappable)
1148                 return;
1149
1150         if (obj->base.dev->dev_mapping)
1151                 unmap_mapping_range(obj->base.dev->dev_mapping,
1152                                     (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
1153                                     obj->base.size, 1);
1154
1155         obj->fault_mappable = false;
1156 }
1157
1158 static uint32_t
1159 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1160 {
1161         uint32_t gtt_size;
1162
1163         if (INTEL_INFO(dev)->gen >= 4 ||
1164             tiling_mode == I915_TILING_NONE)
1165                 return size;
1166
1167         /* Previous chips need a power-of-two fence region when tiling */
1168         if (INTEL_INFO(dev)->gen == 3)
1169                 gtt_size = 1024*1024;
1170         else
1171                 gtt_size = 512*1024;
1172
1173         while (gtt_size < size)
1174                 gtt_size <<= 1;
1175
1176         return gtt_size;
1177 }
1178
1179 /**
1180  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1181  * @obj: object to check
1182  *
1183  * Return the required GTT alignment for an object, taking into account
1184  * potential fence register mapping.
1185  */
1186 static uint32_t
1187 i915_gem_get_gtt_alignment(struct drm_device *dev,
1188                            uint32_t size,
1189                            int tiling_mode)
1190 {
1191         /*
1192          * Minimum alignment is 4k (GTT page size), but might be greater
1193          * if a fence register is needed for the object.
1194          */
1195         if (INTEL_INFO(dev)->gen >= 4 ||
1196             tiling_mode == I915_TILING_NONE)
1197                 return 4096;
1198
1199         /*
1200          * Previous chips need to be aligned to the size of the smallest
1201          * fence register that can contain the object.
1202          */
1203         return i915_gem_get_gtt_size(dev, size, tiling_mode);
1204 }
1205
1206 /**
1207  * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
1208  *                                       unfenced object
1209  * @dev: the device
1210  * @size: size of the object
1211  * @tiling_mode: tiling mode of the object
1212  *
1213  * Return the required GTT alignment for an object, only taking into account
1214  * unfenced tiled surface requirements.
1215  */
1216 uint32_t
1217 i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
1218                                     uint32_t size,
1219                                     int tiling_mode)
1220 {
1221         /*
1222          * Minimum alignment is 4k (GTT page size) for sane hw.
1223          */
1224         if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
1225             tiling_mode == I915_TILING_NONE)
1226                 return 4096;
1227
1228         /* Previous hardware however needs to be aligned to a power-of-two
1229          * tile height. The simplest method for determining this is to reuse
1230          * the power-of-tile object size.
1231          */
1232         return i915_gem_get_gtt_size(dev, size, tiling_mode);
1233 }
1234
1235 int
1236 i915_gem_mmap_gtt(struct drm_file *file,
1237                   struct drm_device *dev,
1238                   uint32_t handle,
1239                   uint64_t *offset)
1240 {
1241         struct drm_i915_private *dev_priv = dev->dev_private;
1242         struct drm_i915_gem_object *obj;
1243         int ret;
1244
1245         ret = i915_mutex_lock_interruptible(dev);
1246         if (ret)
1247                 return ret;
1248
1249         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1250         if (&obj->base == NULL) {
1251                 ret = -ENOENT;
1252                 goto unlock;
1253         }
1254
1255         if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
1256                 ret = -E2BIG;
1257                 goto out;
1258         }
1259
1260         if (obj->madv != I915_MADV_WILLNEED) {
1261                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1262                 ret = -EINVAL;
1263                 goto out;
1264         }
1265
1266         if (!obj->base.map_list.map) {
1267                 ret = drm_gem_create_mmap_offset(&obj->base);
1268                 if (ret)
1269                         goto out;
1270         }
1271
1272         *offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
1273
1274 out:
1275         drm_gem_object_unreference(&obj->base);
1276 unlock:
1277         mutex_unlock(&dev->struct_mutex);
1278         return ret;
1279 }
1280
1281 /**
1282  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1283  * @dev: DRM device
1284  * @data: GTT mapping ioctl data
1285  * @file: GEM object info
1286  *
1287  * Simply returns the fake offset to userspace so it can mmap it.
1288  * The mmap call will end up in drm_gem_mmap(), which will set things
1289  * up so we can get faults in the handler above.
1290  *
1291  * The fault handler will take care of binding the object into the GTT
1292  * (since it may have been evicted to make room for something), allocating
1293  * a fence register, and mapping the appropriate aperture address into
1294  * userspace.
1295  */
1296 int
1297 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1298                         struct drm_file *file)
1299 {
1300         struct drm_i915_gem_mmap_gtt *args = data;
1301
1302         return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
1303 }
1304
1305
1306 static int
1307 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
1308                               gfp_t gfpmask)
1309 {
1310         int page_count, i;
1311         struct address_space *mapping;
1312         struct inode *inode;
1313         struct page *page;
1314
1315         /* Get the list of pages out of our struct file.  They'll be pinned
1316          * at this point until we release them.
1317          */
1318         page_count = obj->base.size / PAGE_SIZE;
1319         BUG_ON(obj->pages != NULL);
1320         obj->pages = drm_malloc_ab(page_count, sizeof(struct page *));
1321         if (obj->pages == NULL)
1322                 return -ENOMEM;
1323
1324         inode = obj->base.filp->f_path.dentry->d_inode;
1325         mapping = inode->i_mapping;
1326         gfpmask |= mapping_gfp_mask(mapping);
1327
1328         for (i = 0; i < page_count; i++) {
1329                 page = shmem_read_mapping_page_gfp(mapping, i, gfpmask);
1330                 if (IS_ERR(page))
1331                         goto err_pages;
1332
1333                 obj->pages[i] = page;
1334         }
1335
1336         if (i915_gem_object_needs_bit17_swizzle(obj))
1337                 i915_gem_object_do_bit_17_swizzle(obj);
1338
1339         return 0;
1340
1341 err_pages:
1342         while (i--)
1343                 page_cache_release(obj->pages[i]);
1344
1345         drm_free_large(obj->pages);
1346         obj->pages = NULL;
1347         return PTR_ERR(page);
1348 }
1349
1350 static void
1351 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1352 {
1353         int page_count = obj->base.size / PAGE_SIZE;
1354         int i;
1355
1356         BUG_ON(obj->madv == __I915_MADV_PURGED);
1357
1358         if (i915_gem_object_needs_bit17_swizzle(obj))
1359                 i915_gem_object_save_bit_17_swizzle(obj);
1360
1361         if (obj->madv == I915_MADV_DONTNEED)
1362                 obj->dirty = 0;
1363
1364         for (i = 0; i < page_count; i++) {
1365                 if (obj->dirty)
1366                         set_page_dirty(obj->pages[i]);
1367
1368                 if (obj->madv == I915_MADV_WILLNEED)
1369                         mark_page_accessed(obj->pages[i]);
1370
1371                 page_cache_release(obj->pages[i]);
1372         }
1373         obj->dirty = 0;
1374
1375         drm_free_large(obj->pages);
1376         obj->pages = NULL;
1377 }
1378
1379 void
1380 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1381                                struct intel_ring_buffer *ring,
1382                                u32 seqno)
1383 {
1384         struct drm_device *dev = obj->base.dev;
1385         struct drm_i915_private *dev_priv = dev->dev_private;
1386
1387         BUG_ON(ring == NULL);
1388         obj->ring = ring;
1389
1390         /* Add a reference if we're newly entering the active list. */
1391         if (!obj->active) {
1392                 drm_gem_object_reference(&obj->base);
1393                 obj->active = 1;
1394         }
1395
1396         /* Move from whatever list we were on to the tail of execution. */
1397         list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
1398         list_move_tail(&obj->ring_list, &ring->active_list);
1399
1400         obj->last_rendering_seqno = seqno;
1401
1402         if (obj->fenced_gpu_access) {
1403                 obj->last_fenced_seqno = seqno;
1404
1405                 /* Bump MRU to take account of the delayed flush */
1406                 if (obj->fence_reg != I915_FENCE_REG_NONE) {
1407                         struct drm_i915_fence_reg *reg;
1408
1409                         reg = &dev_priv->fence_regs[obj->fence_reg];
1410                         list_move_tail(&reg->lru_list,
1411                                        &dev_priv->mm.fence_list);
1412                 }
1413         }
1414 }
1415
1416 static void
1417 i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
1418 {
1419         list_del_init(&obj->ring_list);
1420         obj->last_rendering_seqno = 0;
1421         obj->last_fenced_seqno = 0;
1422 }
1423
1424 static void
1425 i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
1426 {
1427         struct drm_device *dev = obj->base.dev;
1428         drm_i915_private_t *dev_priv = dev->dev_private;
1429
1430         BUG_ON(!obj->active);
1431         list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
1432
1433         i915_gem_object_move_off_active(obj);
1434 }
1435
1436 static void
1437 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1438 {
1439         struct drm_device *dev = obj->base.dev;
1440         struct drm_i915_private *dev_priv = dev->dev_private;
1441
1442         list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1443
1444         BUG_ON(!list_empty(&obj->gpu_write_list));
1445         BUG_ON(!obj->active);
1446         obj->ring = NULL;
1447
1448         i915_gem_object_move_off_active(obj);
1449         obj->fenced_gpu_access = false;
1450
1451         obj->active = 0;
1452         obj->pending_gpu_write = false;
1453         drm_gem_object_unreference(&obj->base);
1454
1455         WARN_ON(i915_verify_lists(dev));
1456 }
1457
1458 /* Immediately discard the backing storage */
1459 static void
1460 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1461 {
1462         struct inode *inode;
1463
1464         /* Our goal here is to return as much of the memory as
1465          * is possible back to the system as we are called from OOM.
1466          * To do this we must instruct the shmfs to drop all of its
1467          * backing pages, *now*.
1468          */
1469         inode = obj->base.filp->f_path.dentry->d_inode;
1470         shmem_truncate_range(inode, 0, (loff_t)-1);
1471
1472         if (obj->base.map_list.map)
1473                 drm_gem_free_mmap_offset(&obj->base);
1474
1475         obj->madv = __I915_MADV_PURGED;
1476 }
1477
1478 static inline int
1479 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
1480 {
1481         return obj->madv == I915_MADV_DONTNEED;
1482 }
1483
1484 static void
1485 i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
1486                                uint32_t flush_domains)
1487 {
1488         struct drm_i915_gem_object *obj, *next;
1489
1490         list_for_each_entry_safe(obj, next,
1491                                  &ring->gpu_write_list,
1492                                  gpu_write_list) {
1493                 if (obj->base.write_domain & flush_domains) {
1494                         uint32_t old_write_domain = obj->base.write_domain;
1495
1496                         obj->base.write_domain = 0;
1497                         list_del_init(&obj->gpu_write_list);
1498                         i915_gem_object_move_to_active(obj, ring,
1499                                                        i915_gem_next_request_seqno(ring));
1500
1501                         trace_i915_gem_object_change_domain(obj,
1502                                                             obj->base.read_domains,
1503                                                             old_write_domain);
1504                 }
1505         }
1506 }
1507
1508 static u32
1509 i915_gem_get_seqno(struct drm_device *dev)
1510 {
1511         drm_i915_private_t *dev_priv = dev->dev_private;
1512         u32 seqno = dev_priv->next_seqno;
1513
1514         /* reserve 0 for non-seqno */
1515         if (++dev_priv->next_seqno == 0)
1516                 dev_priv->next_seqno = 1;
1517
1518         return seqno;
1519 }
1520
1521 u32
1522 i915_gem_next_request_seqno(struct intel_ring_buffer *ring)
1523 {
1524         if (ring->outstanding_lazy_request == 0)
1525                 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev);
1526
1527         return ring->outstanding_lazy_request;
1528 }
1529
1530 int
1531 i915_add_request(struct intel_ring_buffer *ring,
1532                  struct drm_file *file,
1533                  struct drm_i915_gem_request *request)
1534 {
1535         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1536         uint32_t seqno;
1537         u32 request_ring_position;
1538         int was_empty;
1539         int ret;
1540
1541         BUG_ON(request == NULL);
1542         seqno = i915_gem_next_request_seqno(ring);
1543
1544         /* Record the position of the start of the request so that
1545          * should we detect the updated seqno part-way through the
1546          * GPU processing the request, we never over-estimate the
1547          * position of the head.
1548          */
1549         request_ring_position = intel_ring_get_tail(ring);
1550
1551         ret = ring->add_request(ring, &seqno);
1552         if (ret)
1553             return ret;
1554
1555         trace_i915_gem_request_add(ring, seqno);
1556
1557         request->seqno = seqno;
1558         request->ring = ring;
1559         request->tail = request_ring_position;
1560         request->emitted_jiffies = jiffies;
1561         was_empty = list_empty(&ring->request_list);
1562         list_add_tail(&request->list, &ring->request_list);
1563
1564         if (file) {
1565                 struct drm_i915_file_private *file_priv = file->driver_priv;
1566
1567                 spin_lock(&file_priv->mm.lock);
1568                 request->file_priv = file_priv;
1569                 list_add_tail(&request->client_list,
1570                               &file_priv->mm.request_list);
1571                 spin_unlock(&file_priv->mm.lock);
1572         }
1573
1574         ring->outstanding_lazy_request = 0;
1575
1576         if (!dev_priv->mm.suspended) {
1577                 if (i915_enable_hangcheck) {
1578                         mod_timer(&dev_priv->hangcheck_timer,
1579                                   jiffies +
1580                                   msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
1581                 }
1582                 if (was_empty)
1583                         queue_delayed_work(dev_priv->wq,
1584                                            &dev_priv->mm.retire_work, HZ);
1585         }
1586         return 0;
1587 }
1588
1589 static inline void
1590 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1591 {
1592         struct drm_i915_file_private *file_priv = request->file_priv;
1593
1594         if (!file_priv)
1595                 return;
1596
1597         spin_lock(&file_priv->mm.lock);
1598         if (request->file_priv) {
1599                 list_del(&request->client_list);
1600                 request->file_priv = NULL;
1601         }
1602         spin_unlock(&file_priv->mm.lock);
1603 }
1604
1605 static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1606                                       struct intel_ring_buffer *ring)
1607 {
1608         while (!list_empty(&ring->request_list)) {
1609                 struct drm_i915_gem_request *request;
1610
1611                 request = list_first_entry(&ring->request_list,
1612                                            struct drm_i915_gem_request,
1613                                            list);
1614
1615                 list_del(&request->list);
1616                 i915_gem_request_remove_from_client(request);
1617                 kfree(request);
1618         }
1619
1620         while (!list_empty(&ring->active_list)) {
1621                 struct drm_i915_gem_object *obj;
1622
1623                 obj = list_first_entry(&ring->active_list,
1624                                        struct drm_i915_gem_object,
1625                                        ring_list);
1626
1627                 obj->base.write_domain = 0;
1628                 list_del_init(&obj->gpu_write_list);
1629                 i915_gem_object_move_to_inactive(obj);
1630         }
1631 }
1632
1633 static void i915_gem_reset_fences(struct drm_device *dev)
1634 {
1635         struct drm_i915_private *dev_priv = dev->dev_private;
1636         int i;
1637
1638         for (i = 0; i < dev_priv->num_fence_regs; i++) {
1639                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
1640
1641                 i915_gem_write_fence(dev, i, NULL);
1642
1643                 if (reg->obj)
1644                         i915_gem_object_fence_lost(reg->obj);
1645
1646                 reg->pin_count = 0;
1647                 reg->obj = NULL;
1648                 INIT_LIST_HEAD(&reg->lru_list);
1649         }
1650
1651         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
1652 }
1653
1654 void i915_gem_reset(struct drm_device *dev)
1655 {
1656         struct drm_i915_private *dev_priv = dev->dev_private;
1657         struct drm_i915_gem_object *obj;
1658         struct intel_ring_buffer *ring;
1659         int i;
1660
1661         for_each_ring(ring, dev_priv, i)
1662                 i915_gem_reset_ring_lists(dev_priv, ring);
1663
1664         /* Remove anything from the flushing lists. The GPU cache is likely
1665          * to be lost on reset along with the data, so simply move the
1666          * lost bo to the inactive list.
1667          */
1668         while (!list_empty(&dev_priv->mm.flushing_list)) {
1669                 obj = list_first_entry(&dev_priv->mm.flushing_list,
1670                                       struct drm_i915_gem_object,
1671                                       mm_list);
1672
1673                 obj->base.write_domain = 0;
1674                 list_del_init(&obj->gpu_write_list);
1675                 i915_gem_object_move_to_inactive(obj);
1676         }
1677
1678         /* Move everything out of the GPU domains to ensure we do any
1679          * necessary invalidation upon reuse.
1680          */
1681         list_for_each_entry(obj,
1682                             &dev_priv->mm.inactive_list,
1683                             mm_list)
1684         {
1685                 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1686         }
1687
1688         /* The fence registers are invalidated so clear them out */
1689         i915_gem_reset_fences(dev);
1690 }
1691
1692 /**
1693  * This function clears the request list as sequence numbers are passed.
1694  */
1695 void
1696 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
1697 {
1698         uint32_t seqno;
1699         int i;
1700
1701         if (list_empty(&ring->request_list))
1702                 return;
1703
1704         WARN_ON(i915_verify_lists(ring->dev));
1705
1706         seqno = ring->get_seqno(ring);
1707
1708         for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
1709                 if (seqno >= ring->sync_seqno[i])
1710                         ring->sync_seqno[i] = 0;
1711
1712         while (!list_empty(&ring->request_list)) {
1713                 struct drm_i915_gem_request *request;
1714
1715                 request = list_first_entry(&ring->request_list,
1716                                            struct drm_i915_gem_request,
1717                                            list);
1718
1719                 if (!i915_seqno_passed(seqno, request->seqno))
1720                         break;
1721
1722                 trace_i915_gem_request_retire(ring, request->seqno);
1723                 /* We know the GPU must have read the request to have
1724                  * sent us the seqno + interrupt, so use the position
1725                  * of tail of the request to update the last known position
1726                  * of the GPU head.
1727                  */
1728                 ring->last_retired_head = request->tail;
1729
1730                 list_del(&request->list);
1731                 i915_gem_request_remove_from_client(request);
1732                 kfree(request);
1733         }
1734
1735         /* Move any buffers on the active list that are no longer referenced
1736          * by the ringbuffer to the flushing/inactive lists as appropriate.
1737          */
1738         while (!list_empty(&ring->active_list)) {
1739                 struct drm_i915_gem_object *obj;
1740
1741                 obj = list_first_entry(&ring->active_list,
1742                                       struct drm_i915_gem_object,
1743                                       ring_list);
1744
1745                 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
1746                         break;
1747
1748                 if (obj->base.write_domain != 0)
1749                         i915_gem_object_move_to_flushing(obj);
1750                 else
1751                         i915_gem_object_move_to_inactive(obj);
1752         }
1753
1754         if (unlikely(ring->trace_irq_seqno &&
1755                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
1756                 ring->irq_put(ring);
1757                 ring->trace_irq_seqno = 0;
1758         }
1759
1760         WARN_ON(i915_verify_lists(ring->dev));
1761 }
1762
1763 void
1764 i915_gem_retire_requests(struct drm_device *dev)
1765 {
1766         drm_i915_private_t *dev_priv = dev->dev_private;
1767         struct intel_ring_buffer *ring;
1768         int i;
1769
1770         for_each_ring(ring, dev_priv, i)
1771                 i915_gem_retire_requests_ring(ring);
1772 }
1773
1774 static void
1775 i915_gem_retire_work_handler(struct work_struct *work)
1776 {
1777         drm_i915_private_t *dev_priv;
1778         struct drm_device *dev;
1779         struct intel_ring_buffer *ring;
1780         bool idle;
1781         int i;
1782
1783         dev_priv = container_of(work, drm_i915_private_t,
1784                                 mm.retire_work.work);
1785         dev = dev_priv->dev;
1786
1787         /* Come back later if the device is busy... */
1788         if (!mutex_trylock(&dev->struct_mutex)) {
1789                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1790                 return;
1791         }
1792
1793         i915_gem_retire_requests(dev);
1794
1795         /* Send a periodic flush down the ring so we don't hold onto GEM
1796          * objects indefinitely.
1797          */
1798         idle = true;
1799         for_each_ring(ring, dev_priv, i) {
1800                 if (!list_empty(&ring->gpu_write_list)) {
1801                         struct drm_i915_gem_request *request;
1802                         int ret;
1803
1804                         ret = i915_gem_flush_ring(ring,
1805                                                   0, I915_GEM_GPU_DOMAINS);
1806                         request = kzalloc(sizeof(*request), GFP_KERNEL);
1807                         if (ret || request == NULL ||
1808                             i915_add_request(ring, NULL, request))
1809                             kfree(request);
1810                 }
1811
1812                 idle &= list_empty(&ring->request_list);
1813         }
1814
1815         if (!dev_priv->mm.suspended && !idle)
1816                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1817
1818         mutex_unlock(&dev->struct_mutex);
1819 }
1820
1821 static int
1822 i915_gem_check_wedge(struct drm_i915_private *dev_priv)
1823 {
1824         BUG_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex));
1825
1826         if (atomic_read(&dev_priv->mm.wedged)) {
1827                 struct completion *x = &dev_priv->error_completion;
1828                 bool recovery_complete;
1829                 unsigned long flags;
1830
1831                 /* Give the error handler a chance to run. */
1832                 spin_lock_irqsave(&x->wait.lock, flags);
1833                 recovery_complete = x->done > 0;
1834                 spin_unlock_irqrestore(&x->wait.lock, flags);
1835
1836                 return recovery_complete ? -EIO : -EAGAIN;
1837         }
1838
1839         return 0;
1840 }
1841
1842 /*
1843  * Compare seqno against outstanding lazy request. Emit a request if they are
1844  * equal.
1845  */
1846 static int
1847 i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
1848 {
1849         int ret = 0;
1850
1851         BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
1852
1853         if (seqno == ring->outstanding_lazy_request) {
1854                 struct drm_i915_gem_request *request;
1855
1856                 request = kzalloc(sizeof(*request), GFP_KERNEL);
1857                 if (request == NULL)
1858                         return -ENOMEM;
1859
1860                 ret = i915_add_request(ring, NULL, request);
1861                 if (ret) {
1862                         kfree(request);
1863                         return ret;
1864                 }
1865
1866                 BUG_ON(seqno != request->seqno);
1867         }
1868
1869         return ret;
1870 }
1871
1872 static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
1873                         bool interruptible)
1874 {
1875         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1876         int ret = 0;
1877
1878         if (i915_seqno_passed(ring->get_seqno(ring), seqno))
1879                 return 0;
1880
1881         trace_i915_gem_request_wait_begin(ring, seqno);
1882         if (WARN_ON(!ring->irq_get(ring)))
1883                 return -ENODEV;
1884
1885 #define EXIT_COND \
1886         (i915_seqno_passed(ring->get_seqno(ring), seqno) || \
1887         atomic_read(&dev_priv->mm.wedged))
1888
1889         if (interruptible)
1890                 ret = wait_event_interruptible(ring->irq_queue,
1891                                                EXIT_COND);
1892         else
1893                 wait_event(ring->irq_queue, EXIT_COND);
1894
1895         ring->irq_put(ring);
1896         trace_i915_gem_request_wait_end(ring, seqno);
1897 #undef EXIT_COND
1898
1899         return ret;
1900 }
1901
1902 /**
1903  * Waits for a sequence number to be signaled, and cleans up the
1904  * request and object lists appropriately for that event.
1905  */
1906 int
1907 i915_wait_request(struct intel_ring_buffer *ring,
1908                   uint32_t seqno)
1909 {
1910         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1911         int ret = 0;
1912
1913         BUG_ON(seqno == 0);
1914
1915         ret = i915_gem_check_wedge(dev_priv);
1916         if (ret)
1917                 return ret;
1918
1919         ret = i915_gem_check_olr(ring, seqno);
1920         if (ret)
1921                 return ret;
1922
1923         ret = __wait_seqno(ring, seqno, dev_priv->mm.interruptible);
1924         if (atomic_read(&dev_priv->mm.wedged))
1925                 ret = -EAGAIN;
1926
1927         return ret;
1928 }
1929
1930 /**
1931  * Ensures that all rendering to the object has completed and the object is
1932  * safe to unbind from the GTT or access from the CPU.
1933  */
1934 int
1935 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
1936 {
1937         int ret;
1938
1939         /* This function only exists to support waiting for existing rendering,
1940          * not for emitting required flushes.
1941          */
1942         BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
1943
1944         /* If there is rendering queued on the buffer being evicted, wait for
1945          * it.
1946          */
1947         if (obj->active) {
1948                 ret = i915_wait_request(obj->ring, obj->last_rendering_seqno);
1949                 if (ret)
1950                         return ret;
1951                 i915_gem_retire_requests_ring(obj->ring);
1952         }
1953
1954         return 0;
1955 }
1956
1957 /**
1958  * i915_gem_object_sync - sync an object to a ring.
1959  *
1960  * @obj: object which may be in use on another ring.
1961  * @to: ring we wish to use the object on. May be NULL.
1962  *
1963  * This code is meant to abstract object synchronization with the GPU.
1964  * Calling with NULL implies synchronizing the object with the CPU
1965  * rather than a particular GPU ring.
1966  *
1967  * Returns 0 if successful, else propagates up the lower layer error.
1968  */
1969 int
1970 i915_gem_object_sync(struct drm_i915_gem_object *obj,
1971                      struct intel_ring_buffer *to)
1972 {
1973         struct intel_ring_buffer *from = obj->ring;
1974         u32 seqno;
1975         int ret, idx;
1976
1977         if (from == NULL || to == from)
1978                 return 0;
1979
1980         if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
1981                 return i915_gem_object_wait_rendering(obj);
1982
1983         idx = intel_ring_sync_index(from, to);
1984
1985         seqno = obj->last_rendering_seqno;
1986         if (seqno <= from->sync_seqno[idx])
1987                 return 0;
1988
1989         ret = i915_gem_check_olr(obj->ring, seqno);
1990         if (ret)
1991                 return ret;
1992
1993         ret = to->sync_to(to, from, seqno);
1994         if (!ret)
1995                 from->sync_seqno[idx] = seqno;
1996
1997         return ret;
1998 }
1999
2000 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
2001 {
2002         u32 old_write_domain, old_read_domains;
2003
2004         /* Act a barrier for all accesses through the GTT */
2005         mb();
2006
2007         /* Force a pagefault for domain tracking on next user access */
2008         i915_gem_release_mmap(obj);
2009
2010         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
2011                 return;
2012
2013         old_read_domains = obj->base.read_domains;
2014         old_write_domain = obj->base.write_domain;
2015
2016         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
2017         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2018
2019         trace_i915_gem_object_change_domain(obj,
2020                                             old_read_domains,
2021                                             old_write_domain);
2022 }
2023
2024 /**
2025  * Unbinds an object from the GTT aperture.
2026  */
2027 int
2028 i915_gem_object_unbind(struct drm_i915_gem_object *obj)
2029 {
2030         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2031         int ret = 0;
2032
2033         if (obj->gtt_space == NULL)
2034                 return 0;
2035
2036         if (obj->pin_count)
2037                 return -EBUSY;
2038
2039         ret = i915_gem_object_finish_gpu(obj);
2040         if (ret)
2041                 return ret;
2042         /* Continue on if we fail due to EIO, the GPU is hung so we
2043          * should be safe and we need to cleanup or else we might
2044          * cause memory corruption through use-after-free.
2045          */
2046
2047         i915_gem_object_finish_gtt(obj);
2048
2049         /* Move the object to the CPU domain to ensure that
2050          * any possible CPU writes while it's not in the GTT
2051          * are flushed when we go to remap it.
2052          */
2053         if (ret == 0)
2054                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2055         if (ret == -ERESTARTSYS)
2056                 return ret;
2057         if (ret) {
2058                 /* In the event of a disaster, abandon all caches and
2059                  * hope for the best.
2060                  */
2061                 i915_gem_clflush_object(obj);
2062                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2063         }
2064
2065         /* release the fence reg _after_ flushing */
2066         ret = i915_gem_object_put_fence(obj);
2067         if (ret)
2068                 return ret;
2069
2070         trace_i915_gem_object_unbind(obj);
2071
2072         if (obj->has_global_gtt_mapping)
2073                 i915_gem_gtt_unbind_object(obj);
2074         if (obj->has_aliasing_ppgtt_mapping) {
2075                 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
2076                 obj->has_aliasing_ppgtt_mapping = 0;
2077         }
2078         i915_gem_gtt_finish_object(obj);
2079
2080         i915_gem_object_put_pages_gtt(obj);
2081
2082         list_del_init(&obj->gtt_list);
2083         list_del_init(&obj->mm_list);
2084         /* Avoid an unnecessary call to unbind on rebind. */
2085         obj->map_and_fenceable = true;
2086
2087         drm_mm_put_block(obj->gtt_space);
2088         obj->gtt_space = NULL;
2089         obj->gtt_offset = 0;
2090
2091         if (i915_gem_object_is_purgeable(obj))
2092                 i915_gem_object_truncate(obj);
2093
2094         return ret;
2095 }
2096
2097 int
2098 i915_gem_flush_ring(struct intel_ring_buffer *ring,
2099                     uint32_t invalidate_domains,
2100                     uint32_t flush_domains)
2101 {
2102         int ret;
2103
2104         if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
2105                 return 0;
2106
2107         trace_i915_gem_ring_flush(ring, invalidate_domains, flush_domains);
2108
2109         ret = ring->flush(ring, invalidate_domains, flush_domains);
2110         if (ret)
2111                 return ret;
2112
2113         if (flush_domains & I915_GEM_GPU_DOMAINS)
2114                 i915_gem_process_flushing_list(ring, flush_domains);
2115
2116         return 0;
2117 }
2118
2119 static int i915_ring_idle(struct intel_ring_buffer *ring)
2120 {
2121         int ret;
2122
2123         if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
2124                 return 0;
2125
2126         if (!list_empty(&ring->gpu_write_list)) {
2127                 ret = i915_gem_flush_ring(ring,
2128                                     I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2129                 if (ret)
2130                         return ret;
2131         }
2132
2133         return i915_wait_request(ring, i915_gem_next_request_seqno(ring));
2134 }
2135
2136 int i915_gpu_idle(struct drm_device *dev)
2137 {
2138         drm_i915_private_t *dev_priv = dev->dev_private;
2139         struct intel_ring_buffer *ring;
2140         int ret, i;
2141
2142         /* Flush everything onto the inactive list. */
2143         for_each_ring(ring, dev_priv, i) {
2144                 ret = i915_ring_idle(ring);
2145                 if (ret)
2146                         return ret;
2147
2148                 /* Is the device fubar? */
2149                 if (WARN_ON(!list_empty(&ring->gpu_write_list)))
2150                         return -EBUSY;
2151         }
2152
2153         return 0;
2154 }
2155
2156 static void sandybridge_write_fence_reg(struct drm_device *dev, int reg,
2157                                         struct drm_i915_gem_object *obj)
2158 {
2159         drm_i915_private_t *dev_priv = dev->dev_private;
2160         uint64_t val;
2161
2162         if (obj) {
2163                 u32 size = obj->gtt_space->size;
2164
2165                 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2166                                  0xfffff000) << 32;
2167                 val |= obj->gtt_offset & 0xfffff000;
2168                 val |= (uint64_t)((obj->stride / 128) - 1) <<
2169                         SANDYBRIDGE_FENCE_PITCH_SHIFT;
2170
2171                 if (obj->tiling_mode == I915_TILING_Y)
2172                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2173                 val |= I965_FENCE_REG_VALID;
2174         } else
2175                 val = 0;
2176
2177         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
2178         POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
2179 }
2180
2181 static void i965_write_fence_reg(struct drm_device *dev, int reg,
2182                                  struct drm_i915_gem_object *obj)
2183 {
2184         drm_i915_private_t *dev_priv = dev->dev_private;
2185         uint64_t val;
2186
2187         if (obj) {
2188                 u32 size = obj->gtt_space->size;
2189
2190                 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2191                                  0xfffff000) << 32;
2192                 val |= obj->gtt_offset & 0xfffff000;
2193                 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2194                 if (obj->tiling_mode == I915_TILING_Y)
2195                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2196                 val |= I965_FENCE_REG_VALID;
2197         } else
2198                 val = 0;
2199
2200         I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
2201         POSTING_READ(FENCE_REG_965_0 + reg * 8);
2202 }
2203
2204 static void i915_write_fence_reg(struct drm_device *dev, int reg,
2205                                  struct drm_i915_gem_object *obj)
2206 {
2207         drm_i915_private_t *dev_priv = dev->dev_private;
2208         u32 val;
2209
2210         if (obj) {
2211                 u32 size = obj->gtt_space->size;
2212                 int pitch_val;
2213                 int tile_width;
2214
2215                 WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
2216                      (size & -size) != size ||
2217                      (obj->gtt_offset & (size - 1)),
2218                      "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2219                      obj->gtt_offset, obj->map_and_fenceable, size);
2220
2221                 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
2222                         tile_width = 128;
2223                 else
2224                         tile_width = 512;
2225
2226                 /* Note: pitch better be a power of two tile widths */
2227                 pitch_val = obj->stride / tile_width;
2228                 pitch_val = ffs(pitch_val) - 1;
2229
2230                 val = obj->gtt_offset;
2231                 if (obj->tiling_mode == I915_TILING_Y)
2232                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2233                 val |= I915_FENCE_SIZE_BITS(size);
2234                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2235                 val |= I830_FENCE_REG_VALID;
2236         } else
2237                 val = 0;
2238
2239         if (reg < 8)
2240                 reg = FENCE_REG_830_0 + reg * 4;
2241         else
2242                 reg = FENCE_REG_945_8 + (reg - 8) * 4;
2243
2244         I915_WRITE(reg, val);
2245         POSTING_READ(reg);
2246 }
2247
2248 static void i830_write_fence_reg(struct drm_device *dev, int reg,
2249                                 struct drm_i915_gem_object *obj)
2250 {
2251         drm_i915_private_t *dev_priv = dev->dev_private;
2252         uint32_t val;
2253
2254         if (obj) {
2255                 u32 size = obj->gtt_space->size;
2256                 uint32_t pitch_val;
2257
2258                 WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
2259                      (size & -size) != size ||
2260                      (obj->gtt_offset & (size - 1)),
2261                      "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
2262                      obj->gtt_offset, size);
2263
2264                 pitch_val = obj->stride / 128;
2265                 pitch_val = ffs(pitch_val) - 1;
2266
2267                 val = obj->gtt_offset;
2268                 if (obj->tiling_mode == I915_TILING_Y)
2269                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2270                 val |= I830_FENCE_SIZE_BITS(size);
2271                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2272                 val |= I830_FENCE_REG_VALID;
2273         } else
2274                 val = 0;
2275
2276         I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
2277         POSTING_READ(FENCE_REG_830_0 + reg * 4);
2278 }
2279
2280 static void i915_gem_write_fence(struct drm_device *dev, int reg,
2281                                  struct drm_i915_gem_object *obj)
2282 {
2283         switch (INTEL_INFO(dev)->gen) {
2284         case 7:
2285         case 6: sandybridge_write_fence_reg(dev, reg, obj); break;
2286         case 5:
2287         case 4: i965_write_fence_reg(dev, reg, obj); break;
2288         case 3: i915_write_fence_reg(dev, reg, obj); break;
2289         case 2: i830_write_fence_reg(dev, reg, obj); break;
2290         default: break;
2291         }
2292 }
2293
2294 static inline int fence_number(struct drm_i915_private *dev_priv,
2295                                struct drm_i915_fence_reg *fence)
2296 {
2297         return fence - dev_priv->fence_regs;
2298 }
2299
2300 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
2301                                          struct drm_i915_fence_reg *fence,
2302                                          bool enable)
2303 {
2304         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2305         int reg = fence_number(dev_priv, fence);
2306
2307         i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
2308
2309         if (enable) {
2310                 obj->fence_reg = reg;
2311                 fence->obj = obj;
2312                 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
2313         } else {
2314                 obj->fence_reg = I915_FENCE_REG_NONE;
2315                 fence->obj = NULL;
2316                 list_del_init(&fence->lru_list);
2317         }
2318 }
2319
2320 static int
2321 i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
2322 {
2323         int ret;
2324
2325         if (obj->fenced_gpu_access) {
2326                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2327                         ret = i915_gem_flush_ring(obj->ring,
2328                                                   0, obj->base.write_domain);
2329                         if (ret)
2330                                 return ret;
2331                 }
2332
2333                 obj->fenced_gpu_access = false;
2334         }
2335
2336         if (obj->last_fenced_seqno) {
2337                 ret = i915_wait_request(obj->ring, obj->last_fenced_seqno);
2338                 if (ret)
2339                         return ret;
2340
2341                 obj->last_fenced_seqno = 0;
2342         }
2343
2344         /* Ensure that all CPU reads are completed before installing a fence
2345          * and all writes before removing the fence.
2346          */
2347         if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
2348                 mb();
2349
2350         return 0;
2351 }
2352
2353 int
2354 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2355 {
2356         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2357         int ret;
2358
2359         ret = i915_gem_object_flush_fence(obj);
2360         if (ret)
2361                 return ret;
2362
2363         if (obj->fence_reg == I915_FENCE_REG_NONE)
2364                 return 0;
2365
2366         i915_gem_object_update_fence(obj,
2367                                      &dev_priv->fence_regs[obj->fence_reg],
2368                                      false);
2369         i915_gem_object_fence_lost(obj);
2370
2371         return 0;
2372 }
2373
2374 static struct drm_i915_fence_reg *
2375 i915_find_fence_reg(struct drm_device *dev)
2376 {
2377         struct drm_i915_private *dev_priv = dev->dev_private;
2378         struct drm_i915_fence_reg *reg, *avail;
2379         int i;
2380
2381         /* First try to find a free reg */
2382         avail = NULL;
2383         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2384                 reg = &dev_priv->fence_regs[i];
2385                 if (!reg->obj)
2386                         return reg;
2387
2388                 if (!reg->pin_count)
2389                         avail = reg;
2390         }
2391
2392         if (avail == NULL)
2393                 return NULL;
2394
2395         /* None available, try to steal one or wait for a user to finish */
2396         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
2397                 if (reg->pin_count)
2398                         continue;
2399
2400                 return reg;
2401         }
2402
2403         return NULL;
2404 }
2405
2406 /**
2407  * i915_gem_object_get_fence - set up fencing for an object
2408  * @obj: object to map through a fence reg
2409  *
2410  * When mapping objects through the GTT, userspace wants to be able to write
2411  * to them without having to worry about swizzling if the object is tiled.
2412  * This function walks the fence regs looking for a free one for @obj,
2413  * stealing one if it can't find any.
2414  *
2415  * It then sets up the reg based on the object's properties: address, pitch
2416  * and tiling format.
2417  *
2418  * For an untiled surface, this removes any existing fence.
2419  */
2420 int
2421 i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
2422 {
2423         struct drm_device *dev = obj->base.dev;
2424         struct drm_i915_private *dev_priv = dev->dev_private;
2425         bool enable = obj->tiling_mode != I915_TILING_NONE;
2426         struct drm_i915_fence_reg *reg;
2427         int ret;
2428
2429         /* Have we updated the tiling parameters upon the object and so
2430          * will need to serialise the write to the associated fence register?
2431          */
2432         if (obj->fence_dirty) {
2433                 ret = i915_gem_object_flush_fence(obj);
2434                 if (ret)
2435                         return ret;
2436         }
2437
2438         /* Just update our place in the LRU if our fence is getting reused. */
2439         if (obj->fence_reg != I915_FENCE_REG_NONE) {
2440                 reg = &dev_priv->fence_regs[obj->fence_reg];
2441                 if (!obj->fence_dirty) {
2442                         list_move_tail(&reg->lru_list,
2443                                        &dev_priv->mm.fence_list);
2444                         return 0;
2445                 }
2446         } else if (enable) {
2447                 reg = i915_find_fence_reg(dev);
2448                 if (reg == NULL)
2449                         return -EDEADLK;
2450
2451                 if (reg->obj) {
2452                         struct drm_i915_gem_object *old = reg->obj;
2453
2454                         ret = i915_gem_object_flush_fence(old);
2455                         if (ret)
2456                                 return ret;
2457
2458                         i915_gem_object_fence_lost(old);
2459                 }
2460         } else
2461                 return 0;
2462
2463         i915_gem_object_update_fence(obj, reg, enable);
2464         obj->fence_dirty = false;
2465
2466         return 0;
2467 }
2468
2469 /**
2470  * Finds free space in the GTT aperture and binds the object there.
2471  */
2472 static int
2473 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
2474                             unsigned alignment,
2475                             bool map_and_fenceable)
2476 {
2477         struct drm_device *dev = obj->base.dev;
2478         drm_i915_private_t *dev_priv = dev->dev_private;
2479         struct drm_mm_node *free_space;
2480         gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
2481         u32 size, fence_size, fence_alignment, unfenced_alignment;
2482         bool mappable, fenceable;
2483         int ret;
2484
2485         if (obj->madv != I915_MADV_WILLNEED) {
2486                 DRM_ERROR("Attempting to bind a purgeable object\n");
2487                 return -EINVAL;
2488         }
2489
2490         fence_size = i915_gem_get_gtt_size(dev,
2491                                            obj->base.size,
2492                                            obj->tiling_mode);
2493         fence_alignment = i915_gem_get_gtt_alignment(dev,
2494                                                      obj->base.size,
2495                                                      obj->tiling_mode);
2496         unfenced_alignment =
2497                 i915_gem_get_unfenced_gtt_alignment(dev,
2498                                                     obj->base.size,
2499                                                     obj->tiling_mode);
2500
2501         if (alignment == 0)
2502                 alignment = map_and_fenceable ? fence_alignment :
2503                                                 unfenced_alignment;
2504         if (map_and_fenceable && alignment & (fence_alignment - 1)) {
2505                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2506                 return -EINVAL;
2507         }
2508
2509         size = map_and_fenceable ? fence_size : obj->base.size;
2510
2511         /* If the object is bigger than the entire aperture, reject it early
2512          * before evicting everything in a vain attempt to find space.
2513          */
2514         if (obj->base.size >
2515             (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
2516                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2517                 return -E2BIG;
2518         }
2519
2520  search_free:
2521         if (map_and_fenceable)
2522                 free_space =
2523                         drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
2524                                                     size, alignment, 0,
2525                                                     dev_priv->mm.gtt_mappable_end,
2526                                                     0);
2527         else
2528                 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2529                                                 size, alignment, 0);
2530
2531         if (free_space != NULL) {
2532                 if (map_and_fenceable)
2533                         obj->gtt_space =
2534                                 drm_mm_get_block_range_generic(free_space,
2535                                                                size, alignment, 0,
2536                                                                dev_priv->mm.gtt_mappable_end,
2537                                                                0);
2538                 else
2539                         obj->gtt_space =
2540                                 drm_mm_get_block(free_space, size, alignment);
2541         }
2542         if (obj->gtt_space == NULL) {
2543                 /* If the gtt is empty and we're still having trouble
2544                  * fitting our object in, we're out of memory.
2545                  */
2546                 ret = i915_gem_evict_something(dev, size, alignment,
2547                                                map_and_fenceable);
2548                 if (ret)
2549                         return ret;
2550
2551                 goto search_free;
2552         }
2553
2554         ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
2555         if (ret) {
2556                 drm_mm_put_block(obj->gtt_space);
2557                 obj->gtt_space = NULL;
2558
2559                 if (ret == -ENOMEM) {
2560                         /* first try to reclaim some memory by clearing the GTT */
2561                         ret = i915_gem_evict_everything(dev, false);
2562                         if (ret) {
2563                                 /* now try to shrink everyone else */
2564                                 if (gfpmask) {
2565                                         gfpmask = 0;
2566                                         goto search_free;
2567                                 }
2568
2569                                 return -ENOMEM;
2570                         }
2571
2572                         goto search_free;
2573                 }
2574
2575                 return ret;
2576         }
2577
2578         ret = i915_gem_gtt_prepare_object(obj);
2579         if (ret) {
2580                 i915_gem_object_put_pages_gtt(obj);
2581                 drm_mm_put_block(obj->gtt_space);
2582                 obj->gtt_space = NULL;
2583
2584                 if (i915_gem_evict_everything(dev, false))
2585                         return ret;
2586
2587                 goto search_free;
2588         }
2589
2590         if (!dev_priv->mm.aliasing_ppgtt)
2591                 i915_gem_gtt_bind_object(obj, obj->cache_level);
2592
2593         list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
2594         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2595
2596         /* Assert that the object is not currently in any GPU domain. As it
2597          * wasn't in the GTT, there shouldn't be any way it could have been in
2598          * a GPU cache
2599          */
2600         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2601         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2602
2603         obj->gtt_offset = obj->gtt_space->start;
2604
2605         fenceable =
2606                 obj->gtt_space->size == fence_size &&
2607                 (obj->gtt_space->start & (fence_alignment - 1)) == 0;
2608
2609         mappable =
2610                 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
2611
2612         obj->map_and_fenceable = mappable && fenceable;
2613
2614         trace_i915_gem_object_bind(obj, map_and_fenceable);
2615         return 0;
2616 }
2617
2618 void
2619 i915_gem_clflush_object(struct drm_i915_gem_object *obj)
2620 {
2621         /* If we don't have a page list set up, then we're not pinned
2622          * to GPU, and we can ignore the cache flush because it'll happen
2623          * again at bind time.
2624          */
2625         if (obj->pages == NULL)
2626                 return;
2627
2628         /* If the GPU is snooping the contents of the CPU cache,
2629          * we do not need to manually clear the CPU cache lines.  However,
2630          * the caches are only snooped when the render cache is
2631          * flushed/invalidated.  As we always have to emit invalidations
2632          * and flushes when moving into and out of the RENDER domain, correct
2633          * snooping behaviour occurs naturally as the result of our domain
2634          * tracking.
2635          */
2636         if (obj->cache_level != I915_CACHE_NONE)
2637                 return;
2638
2639         trace_i915_gem_object_clflush(obj);
2640
2641         drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
2642 }
2643
2644 /** Flushes any GPU write domain for the object if it's dirty. */
2645 static int
2646 i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
2647 {
2648         if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
2649                 return 0;
2650
2651         /* Queue the GPU write cache flushing we need. */
2652         return i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
2653 }
2654
2655 /** Flushes the GTT write domain for the object if it's dirty. */
2656 static void
2657 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
2658 {
2659         uint32_t old_write_domain;
2660
2661         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
2662                 return;
2663
2664         /* No actual flushing is required for the GTT write domain.  Writes
2665          * to it immediately go to main memory as far as we know, so there's
2666          * no chipset flush.  It also doesn't land in render cache.
2667          *
2668          * However, we do have to enforce the order so that all writes through
2669          * the GTT land before any writes to the device, such as updates to
2670          * the GATT itself.
2671          */
2672         wmb();
2673
2674         old_write_domain = obj->base.write_domain;
2675         obj->base.write_domain = 0;
2676
2677         trace_i915_gem_object_change_domain(obj,
2678                                             obj->base.read_domains,
2679                                             old_write_domain);
2680 }
2681
2682 /** Flushes the CPU write domain for the object if it's dirty. */
2683 static void
2684 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
2685 {
2686         uint32_t old_write_domain;
2687
2688         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
2689                 return;
2690
2691         i915_gem_clflush_object(obj);
2692         intel_gtt_chipset_flush();
2693         old_write_domain = obj->base.write_domain;
2694         obj->base.write_domain = 0;
2695
2696         trace_i915_gem_object_change_domain(obj,
2697                                             obj->base.read_domains,
2698                                             old_write_domain);
2699 }
2700
2701 /**
2702  * Moves a single object to the GTT read, and possibly write domain.
2703  *
2704  * This function returns when the move is complete, including waiting on
2705  * flushes to occur.
2706  */
2707 int
2708 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
2709 {
2710         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2711         uint32_t old_write_domain, old_read_domains;
2712         int ret;
2713
2714         /* Not valid to be called on unbound objects. */
2715         if (obj->gtt_space == NULL)
2716                 return -EINVAL;
2717
2718         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
2719                 return 0;
2720
2721         ret = i915_gem_object_flush_gpu_write_domain(obj);
2722         if (ret)
2723                 return ret;
2724
2725         if (obj->pending_gpu_write || write) {
2726                 ret = i915_gem_object_wait_rendering(obj);
2727                 if (ret)
2728                         return ret;
2729         }
2730
2731         i915_gem_object_flush_cpu_write_domain(obj);
2732
2733         old_write_domain = obj->base.write_domain;
2734         old_read_domains = obj->base.read_domains;
2735
2736         /* It should now be out of any other write domains, and we can update
2737          * the domain values for our changes.
2738          */
2739         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2740         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2741         if (write) {
2742                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
2743                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
2744                 obj->dirty = 1;
2745         }
2746
2747         trace_i915_gem_object_change_domain(obj,
2748                                             old_read_domains,
2749                                             old_write_domain);
2750
2751         /* And bump the LRU for this access */
2752         if (i915_gem_object_is_inactive(obj))
2753                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2754
2755         return 0;
2756 }
2757
2758 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
2759                                     enum i915_cache_level cache_level)
2760 {
2761         struct drm_device *dev = obj->base.dev;
2762         drm_i915_private_t *dev_priv = dev->dev_private;
2763         int ret;
2764
2765         if (obj->cache_level == cache_level)
2766                 return 0;
2767
2768         if (obj->pin_count) {
2769                 DRM_DEBUG("can not change the cache level of pinned objects\n");
2770                 return -EBUSY;
2771         }
2772
2773         if (obj->gtt_space) {
2774                 ret = i915_gem_object_finish_gpu(obj);
2775                 if (ret)
2776                         return ret;
2777
2778                 i915_gem_object_finish_gtt(obj);
2779
2780                 /* Before SandyBridge, you could not use tiling or fence
2781                  * registers with snooped memory, so relinquish any fences
2782                  * currently pointing to our region in the aperture.
2783                  */
2784                 if (INTEL_INFO(obj->base.dev)->gen < 6) {
2785                         ret = i915_gem_object_put_fence(obj);
2786                         if (ret)
2787                                 return ret;
2788                 }
2789
2790                 if (obj->has_global_gtt_mapping)
2791                         i915_gem_gtt_bind_object(obj, cache_level);
2792                 if (obj->has_aliasing_ppgtt_mapping)
2793                         i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
2794                                                obj, cache_level);
2795         }
2796
2797         if (cache_level == I915_CACHE_NONE) {
2798                 u32 old_read_domains, old_write_domain;
2799
2800                 /* If we're coming from LLC cached, then we haven't
2801                  * actually been tracking whether the data is in the
2802                  * CPU cache or not, since we only allow one bit set
2803                  * in obj->write_domain and have been skipping the clflushes.
2804                  * Just set it to the CPU cache for now.
2805                  */
2806                 WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
2807                 WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
2808
2809                 old_read_domains = obj->base.read_domains;
2810                 old_write_domain = obj->base.write_domain;
2811
2812                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2813                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2814
2815                 trace_i915_gem_object_change_domain(obj,
2816                                                     old_read_domains,
2817                                                     old_write_domain);
2818         }
2819
2820         obj->cache_level = cache_level;
2821         return 0;
2822 }
2823
2824 /*
2825  * Prepare buffer for display plane (scanout, cursors, etc).
2826  * Can be called from an uninterruptible phase (modesetting) and allows
2827  * any flushes to be pipelined (for pageflips).
2828  */
2829 int
2830 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
2831                                      u32 alignment,
2832                                      struct intel_ring_buffer *pipelined)
2833 {
2834         u32 old_read_domains, old_write_domain;
2835         int ret;
2836
2837         ret = i915_gem_object_flush_gpu_write_domain(obj);
2838         if (ret)
2839                 return ret;
2840
2841         if (pipelined != obj->ring) {
2842                 ret = i915_gem_object_sync(obj, pipelined);
2843                 if (ret)
2844                         return ret;
2845         }
2846
2847         /* The display engine is not coherent with the LLC cache on gen6.  As
2848          * a result, we make sure that the pinning that is about to occur is
2849          * done with uncached PTEs. This is lowest common denominator for all
2850          * chipsets.
2851          *
2852          * However for gen6+, we could do better by using the GFDT bit instead
2853          * of uncaching, which would allow us to flush all the LLC-cached data
2854          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
2855          */
2856         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
2857         if (ret)
2858                 return ret;
2859
2860         /* As the user may map the buffer once pinned in the display plane
2861          * (e.g. libkms for the bootup splash), we have to ensure that we
2862          * always use map_and_fenceable for all scanout buffers.
2863          */
2864         ret = i915_gem_object_pin(obj, alignment, true);
2865         if (ret)
2866                 return ret;
2867
2868         i915_gem_object_flush_cpu_write_domain(obj);
2869
2870         old_write_domain = obj->base.write_domain;
2871         old_read_domains = obj->base.read_domains;
2872
2873         /* It should now be out of any other write domains, and we can update
2874          * the domain values for our changes.
2875          */
2876         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2877         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2878
2879         trace_i915_gem_object_change_domain(obj,
2880                                             old_read_domains,
2881                                             old_write_domain);
2882
2883         return 0;
2884 }
2885
2886 int
2887 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
2888 {
2889         int ret;
2890
2891         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
2892                 return 0;
2893
2894         if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2895                 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
2896                 if (ret)
2897                         return ret;
2898         }
2899
2900         ret = i915_gem_object_wait_rendering(obj);
2901         if (ret)
2902                 return ret;
2903
2904         /* Ensure that we invalidate the GPU's caches and TLBs. */
2905         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
2906         return 0;
2907 }
2908
2909 /**
2910  * Moves a single object to the CPU read, and possibly write domain.
2911  *
2912  * This function returns when the move is complete, including waiting on
2913  * flushes to occur.
2914  */
2915 int
2916 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
2917 {
2918         uint32_t old_write_domain, old_read_domains;
2919         int ret;
2920
2921         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
2922                 return 0;
2923
2924         ret = i915_gem_object_flush_gpu_write_domain(obj);
2925         if (ret)
2926                 return ret;
2927
2928         if (write || obj->pending_gpu_write) {
2929                 ret = i915_gem_object_wait_rendering(obj);
2930                 if (ret)
2931                         return ret;
2932         }
2933
2934         i915_gem_object_flush_gtt_write_domain(obj);
2935
2936         old_write_domain = obj->base.write_domain;
2937         old_read_domains = obj->base.read_domains;
2938
2939         /* Flush the CPU cache if it's still invalid. */
2940         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2941                 i915_gem_clflush_object(obj);
2942
2943                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
2944         }
2945
2946         /* It should now be out of any other write domains, and we can update
2947          * the domain values for our changes.
2948          */
2949         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2950
2951         /* If we're writing through the CPU, then the GPU read domains will
2952          * need to be invalidated at next use.
2953          */
2954         if (write) {
2955                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2956                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2957         }
2958
2959         trace_i915_gem_object_change_domain(obj,
2960                                             old_read_domains,
2961                                             old_write_domain);
2962
2963         return 0;
2964 }
2965
2966 /* Throttle our rendering by waiting until the ring has completed our requests
2967  * emitted over 20 msec ago.
2968  *
2969  * Note that if we were to use the current jiffies each time around the loop,
2970  * we wouldn't escape the function with any frames outstanding if the time to
2971  * render a frame was over 20ms.
2972  *
2973  * This should get us reasonable parallelism between CPU and GPU but also
2974  * relatively low latency when blocking on a particular request to finish.
2975  */
2976 static int
2977 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
2978 {
2979         struct drm_i915_private *dev_priv = dev->dev_private;
2980         struct drm_i915_file_private *file_priv = file->driver_priv;
2981         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
2982         struct drm_i915_gem_request *request;
2983         struct intel_ring_buffer *ring = NULL;
2984         u32 seqno = 0;
2985         int ret;
2986
2987         if (atomic_read(&dev_priv->mm.wedged))
2988                 return -EIO;
2989
2990         spin_lock(&file_priv->mm.lock);
2991         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
2992                 if (time_after_eq(request->emitted_jiffies, recent_enough))
2993                         break;
2994
2995                 ring = request->ring;
2996                 seqno = request->seqno;
2997         }
2998         spin_unlock(&file_priv->mm.lock);
2999
3000         if (seqno == 0)
3001                 return 0;
3002
3003         ret = __wait_seqno(ring, seqno, true);
3004         if (ret == 0)
3005                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3006
3007         return ret;
3008 }
3009
3010 int
3011 i915_gem_object_pin(struct drm_i915_gem_object *obj,
3012                     uint32_t alignment,
3013                     bool map_and_fenceable)
3014 {
3015         int ret;
3016
3017         BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
3018
3019         if (obj->gtt_space != NULL) {
3020                 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
3021                     (map_and_fenceable && !obj->map_and_fenceable)) {
3022                         WARN(obj->pin_count,
3023                              "bo is already pinned with incorrect alignment:"
3024                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
3025                              " obj->map_and_fenceable=%d\n",
3026                              obj->gtt_offset, alignment,
3027                              map_and_fenceable,
3028                              obj->map_and_fenceable);
3029                         ret = i915_gem_object_unbind(obj);
3030                         if (ret)
3031                                 return ret;
3032                 }
3033         }
3034
3035         if (obj->gtt_space == NULL) {
3036                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
3037                                                   map_and_fenceable);
3038                 if (ret)
3039                         return ret;
3040         }
3041
3042         if (!obj->has_global_gtt_mapping && map_and_fenceable)
3043                 i915_gem_gtt_bind_object(obj, obj->cache_level);
3044
3045         obj->pin_count++;
3046         obj->pin_mappable |= map_and_fenceable;
3047
3048         return 0;
3049 }
3050
3051 void
3052 i915_gem_object_unpin(struct drm_i915_gem_object *obj)
3053 {
3054         BUG_ON(obj->pin_count == 0);
3055         BUG_ON(obj->gtt_space == NULL);
3056
3057         if (--obj->pin_count == 0)
3058                 obj->pin_mappable = false;
3059 }
3060
3061 int
3062 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
3063                    struct drm_file *file)
3064 {
3065         struct drm_i915_gem_pin *args = data;
3066         struct drm_i915_gem_object *obj;
3067         int ret;
3068
3069         ret = i915_mutex_lock_interruptible(dev);
3070         if (ret)
3071                 return ret;
3072
3073         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3074         if (&obj->base == NULL) {
3075                 ret = -ENOENT;
3076                 goto unlock;
3077         }
3078
3079         if (obj->madv != I915_MADV_WILLNEED) {
3080                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
3081                 ret = -EINVAL;
3082                 goto out;
3083         }
3084
3085         if (obj->pin_filp != NULL && obj->pin_filp != file) {
3086                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
3087                           args->handle);
3088                 ret = -EINVAL;
3089                 goto out;
3090         }
3091
3092         obj->user_pin_count++;
3093         obj->pin_filp = file;
3094         if (obj->user_pin_count == 1) {
3095                 ret = i915_gem_object_pin(obj, args->alignment, true);
3096                 if (ret)
3097                         goto out;
3098         }
3099
3100         /* XXX - flush the CPU caches for pinned objects
3101          * as the X server doesn't manage domains yet
3102          */
3103         i915_gem_object_flush_cpu_write_domain(obj);
3104         args->offset = obj->gtt_offset;
3105 out:
3106         drm_gem_object_unreference(&obj->base);
3107 unlock:
3108         mutex_unlock(&dev->struct_mutex);
3109         return ret;
3110 }
3111
3112 int
3113 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
3114                      struct drm_file *file)
3115 {
3116         struct drm_i915_gem_pin *args = data;
3117         struct drm_i915_gem_object *obj;
3118         int ret;
3119
3120         ret = i915_mutex_lock_interruptible(dev);
3121         if (ret)
3122                 return ret;
3123
3124         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3125         if (&obj->base == NULL) {
3126                 ret = -ENOENT;
3127                 goto unlock;
3128         }
3129
3130         if (obj->pin_filp != file) {
3131                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
3132                           args->handle);
3133                 ret = -EINVAL;
3134                 goto out;
3135         }
3136         obj->user_pin_count--;
3137         if (obj->user_pin_count == 0) {
3138                 obj->pin_filp = NULL;
3139                 i915_gem_object_unpin(obj);
3140         }
3141
3142 out:
3143         drm_gem_object_unreference(&obj->base);
3144 unlock:
3145         mutex_unlock(&dev->struct_mutex);
3146         return ret;
3147 }
3148
3149 int
3150 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3151                     struct drm_file *file)
3152 {
3153         struct drm_i915_gem_busy *args = data;
3154         struct drm_i915_gem_object *obj;
3155         int ret;
3156
3157         ret = i915_mutex_lock_interruptible(dev);
3158         if (ret)
3159                 return ret;
3160
3161         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3162         if (&obj->base == NULL) {
3163                 ret = -ENOENT;
3164                 goto unlock;
3165         }
3166
3167         /* Count all active objects as busy, even if they are currently not used
3168          * by the gpu. Users of this interface expect objects to eventually
3169          * become non-busy without any further actions, therefore emit any
3170          * necessary flushes here.
3171          */
3172         args->busy = obj->active;
3173         if (args->busy) {
3174                 /* Unconditionally flush objects, even when the gpu still uses this
3175                  * object. Userspace calling this function indicates that it wants to
3176                  * use this buffer rather sooner than later, so issuing the required
3177                  * flush earlier is beneficial.
3178                  */
3179                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
3180                         ret = i915_gem_flush_ring(obj->ring,
3181                                                   0, obj->base.write_domain);
3182                 } else {
3183                         ret = i915_gem_check_olr(obj->ring,
3184                                                  obj->last_rendering_seqno);
3185                 }
3186
3187                 /* Update the active list for the hardware's current position.
3188                  * Otherwise this only updates on a delayed timer or when irqs
3189                  * are actually unmasked, and our working set ends up being
3190                  * larger than required.
3191                  */
3192                 i915_gem_retire_requests_ring(obj->ring);
3193
3194                 args->busy = obj->active;
3195         }
3196
3197         drm_gem_object_unreference(&obj->base);
3198 unlock:
3199         mutex_unlock(&dev->struct_mutex);
3200         return ret;
3201 }
3202
3203 int
3204 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
3205                         struct drm_file *file_priv)
3206 {
3207         return i915_gem_ring_throttle(dev, file_priv);
3208 }
3209
3210 int
3211 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
3212                        struct drm_file *file_priv)
3213 {
3214         struct drm_i915_gem_madvise *args = data;
3215         struct drm_i915_gem_object *obj;
3216         int ret;
3217
3218         switch (args->madv) {
3219         case I915_MADV_DONTNEED:
3220         case I915_MADV_WILLNEED:
3221             break;
3222         default:
3223             return -EINVAL;
3224         }
3225
3226         ret = i915_mutex_lock_interruptible(dev);
3227         if (ret)
3228                 return ret;
3229
3230         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
3231         if (&obj->base == NULL) {
3232                 ret = -ENOENT;
3233                 goto unlock;
3234         }
3235
3236         if (obj->pin_count) {
3237                 ret = -EINVAL;
3238                 goto out;
3239         }
3240
3241         if (obj->madv != __I915_MADV_PURGED)
3242                 obj->madv = args->madv;
3243
3244         /* if the object is no longer bound, discard its backing storage */
3245         if (i915_gem_object_is_purgeable(obj) &&
3246             obj->gtt_space == NULL)
3247                 i915_gem_object_truncate(obj);
3248
3249         args->retained = obj->madv != __I915_MADV_PURGED;
3250
3251 out:
3252         drm_gem_object_unreference(&obj->base);
3253 unlock:
3254         mutex_unlock(&dev->struct_mutex);
3255         return ret;
3256 }
3257
3258 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
3259                                                   size_t size)
3260 {
3261         struct drm_i915_private *dev_priv = dev->dev_private;
3262         struct drm_i915_gem_object *obj;
3263         struct address_space *mapping;
3264         u32 mask;
3265
3266         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
3267         if (obj == NULL)
3268                 return NULL;
3269
3270         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
3271                 kfree(obj);
3272                 return NULL;
3273         }
3274
3275         mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
3276         if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
3277                 /* 965gm cannot relocate objects above 4GiB. */
3278                 mask &= ~__GFP_HIGHMEM;
3279                 mask |= __GFP_DMA32;
3280         }
3281
3282         mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
3283         mapping_set_gfp_mask(mapping, mask);
3284
3285         i915_gem_info_add_obj(dev_priv, size);
3286
3287         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3288         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3289
3290         if (HAS_LLC(dev)) {
3291                 /* On some devices, we can have the GPU use the LLC (the CPU
3292                  * cache) for about a 10% performance improvement
3293                  * compared to uncached.  Graphics requests other than
3294                  * display scanout are coherent with the CPU in
3295                  * accessing this cache.  This means in this mode we
3296                  * don't need to clflush on the CPU side, and on the
3297                  * GPU side we only need to flush internal caches to
3298                  * get data visible to the CPU.
3299                  *
3300                  * However, we maintain the display planes as UC, and so
3301                  * need to rebind when first used as such.
3302                  */
3303                 obj->cache_level = I915_CACHE_LLC;
3304         } else
3305                 obj->cache_level = I915_CACHE_NONE;
3306
3307         obj->base.driver_private = NULL;
3308         obj->fence_reg = I915_FENCE_REG_NONE;
3309         INIT_LIST_HEAD(&obj->mm_list);
3310         INIT_LIST_HEAD(&obj->gtt_list);
3311         INIT_LIST_HEAD(&obj->ring_list);
3312         INIT_LIST_HEAD(&obj->exec_list);
3313         INIT_LIST_HEAD(&obj->gpu_write_list);
3314         obj->madv = I915_MADV_WILLNEED;
3315         /* Avoid an unnecessary call to unbind on the first bind. */
3316         obj->map_and_fenceable = true;
3317
3318         return obj;
3319 }
3320
3321 int i915_gem_init_object(struct drm_gem_object *obj)
3322 {
3323         BUG();
3324
3325         return 0;
3326 }
3327
3328 void i915_gem_free_object(struct drm_gem_object *gem_obj)
3329 {
3330         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
3331         struct drm_device *dev = obj->base.dev;
3332         drm_i915_private_t *dev_priv = dev->dev_private;
3333
3334         trace_i915_gem_object_destroy(obj);
3335
3336         if (obj->phys_obj)
3337                 i915_gem_detach_phys_object(dev, obj);
3338
3339         obj->pin_count = 0;
3340         if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
3341                 bool was_interruptible;
3342
3343                 was_interruptible = dev_priv->mm.interruptible;
3344                 dev_priv->mm.interruptible = false;
3345
3346                 WARN_ON(i915_gem_object_unbind(obj));
3347
3348                 dev_priv->mm.interruptible = was_interruptible;
3349         }
3350
3351         if (obj->base.map_list.map)
3352                 drm_gem_free_mmap_offset(&obj->base);
3353
3354         drm_gem_object_release(&obj->base);
3355         i915_gem_info_remove_obj(dev_priv, obj->base.size);
3356
3357         kfree(obj->bit_17);
3358         kfree(obj);
3359 }
3360
3361 int
3362 i915_gem_idle(struct drm_device *dev)
3363 {
3364         drm_i915_private_t *dev_priv = dev->dev_private;
3365         int ret;
3366
3367         mutex_lock(&dev->struct_mutex);
3368
3369         if (dev_priv->mm.suspended) {
3370                 mutex_unlock(&dev->struct_mutex);
3371                 return 0;
3372         }
3373
3374         ret = i915_gpu_idle(dev);
3375         if (ret) {
3376                 mutex_unlock(&dev->struct_mutex);
3377                 return ret;
3378         }
3379         i915_gem_retire_requests(dev);
3380
3381         /* Under UMS, be paranoid and evict. */
3382         if (!drm_core_check_feature(dev, DRIVER_MODESET))
3383                 i915_gem_evict_everything(dev, false);
3384
3385         i915_gem_reset_fences(dev);
3386
3387         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
3388          * We need to replace this with a semaphore, or something.
3389          * And not confound mm.suspended!
3390          */
3391         dev_priv->mm.suspended = 1;
3392         del_timer_sync(&dev_priv->hangcheck_timer);
3393
3394         i915_kernel_lost_context(dev);
3395         i915_gem_cleanup_ringbuffer(dev);
3396
3397         mutex_unlock(&dev->struct_mutex);
3398
3399         /* Cancel the retire work handler, which should be idle now. */
3400         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3401
3402         return 0;
3403 }
3404
3405 void i915_gem_init_swizzling(struct drm_device *dev)
3406 {
3407         drm_i915_private_t *dev_priv = dev->dev_private;
3408
3409         if (INTEL_INFO(dev)->gen < 5 ||
3410             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
3411                 return;
3412
3413         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
3414                                  DISP_TILE_SURFACE_SWIZZLING);
3415
3416         if (IS_GEN5(dev))
3417                 return;
3418
3419         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
3420         if (IS_GEN6(dev))
3421                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
3422         else
3423                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
3424 }
3425
3426 void i915_gem_init_ppgtt(struct drm_device *dev)
3427 {
3428         drm_i915_private_t *dev_priv = dev->dev_private;
3429         uint32_t pd_offset;
3430         struct intel_ring_buffer *ring;
3431         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
3432         uint32_t __iomem *pd_addr;
3433         uint32_t pd_entry;
3434         int i;
3435
3436         if (!dev_priv->mm.aliasing_ppgtt)
3437                 return;
3438
3439
3440         pd_addr = dev_priv->mm.gtt->gtt + ppgtt->pd_offset/sizeof(uint32_t);
3441         for (i = 0; i < ppgtt->num_pd_entries; i++) {
3442                 dma_addr_t pt_addr;
3443
3444                 if (dev_priv->mm.gtt->needs_dmar)
3445                         pt_addr = ppgtt->pt_dma_addr[i];
3446                 else
3447                         pt_addr = page_to_phys(ppgtt->pt_pages[i]);
3448
3449                 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
3450                 pd_entry |= GEN6_PDE_VALID;
3451
3452                 writel(pd_entry, pd_addr + i);
3453         }
3454         readl(pd_addr);
3455
3456         pd_offset = ppgtt->pd_offset;
3457         pd_offset /= 64; /* in cachelines, */
3458         pd_offset <<= 16;
3459
3460         if (INTEL_INFO(dev)->gen == 6) {
3461                 uint32_t ecochk, gab_ctl, ecobits;
3462
3463                 ecobits = I915_READ(GAC_ECO_BITS);
3464                 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
3465
3466                 gab_ctl = I915_READ(GAB_CTL);
3467                 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
3468
3469                 ecochk = I915_READ(GAM_ECOCHK);
3470                 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
3471                                        ECOCHK_PPGTT_CACHE64B);
3472                 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
3473         } else if (INTEL_INFO(dev)->gen >= 7) {
3474                 I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B);
3475                 /* GFX_MODE is per-ring on gen7+ */
3476         }
3477
3478         for_each_ring(ring, dev_priv, i) {
3479                 if (INTEL_INFO(dev)->gen >= 7)
3480                         I915_WRITE(RING_MODE_GEN7(ring),
3481                                    _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
3482
3483                 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
3484                 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
3485         }
3486 }
3487
3488 int
3489 i915_gem_init_hw(struct drm_device *dev)
3490 {
3491         drm_i915_private_t *dev_priv = dev->dev_private;
3492         int ret;
3493
3494         i915_gem_init_swizzling(dev);
3495
3496         ret = intel_init_render_ring_buffer(dev);
3497         if (ret)
3498                 return ret;
3499
3500         if (HAS_BSD(dev)) {
3501                 ret = intel_init_bsd_ring_buffer(dev);
3502                 if (ret)
3503                         goto cleanup_render_ring;
3504         }
3505
3506         if (HAS_BLT(dev)) {
3507                 ret = intel_init_blt_ring_buffer(dev);
3508                 if (ret)
3509                         goto cleanup_bsd_ring;
3510         }
3511
3512         dev_priv->next_seqno = 1;
3513
3514         i915_gem_init_ppgtt(dev);
3515
3516         return 0;
3517
3518 cleanup_bsd_ring:
3519         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
3520 cleanup_render_ring:
3521         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
3522         return ret;
3523 }
3524
3525 static bool
3526 intel_enable_ppgtt(struct drm_device *dev)
3527 {
3528         if (i915_enable_ppgtt >= 0)
3529                 return i915_enable_ppgtt;
3530
3531 #ifdef CONFIG_INTEL_IOMMU
3532         /* Disable ppgtt on SNB if VT-d is on. */
3533         if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
3534                 return false;
3535 #endif
3536
3537         return true;
3538 }
3539
3540 int i915_gem_init(struct drm_device *dev)
3541 {
3542         struct drm_i915_private *dev_priv = dev->dev_private;
3543         unsigned long gtt_size, mappable_size;
3544         int ret;
3545
3546         gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
3547         mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
3548
3549         mutex_lock(&dev->struct_mutex);
3550         if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
3551                 /* PPGTT pdes are stolen from global gtt ptes, so shrink the
3552                  * aperture accordingly when using aliasing ppgtt. */
3553                 gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
3554
3555                 i915_gem_init_global_gtt(dev, 0, mappable_size, gtt_size);
3556
3557                 ret = i915_gem_init_aliasing_ppgtt(dev);
3558                 if (ret) {
3559                         mutex_unlock(&dev->struct_mutex);
3560                         return ret;
3561                 }
3562         } else {
3563                 /* Let GEM Manage all of the aperture.
3564                  *
3565                  * However, leave one page at the end still bound to the scratch
3566                  * page.  There are a number of places where the hardware
3567                  * apparently prefetches past the end of the object, and we've
3568                  * seen multiple hangs with the GPU head pointer stuck in a
3569                  * batchbuffer bound at the last page of the aperture.  One page
3570                  * should be enough to keep any prefetching inside of the
3571                  * aperture.
3572                  */
3573                 i915_gem_init_global_gtt(dev, 0, mappable_size,
3574                                          gtt_size);
3575         }
3576
3577         ret = i915_gem_init_hw(dev);
3578         mutex_unlock(&dev->struct_mutex);
3579         if (ret) {
3580                 i915_gem_cleanup_aliasing_ppgtt(dev);
3581                 return ret;
3582         }
3583
3584         /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */
3585         if (!drm_core_check_feature(dev, DRIVER_MODESET))
3586                 dev_priv->dri1.allow_batchbuffer = 1;
3587         return 0;
3588 }
3589
3590 void
3591 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
3592 {
3593         drm_i915_private_t *dev_priv = dev->dev_private;
3594         struct intel_ring_buffer *ring;
3595         int i;
3596
3597         for_each_ring(ring, dev_priv, i)
3598                 intel_cleanup_ring_buffer(ring);
3599 }
3600
3601 int
3602 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
3603                        struct drm_file *file_priv)
3604 {
3605         drm_i915_private_t *dev_priv = dev->dev_private;
3606         int ret;
3607
3608         if (drm_core_check_feature(dev, DRIVER_MODESET))
3609                 return 0;
3610
3611         if (atomic_read(&dev_priv->mm.wedged)) {
3612                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
3613                 atomic_set(&dev_priv->mm.wedged, 0);
3614         }
3615
3616         mutex_lock(&dev->struct_mutex);
3617         dev_priv->mm.suspended = 0;
3618
3619         ret = i915_gem_init_hw(dev);
3620         if (ret != 0) {
3621                 mutex_unlock(&dev->struct_mutex);
3622                 return ret;
3623         }
3624
3625         BUG_ON(!list_empty(&dev_priv->mm.active_list));
3626         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
3627         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
3628         mutex_unlock(&dev->struct_mutex);
3629
3630         ret = drm_irq_install(dev);
3631         if (ret)
3632                 goto cleanup_ringbuffer;
3633
3634         return 0;
3635
3636 cleanup_ringbuffer:
3637         mutex_lock(&dev->struct_mutex);
3638         i915_gem_cleanup_ringbuffer(dev);
3639         dev_priv->mm.suspended = 1;
3640         mutex_unlock(&dev->struct_mutex);
3641
3642         return ret;
3643 }
3644
3645 int
3646 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
3647                        struct drm_file *file_priv)
3648 {
3649         if (drm_core_check_feature(dev, DRIVER_MODESET))
3650                 return 0;
3651
3652         drm_irq_uninstall(dev);
3653         return i915_gem_idle(dev);
3654 }
3655
3656 void
3657 i915_gem_lastclose(struct drm_device *dev)
3658 {
3659         int ret;
3660
3661         if (drm_core_check_feature(dev, DRIVER_MODESET))
3662                 return;
3663
3664         ret = i915_gem_idle(dev);
3665         if (ret)
3666                 DRM_ERROR("failed to idle hardware: %d\n", ret);
3667 }
3668
3669 static void
3670 init_ring_lists(struct intel_ring_buffer *ring)
3671 {
3672         INIT_LIST_HEAD(&ring->active_list);
3673         INIT_LIST_HEAD(&ring->request_list);
3674         INIT_LIST_HEAD(&ring->gpu_write_list);
3675 }
3676
3677 void
3678 i915_gem_load(struct drm_device *dev)
3679 {
3680         int i;
3681         drm_i915_private_t *dev_priv = dev->dev_private;
3682
3683         INIT_LIST_HEAD(&dev_priv->mm.active_list);
3684         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
3685         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
3686         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
3687         INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
3688         for (i = 0; i < I915_NUM_RINGS; i++)
3689                 init_ring_lists(&dev_priv->ring[i]);
3690         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
3691                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
3692         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
3693                           i915_gem_retire_work_handler);
3694         init_completion(&dev_priv->error_completion);
3695
3696         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
3697         if (IS_GEN3(dev)) {
3698                 I915_WRITE(MI_ARB_STATE,
3699                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
3700         }
3701
3702         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
3703
3704         /* Old X drivers will take 0-2 for front, back, depth buffers */
3705         if (!drm_core_check_feature(dev, DRIVER_MODESET))
3706                 dev_priv->fence_reg_start = 3;
3707
3708         if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
3709                 dev_priv->num_fence_regs = 16;
3710         else
3711                 dev_priv->num_fence_regs = 8;
3712
3713         /* Initialize fence registers to zero */
3714         i915_gem_reset_fences(dev);
3715
3716         i915_gem_detect_bit_6_swizzle(dev);
3717         init_waitqueue_head(&dev_priv->pending_flip_queue);
3718
3719         dev_priv->mm.interruptible = true;
3720
3721         dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
3722         dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
3723         register_shrinker(&dev_priv->mm.inactive_shrinker);
3724 }
3725
3726 /*
3727  * Create a physically contiguous memory object for this object
3728  * e.g. for cursor + overlay regs
3729  */
3730 static int i915_gem_init_phys_object(struct drm_device *dev,
3731                                      int id, int size, int align)
3732 {
3733         drm_i915_private_t *dev_priv = dev->dev_private;
3734         struct drm_i915_gem_phys_object *phys_obj;
3735         int ret;
3736
3737         if (dev_priv->mm.phys_objs[id - 1] || !size)
3738                 return 0;
3739
3740         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
3741         if (!phys_obj)
3742                 return -ENOMEM;
3743
3744         phys_obj->id = id;
3745
3746         phys_obj->handle = drm_pci_alloc(dev, size, align);
3747         if (!phys_obj->handle) {
3748                 ret = -ENOMEM;
3749                 goto kfree_obj;
3750         }
3751 #ifdef CONFIG_X86
3752         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
3753 #endif
3754
3755         dev_priv->mm.phys_objs[id - 1] = phys_obj;
3756
3757         return 0;
3758 kfree_obj:
3759         kfree(phys_obj);
3760         return ret;
3761 }
3762
3763 static void i915_gem_free_phys_object(struct drm_device *dev, int id)
3764 {
3765         drm_i915_private_t *dev_priv = dev->dev_private;
3766         struct drm_i915_gem_phys_object *phys_obj;
3767
3768         if (!dev_priv->mm.phys_objs[id - 1])
3769                 return;
3770
3771         phys_obj = dev_priv->mm.phys_objs[id - 1];
3772         if (phys_obj->cur_obj) {
3773                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
3774         }
3775
3776 #ifdef CONFIG_X86
3777         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
3778 #endif
3779         drm_pci_free(dev, phys_obj->handle);
3780         kfree(phys_obj);
3781         dev_priv->mm.phys_objs[id - 1] = NULL;
3782 }
3783
3784 void i915_gem_free_all_phys_object(struct drm_device *dev)
3785 {
3786         int i;
3787
3788         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
3789                 i915_gem_free_phys_object(dev, i);
3790 }
3791
3792 void i915_gem_detach_phys_object(struct drm_device *dev,
3793                                  struct drm_i915_gem_object *obj)
3794 {
3795         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
3796         char *vaddr;
3797         int i;
3798         int page_count;
3799
3800         if (!obj->phys_obj)
3801                 return;
3802         vaddr = obj->phys_obj->handle->vaddr;
3803
3804         page_count = obj->base.size / PAGE_SIZE;
3805         for (i = 0; i < page_count; i++) {
3806                 struct page *page = shmem_read_mapping_page(mapping, i);
3807                 if (!IS_ERR(page)) {
3808                         char *dst = kmap_atomic(page);
3809                         memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
3810                         kunmap_atomic(dst);
3811
3812                         drm_clflush_pages(&page, 1);
3813
3814                         set_page_dirty(page);
3815                         mark_page_accessed(page);
3816                         page_cache_release(page);
3817                 }
3818         }
3819         intel_gtt_chipset_flush();
3820
3821         obj->phys_obj->cur_obj = NULL;
3822         obj->phys_obj = NULL;
3823 }
3824
3825 int
3826 i915_gem_attach_phys_object(struct drm_device *dev,
3827                             struct drm_i915_gem_object *obj,
3828                             int id,
3829                             int align)
3830 {
3831         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
3832         drm_i915_private_t *dev_priv = dev->dev_private;
3833         int ret = 0;
3834         int page_count;
3835         int i;
3836
3837         if (id > I915_MAX_PHYS_OBJECT)
3838                 return -EINVAL;
3839
3840         if (obj->phys_obj) {
3841                 if (obj->phys_obj->id == id)
3842                         return 0;
3843                 i915_gem_detach_phys_object(dev, obj);
3844         }
3845
3846         /* create a new object */
3847         if (!dev_priv->mm.phys_objs[id - 1]) {
3848                 ret = i915_gem_init_phys_object(dev, id,
3849                                                 obj->base.size, align);
3850                 if (ret) {
3851                         DRM_ERROR("failed to init phys object %d size: %zu\n",
3852                                   id, obj->base.size);
3853                         return ret;
3854                 }
3855         }
3856
3857         /* bind to the object */
3858         obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
3859         obj->phys_obj->cur_obj = obj;
3860
3861         page_count = obj->base.size / PAGE_SIZE;
3862
3863         for (i = 0; i < page_count; i++) {
3864                 struct page *page;
3865                 char *dst, *src;
3866
3867                 page = shmem_read_mapping_page(mapping, i);
3868                 if (IS_ERR(page))
3869                         return PTR_ERR(page);
3870
3871                 src = kmap_atomic(page);
3872                 dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
3873                 memcpy(dst, src, PAGE_SIZE);
3874                 kunmap_atomic(src);
3875
3876                 mark_page_accessed(page);
3877                 page_cache_release(page);
3878         }
3879
3880         return 0;
3881 }
3882
3883 static int
3884 i915_gem_phys_pwrite(struct drm_device *dev,
3885                      struct drm_i915_gem_object *obj,
3886                      struct drm_i915_gem_pwrite *args,
3887                      struct drm_file *file_priv)
3888 {
3889         void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
3890         char __user *user_data = (char __user *) (uintptr_t) args->data_ptr;
3891
3892         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
3893                 unsigned long unwritten;
3894
3895                 /* The physical object once assigned is fixed for the lifetime
3896                  * of the obj, so we can safely drop the lock and continue
3897                  * to access vaddr.
3898                  */
3899                 mutex_unlock(&dev->struct_mutex);
3900                 unwritten = copy_from_user(vaddr, user_data, args->size);
3901                 mutex_lock(&dev->struct_mutex);
3902                 if (unwritten)
3903                         return -EFAULT;
3904         }
3905
3906         intel_gtt_chipset_flush();
3907         return 0;
3908 }
3909
3910 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
3911 {
3912         struct drm_i915_file_private *file_priv = file->driver_priv;
3913
3914         /* Clean up our request list when the client is going away, so that
3915          * later retire_requests won't dereference our soon-to-be-gone
3916          * file_priv.
3917          */
3918         spin_lock(&file_priv->mm.lock);
3919         while (!list_empty(&file_priv->mm.request_list)) {
3920                 struct drm_i915_gem_request *request;
3921
3922                 request = list_first_entry(&file_priv->mm.request_list,
3923                                            struct drm_i915_gem_request,
3924                                            client_list);
3925                 list_del(&request->client_list);
3926                 request->file_priv = NULL;
3927         }
3928         spin_unlock(&file_priv->mm.lock);
3929 }
3930
3931 static int
3932 i915_gpu_is_active(struct drm_device *dev)
3933 {
3934         drm_i915_private_t *dev_priv = dev->dev_private;
3935         int lists_empty;
3936
3937         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
3938                       list_empty(&dev_priv->mm.active_list);
3939
3940         return !lists_empty;
3941 }
3942
3943 static int
3944 i915_gem_inactive_shrink(struct shrinker *shrinker, struct shrink_control *sc)
3945 {
3946         struct drm_i915_private *dev_priv =
3947                 container_of(shrinker,
3948                              struct drm_i915_private,
3949                              mm.inactive_shrinker);
3950         struct drm_device *dev = dev_priv->dev;
3951         struct drm_i915_gem_object *obj, *next;
3952         int nr_to_scan = sc->nr_to_scan;
3953         int cnt;
3954
3955         if (!mutex_trylock(&dev->struct_mutex))
3956                 return 0;
3957
3958         /* "fast-path" to count number of available objects */
3959         if (nr_to_scan == 0) {
3960                 cnt = 0;
3961                 list_for_each_entry(obj,
3962                                     &dev_priv->mm.inactive_list,
3963                                     mm_list)
3964                         cnt++;
3965                 mutex_unlock(&dev->struct_mutex);
3966                 return cnt / 100 * sysctl_vfs_cache_pressure;
3967         }
3968
3969 rescan:
3970         /* first scan for clean buffers */
3971         i915_gem_retire_requests(dev);
3972
3973         list_for_each_entry_safe(obj, next,
3974                                  &dev_priv->mm.inactive_list,
3975                                  mm_list) {
3976                 if (i915_gem_object_is_purgeable(obj)) {
3977                         if (i915_gem_object_unbind(obj) == 0 &&
3978                             --nr_to_scan == 0)
3979                                 break;
3980                 }
3981         }
3982
3983         /* second pass, evict/count anything still on the inactive list */
3984         cnt = 0;
3985         list_for_each_entry_safe(obj, next,
3986                                  &dev_priv->mm.inactive_list,
3987                                  mm_list) {
3988                 if (nr_to_scan &&
3989                     i915_gem_object_unbind(obj) == 0)
3990                         nr_to_scan--;
3991                 else
3992                         cnt++;
3993         }
3994
3995         if (nr_to_scan && i915_gpu_is_active(dev)) {
3996                 /*
3997                  * We are desperate for pages, so as a last resort, wait
3998                  * for the GPU to finish and discard whatever we can.
3999                  * This has a dramatic impact to reduce the number of
4000                  * OOM-killer events whilst running the GPU aggressively.
4001                  */
4002                 if (i915_gpu_idle(dev) == 0)
4003                         goto rescan;
4004         }
4005         mutex_unlock(&dev->struct_mutex);
4006         return cnt / 100 * sysctl_vfs_cache_pressure;
4007 }