Pileus Git - ~andy/linux/blob - drivers/gpu/drm/i915/intel_ringbuffer.c

   1 /*
   2  * Copyright © 2008-2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Zou Nan hai <nanhai.zou@intel.com>
  26  *    Xiang Hai hao<haihao.xiang@intel.com>
  27  *
  28  */
  29
  30 #include "drmP.h"
  31 #include "drm.h"
  32 #include "i915_drv.h"
  33 #include "i915_drm.h"
  34 #include "i915_trace.h"
  35 #include "intel_drv.h"
  36
  37 /*
  38  * 965+ support PIPE_CONTROL commands, which provide finer grained control
  39  * over cache flushing.
  40  */
  41 struct pipe_control {
  42         struct drm_i915_gem_object *obj;
  43         volatile u32 *cpu_page;
  44         u32 gtt_offset;
  45 };
  46
  47 static inline int ring_space(struct intel_ring_buffer *ring)
  48 {
  49         int space = (ring->head & HEAD_ADDR) - (ring->tail + 8);
  50         if (space < 0)
  51                 space += ring->size;
  52         return space;
  53 }
  54
  55 static int
  56 gen2_render_ring_flush(struct intel_ring_buffer *ring,
  57                        u32      invalidate_domains,
  58                        u32      flush_domains)
  59 {
  60         u32 cmd;
  61         int ret;
  62
  63         cmd = MI_FLUSH;
  64         if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
  65                 cmd |= MI_NO_WRITE_FLUSH;
  66
  67         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
  68                 cmd |= MI_READ_FLUSH;
  69
  70         ret = intel_ring_begin(ring, 2);
  71         if (ret)
  72                 return ret;
  73
  74         intel_ring_emit(ring, cmd);
  75         intel_ring_emit(ring, MI_NOOP);
  76         intel_ring_advance(ring);
  77
  78         return 0;
  79 }
  80
  81 static int
  82 gen4_render_ring_flush(struct intel_ring_buffer *ring,
  83                        u32      invalidate_domains,
  84                        u32      flush_domains)
  85 {
  86         struct drm_device *dev = ring->dev;
  87         u32 cmd;
  88         int ret;
  89
  90         /*
  91          * read/write caches:
  92          *
  93          * I915_GEM_DOMAIN_RENDER is always invalidated, but is
  94          * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
  95          * also flushed at 2d versus 3d pipeline switches.
  96          *
  97          * read-only caches:
  98          *
  99          * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
 100          * MI_READ_FLUSH is set, and is always flushed on 965.
 101          *
 102          * I915_GEM_DOMAIN_COMMAND may not exist?
 103          *
 104          * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
 105          * invalidated when MI_EXE_FLUSH is set.
 106          *
 107          * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
 108          * invalidated with every MI_FLUSH.
 109          *
 110          * TLBs:
 111          *
 112          * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
 113          * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
 114          * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
 115          * are flushed at any MI_FLUSH.
 116          */
 117
 118         cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
 119         if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)
 120                 cmd &= ~MI_NO_WRITE_FLUSH;
 121         if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
 122                 cmd |= MI_EXE_FLUSH;
 123
 124         if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
 125             (IS_G4X(dev) || IS_GEN5(dev)))
 126                 cmd |= MI_INVALIDATE_ISP;
 127
 128         ret = intel_ring_begin(ring, 2);
 129         if (ret)
 130                 return ret;
 131
 132         intel_ring_emit(ring, cmd);
 133         intel_ring_emit(ring, MI_NOOP);
 134         intel_ring_advance(ring);
 135
 136         return 0;
 137 }
 138
 139 /**
 140  * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
 141  * implementing two workarounds on gen6.  From section 1.4.7.1
 142  * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
 143  *
 144  * [DevSNB-C+{W/A}] Before any depth stall flush (including those
 145  * produced by non-pipelined state commands), software needs to first
 146  * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
 147  * 0.
 148  *
 149  * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
 150  * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
 151  *
 152  * And the workaround for these two requires this workaround first:
 153  *
 154  * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
 155  * BEFORE the pipe-control with a post-sync op and no write-cache
 156  * flushes.
 157  *
 158  * And this last workaround is tricky because of the requirements on
 159  * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
 160  * volume 2 part 1:
 161  *
 162  *     "1 of the following must also be set:
 163  *      - Render Target Cache Flush Enable ([12] of DW1)
 164  *      - Depth Cache Flush Enable ([0] of DW1)
 165  *      - Stall at Pixel Scoreboard ([1] of DW1)
 166  *      - Depth Stall ([13] of DW1)
 167  *      - Post-Sync Operation ([13] of DW1)
 168  *      - Notify Enable ([8] of DW1)"
 169  *
 170  * The cache flushes require the workaround flush that triggered this
 171  * one, so we can't use it.  Depth stall would trigger the same.
 172  * Post-sync nonzero is what triggered this second workaround, so we
 173  * can't use that one either.  Notify enable is IRQs, which aren't
 174  * really our business.  That leaves only stall at scoreboard.
 175  */
 176 static int
 177 intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring)
 178 {
 179         struct pipe_control *pc = ring->private;
 180         u32 scratch_addr = pc->gtt_offset + 128;
 181         int ret;
 182
 183
 184         ret = intel_ring_begin(ring, 6);
 185         if (ret)
 186                 return ret;
 187
 188         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
 189         intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
 190                         PIPE_CONTROL_STALL_AT_SCOREBOARD);
 191         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
 192         intel_ring_emit(ring, 0); /* low dword */
 193         intel_ring_emit(ring, 0); /* high dword */
 194         intel_ring_emit(ring, MI_NOOP);
 195         intel_ring_advance(ring);
 196
 197         ret = intel_ring_begin(ring, 6);
 198         if (ret)
 199                 return ret;
 200
 201         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
 202         intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
 203         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
 204         intel_ring_emit(ring, 0);
 205         intel_ring_emit(ring, 0);
 206         intel_ring_emit(ring, MI_NOOP);
 207         intel_ring_advance(ring);
 208
 209         return 0;
 210 }
 211
 212 static int
 213 gen6_render_ring_flush(struct intel_ring_buffer *ring,
 214                          u32 invalidate_domains, u32 flush_domains)
 215 {
 216         u32 flags = 0;
 217         struct pipe_control *pc = ring->private;
 218         u32 scratch_addr = pc->gtt_offset + 128;
 219         int ret;
 220
 221         /* Force SNB workarounds for PIPE_CONTROL flushes */
 222         intel_emit_post_sync_nonzero_flush(ring);
 223
 224         /* Just flush everything.  Experiments have shown that reducing the
 225          * number of bits based on the write domains has little performance
 226          * impact.
 227          */
 228         flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
 229         flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
 230         flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
 231         flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
 232         flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
 233         flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
 234         flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
 235
 236         ret = intel_ring_begin(ring, 6);
 237         if (ret)
 238                 return ret;
 239
 240         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
 241         intel_ring_emit(ring, flags);
 242         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
 243         intel_ring_emit(ring, 0); /* lower dword */
 244         intel_ring_emit(ring, 0); /* uppwer dword */
 245         intel_ring_emit(ring, MI_NOOP);
 246         intel_ring_advance(ring);
 247
 248         return 0;
 249 }
 250
 251 static void ring_write_tail(struct intel_ring_buffer *ring,
 252                             u32 value)
 253 {
 254         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 255         I915_WRITE_TAIL(ring, value);
 256 }
 257
 258 u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
 259 {
 260         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 261         u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
 262                         RING_ACTHD(ring->mmio_base) : ACTHD;
 263
 264         return I915_READ(acthd_reg);
 265 }
 266
 267 static int init_ring_common(struct intel_ring_buffer *ring)
 268 {
 269         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 270         struct drm_i915_gem_object *obj = ring->obj;
 271         u32 head;
 272
 273         /* Stop the ring if it's running. */
 274         I915_WRITE_CTL(ring, 0);
 275         I915_WRITE_HEAD(ring, 0);
 276         ring->write_tail(ring, 0);
 277
 278         /* Initialize the ring. */
 279         I915_WRITE_START(ring, obj->gtt_offset);
 280         head = I915_READ_HEAD(ring) & HEAD_ADDR;
 281
 282         /* G45 ring initialization fails to reset head to zero */
 283         if (head != 0) {
 284                 DRM_DEBUG_KMS("%s head not reset to zero "
 285                               "ctl %08x head %08x tail %08x start %08x\n",
 286                               ring->name,
 287                               I915_READ_CTL(ring),
 288                               I915_READ_HEAD(ring),
 289                               I915_READ_TAIL(ring),
 290                               I915_READ_START(ring));
 291
 292                 I915_WRITE_HEAD(ring, 0);
 293
 294                 if (I915_READ_HEAD(ring) & HEAD_ADDR) {
 295                         DRM_ERROR("failed to set %s head to zero "
 296                                   "ctl %08x head %08x tail %08x start %08x\n",
 297                                   ring->name,
 298                                   I915_READ_CTL(ring),
 299                                   I915_READ_HEAD(ring),
 300                                   I915_READ_TAIL(ring),
 301                                   I915_READ_START(ring));
 302                 }
 303         }
 304
 305         I915_WRITE_CTL(ring,
 306                         ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
 307                         | RING_VALID);
 308
 309         /* If the head is still not zero, the ring is dead */
 310         if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
 311                      I915_READ_START(ring) == obj->gtt_offset &&
 312                      (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
 313                 DRM_ERROR("%s initialization failed "
 314                                 "ctl %08x head %08x tail %08x start %08x\n",
 315                                 ring->name,
 316                                 I915_READ_CTL(ring),
 317                                 I915_READ_HEAD(ring),
 318                                 I915_READ_TAIL(ring),
 319                                 I915_READ_START(ring));
 320                 return -EIO;
 321         }
 322
 323         if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
 324                 i915_kernel_lost_context(ring->dev);
 325         else {
 326                 ring->head = I915_READ_HEAD(ring);
 327                 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
 328                 ring->space = ring_space(ring);
 329         }
 330
 331         return 0;
 332 }
 333
 334 static int
 335 init_pipe_control(struct intel_ring_buffer *ring)
 336 {
 337         struct pipe_control *pc;
 338         struct drm_i915_gem_object *obj;
 339         int ret;
 340
 341         if (ring->private)
 342                 return 0;
 343
 344         pc = kmalloc(sizeof(*pc), GFP_KERNEL);
 345         if (!pc)
 346                 return -ENOMEM;
 347
 348         obj = i915_gem_alloc_object(ring->dev, 4096);
 349         if (obj == NULL) {
 350                 DRM_ERROR("Failed to allocate seqno page\n");
 351                 ret = -ENOMEM;
 352                 goto err;
 353         }
 354
 355         i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
 356
 357         ret = i915_gem_object_pin(obj, 4096, true);
 358         if (ret)
 359                 goto err_unref;
 360
 361         pc->gtt_offset = obj->gtt_offset;
 362         pc->cpu_page =  kmap(obj->pages[0]);
 363         if (pc->cpu_page == NULL)
 364                 goto err_unpin;
 365
 366         pc->obj = obj;
 367         ring->private = pc;
 368         return 0;
 369
 370 err_unpin:
 371         i915_gem_object_unpin(obj);
 372 err_unref:
 373         drm_gem_object_unreference(&obj->base);
 374 err:
 375         kfree(pc);
 376         return ret;
 377 }
 378
 379 static void
 380 cleanup_pipe_control(struct intel_ring_buffer *ring)
 381 {
 382         struct pipe_control *pc = ring->private;
 383         struct drm_i915_gem_object *obj;
 384
 385         if (!ring->private)
 386                 return;
 387
 388         obj = pc->obj;
 389         kunmap(obj->pages[0]);
 390         i915_gem_object_unpin(obj);
 391         drm_gem_object_unreference(&obj->base);
 392
 393         kfree(pc);
 394         ring->private = NULL;
 395 }
 396
 397 static int init_render_ring(struct intel_ring_buffer *ring)
 398 {
 399         struct drm_device *dev = ring->dev;
 400         struct drm_i915_private *dev_priv = dev->dev_private;
 401         int ret = init_ring_common(ring);
 402
 403         if (INTEL_INFO(dev)->gen > 3) {
 404                 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
 405                 if (IS_GEN7(dev))
 406                         I915_WRITE(GFX_MODE_GEN7,
 407                                    _MASKED_BIT_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) |
 408                                    _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
 409         }
 410
 411         if (INTEL_INFO(dev)->gen >= 5) {
 412                 ret = init_pipe_control(ring);
 413                 if (ret)
 414                         return ret;
 415         }
 416
 417         if (IS_GEN6(dev)) {
 418                 /* From the Sandybridge PRM, volume 1 part 3, page 24:
 419                  * "If this bit is set, STCunit will have LRA as replacement
 420                  *  policy. [...] This bit must be reset.  LRA replacement
 421                  *  policy is not supported."
 422                  */
 423                 I915_WRITE(CACHE_MODE_0,
 424                            _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
 425         }
 426
 427         if (INTEL_INFO(dev)->gen >= 6)
 428                 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
 429
 430         if (IS_IVYBRIDGE(dev))
 431                 I915_WRITE_IMR(ring, ~GEN6_RENDER_L3_PARITY_ERROR);
 432
 433         return ret;
 434 }
 435
 436 static void render_ring_cleanup(struct intel_ring_buffer *ring)
 437 {
 438         if (!ring->private)
 439                 return;
 440
 441         cleanup_pipe_control(ring);
 442 }
 443
 444 static void
 445 update_mboxes(struct intel_ring_buffer *ring,
 446             u32 seqno,
 447             u32 mmio_offset)
 448 {
 449         intel_ring_emit(ring, MI_SEMAPHORE_MBOX |
 450                               MI_SEMAPHORE_GLOBAL_GTT |
 451                               MI_SEMAPHORE_REGISTER |
 452                               MI_SEMAPHORE_UPDATE);
 453         intel_ring_emit(ring, seqno);
 454         intel_ring_emit(ring, mmio_offset);
 455 }
 456
 457 /**
 458  * gen6_add_request - Update the semaphore mailbox registers
 459  *
 460  * @ring - ring that is adding a request
 461  * @seqno - return seqno stuck into the ring
 462  *
 463  * Update the mailbox registers in the *other* rings with the current seqno.
 464  * This acts like a signal in the canonical semaphore.
 465  */
 466 static int
 467 gen6_add_request(struct intel_ring_buffer *ring,
 468                  u32 *seqno)
 469 {
 470         u32 mbox1_reg;
 471         u32 mbox2_reg;
 472         int ret;
 473
 474         ret = intel_ring_begin(ring, 10);
 475         if (ret)
 476                 return ret;
 477
 478         mbox1_reg = ring->signal_mbox[0];
 479         mbox2_reg = ring->signal_mbox[1];
 480
 481         *seqno = i915_gem_next_request_seqno(ring);
 482
 483         update_mboxes(ring, *seqno, mbox1_reg);
 484         update_mboxes(ring, *seqno, mbox2_reg);
 485         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
 486         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 487         intel_ring_emit(ring, *seqno);
 488         intel_ring_emit(ring, MI_USER_INTERRUPT);
 489         intel_ring_advance(ring);
 490
 491         return 0;
 492 }
 493
 494 /**
 495  * intel_ring_sync - sync the waiter to the signaller on seqno
 496  *
 497  * @waiter - ring that is waiting
 498  * @signaller - ring which has, or will signal
 499  * @seqno - seqno which the waiter will block on
 500  */
 501 static int
 502 gen6_ring_sync(struct intel_ring_buffer *waiter,
 503                struct intel_ring_buffer *signaller,
 504                u32 seqno)
 505 {
 506         int ret;
 507         u32 dw1 = MI_SEMAPHORE_MBOX |
 508                   MI_SEMAPHORE_COMPARE |
 509                   MI_SEMAPHORE_REGISTER;
 510
 511         /* Throughout all of the GEM code, seqno passed implies our current
 512          * seqno is >= the last seqno executed. However for hardware the
 513          * comparison is strictly greater than.
 514          */
 515         seqno -= 1;
 516
 517         WARN_ON(signaller->semaphore_register[waiter->id] ==
 518                 MI_SEMAPHORE_SYNC_INVALID);
 519
 520         ret = intel_ring_begin(waiter, 4);
 521         if (ret)
 522                 return ret;
 523
 524         intel_ring_emit(waiter,
 525                         dw1 | signaller->semaphore_register[waiter->id]);
 526         intel_ring_emit(waiter, seqno);
 527         intel_ring_emit(waiter, 0);
 528         intel_ring_emit(waiter, MI_NOOP);
 529         intel_ring_advance(waiter);
 530
 531         return 0;
 532 }
 533
 534 #define PIPE_CONTROL_FLUSH(ring__, addr__)                                      \
 535 do {                                                                    \
 536         intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |                \
 537                  PIPE_CONTROL_DEPTH_STALL);                             \
 538         intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
 539         intel_ring_emit(ring__, 0);                                                     \
 540         intel_ring_emit(ring__, 0);                                                     \
 541 } while (0)
 542
 543 static int
 544 pc_render_add_request(struct intel_ring_buffer *ring,
 545                       u32 *result)
 546 {
 547         u32 seqno = i915_gem_next_request_seqno(ring);
 548         struct pipe_control *pc = ring->private;
 549         u32 scratch_addr = pc->gtt_offset + 128;
 550         int ret;
 551
 552         /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
 553          * incoherent with writes to memory, i.e. completely fubar,
 554          * so we need to use PIPE_NOTIFY instead.
 555          *
 556          * However, we also need to workaround the qword write
 557          * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
 558          * memory before requesting an interrupt.
 559          */
 560         ret = intel_ring_begin(ring, 32);
 561         if (ret)
 562                 return ret;
 563
 564         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
 565                         PIPE_CONTROL_WRITE_FLUSH |
 566                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
 567         intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
 568         intel_ring_emit(ring, seqno);
 569         intel_ring_emit(ring, 0);
 570         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 571         scratch_addr += 128; /* write to separate cachelines */
 572         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 573         scratch_addr += 128;
 574         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 575         scratch_addr += 128;
 576         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 577         scratch_addr += 128;
 578         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 579         scratch_addr += 128;
 580         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 581
 582         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
 583                         PIPE_CONTROL_WRITE_FLUSH |
 584                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
 585                         PIPE_CONTROL_NOTIFY);
 586         intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
 587         intel_ring_emit(ring, seqno);
 588         intel_ring_emit(ring, 0);
 589         intel_ring_advance(ring);
 590
 591         *result = seqno;
 592         return 0;
 593 }
 594
 595 static u32
 596 gen6_ring_get_seqno(struct intel_ring_buffer *ring)
 597 {
 598         struct drm_device *dev = ring->dev;
 599
 600         /* Workaround to force correct ordering between irq and seqno writes on
 601          * ivb (and maybe also on snb) by reading from a CS register (like
 602          * ACTHD) before reading the status page. */
 603         if (IS_GEN6(dev) || IS_GEN7(dev))
 604                 intel_ring_get_active_head(ring);
 605         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 606 }
 607
 608 static u32
 609 ring_get_seqno(struct intel_ring_buffer *ring)
 610 {
 611         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 612 }
 613
 614 static u32
 615 pc_render_get_seqno(struct intel_ring_buffer *ring)
 616 {
 617         struct pipe_control *pc = ring->private;
 618         return pc->cpu_page[0];
 619 }
 620
 621 static bool
 622 gen5_ring_get_irq(struct intel_ring_buffer *ring)
 623 {
 624         struct drm_device *dev = ring->dev;
 625         drm_i915_private_t *dev_priv = dev->dev_private;
 626         unsigned long flags;
 627
 628         if (!dev->irq_enabled)
 629                 return false;
 630
 631         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 632         if (ring->irq_refcount++ == 0) {
 633                 dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
 634                 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 635                 POSTING_READ(GTIMR);
 636         }
 637         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 638
 639         return true;
 640 }
 641
 642 static void
 643 gen5_ring_put_irq(struct intel_ring_buffer *ring)
 644 {
 645         struct drm_device *dev = ring->dev;
 646         drm_i915_private_t *dev_priv = dev->dev_private;
 647         unsigned long flags;
 648
 649         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 650         if (--ring->irq_refcount == 0) {
 651                 dev_priv->gt_irq_mask |= ring->irq_enable_mask;
 652                 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 653                 POSTING_READ(GTIMR);
 654         }
 655         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 656 }
 657
 658 static bool
 659 i9xx_ring_get_irq(struct intel_ring_buffer *ring)
 660 {
 661         struct drm_device *dev = ring->dev;
 662         drm_i915_private_t *dev_priv = dev->dev_private;
 663         unsigned long flags;
 664
 665         if (!dev->irq_enabled)
 666                 return false;
 667
 668         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 669         if (ring->irq_refcount++ == 0) {
 670                 dev_priv->irq_mask &= ~ring->irq_enable_mask;
 671                 I915_WRITE(IMR, dev_priv->irq_mask);
 672                 POSTING_READ(IMR);
 673         }
 674         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 675
 676         return true;
 677 }
 678
 679 static void
 680 i9xx_ring_put_irq(struct intel_ring_buffer *ring)
 681 {
 682         struct drm_device *dev = ring->dev;
 683         drm_i915_private_t *dev_priv = dev->dev_private;
 684         unsigned long flags;
 685
 686         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 687         if (--ring->irq_refcount == 0) {
 688                 dev_priv->irq_mask |= ring->irq_enable_mask;
 689                 I915_WRITE(IMR, dev_priv->irq_mask);
 690                 POSTING_READ(IMR);
 691         }
 692         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 693 }
 694
 695 static bool
 696 i8xx_ring_get_irq(struct intel_ring_buffer *ring)
 697 {
 698         struct drm_device *dev = ring->dev;
 699         drm_i915_private_t *dev_priv = dev->dev_private;
 700         unsigned long flags;
 701
 702         if (!dev->irq_enabled)
 703                 return false;
 704
 705         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 706         if (ring->irq_refcount++ == 0) {
 707                 dev_priv->irq_mask &= ~ring->irq_enable_mask;
 708                 I915_WRITE16(IMR, dev_priv->irq_mask);
 709                 POSTING_READ16(IMR);
 710         }
 711         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 712
 713         return true;
 714 }
 715
 716 static void
 717 i8xx_ring_put_irq(struct intel_ring_buffer *ring)
 718 {
 719         struct drm_device *dev = ring->dev;
 720         drm_i915_private_t *dev_priv = dev->dev_private;
 721         unsigned long flags;
 722
 723         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 724         if (--ring->irq_refcount == 0) {
 725                 dev_priv->irq_mask |= ring->irq_enable_mask;
 726                 I915_WRITE16(IMR, dev_priv->irq_mask);
 727                 POSTING_READ16(IMR);
 728         }
 729         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 730 }
 731
 732 void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
 733 {
 734         struct drm_device *dev = ring->dev;
 735         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 736         u32 mmio = 0;
 737
 738         /* The ring status page addresses are no longer next to the rest of
 739          * the ring registers as of gen7.
 740          */
 741         if (IS_GEN7(dev)) {
 742                 switch (ring->id) {
 743                 case RCS:
 744                         mmio = RENDER_HWS_PGA_GEN7;
 745                         break;
 746                 case BCS:
 747                         mmio = BLT_HWS_PGA_GEN7;
 748                         break;
 749                 case VCS:
 750                         mmio = BSD_HWS_PGA_GEN7;
 751                         break;
 752                 }
 753         } else if (IS_GEN6(ring->dev)) {
 754                 mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
 755         } else {
 756                 mmio = RING_HWS_PGA(ring->mmio_base);
 757         }
 758
 759         I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
 760         POSTING_READ(mmio);
 761 }
 762
 763 static int
 764 bsd_ring_flush(struct intel_ring_buffer *ring,
 765                u32     invalidate_domains,
 766                u32     flush_domains)
 767 {
 768         int ret;
 769
 770         ret = intel_ring_begin(ring, 2);
 771         if (ret)
 772                 return ret;
 773
 774         intel_ring_emit(ring, MI_FLUSH);
 775         intel_ring_emit(ring, MI_NOOP);
 776         intel_ring_advance(ring);
 777         return 0;
 778 }
 779
 780 static int
 781 i9xx_add_request(struct intel_ring_buffer *ring,
 782                  u32 *result)
 783 {
 784         u32 seqno;
 785         int ret;
 786
 787         ret = intel_ring_begin(ring, 4);
 788         if (ret)
 789                 return ret;
 790
 791         seqno = i915_gem_next_request_seqno(ring);
 792
 793         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
 794         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 795         intel_ring_emit(ring, seqno);
 796         intel_ring_emit(ring, MI_USER_INTERRUPT);
 797         intel_ring_advance(ring);
 798
 799         *result = seqno;
 800         return 0;
 801 }
 802
 803 static bool
 804 gen6_ring_get_irq(struct intel_ring_buffer *ring)
 805 {
 806         struct drm_device *dev = ring->dev;
 807         drm_i915_private_t *dev_priv = dev->dev_private;
 808         unsigned long flags;
 809
 810         if (!dev->irq_enabled)
 811                return false;
 812
 813         /* It looks like we need to prevent the gt from suspending while waiting
 814          * for an notifiy irq, otherwise irqs seem to get lost on at least the
 815          * blt/bsd rings on ivb. */
 816         gen6_gt_force_wake_get(dev_priv);
 817
 818         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 819         if (ring->irq_refcount++ == 0) {
 820                 if (IS_IVYBRIDGE(dev) && ring->id == RCS)
 821                         I915_WRITE_IMR(ring, ~(ring->irq_enable_mask |
 822                                                 GEN6_RENDER_L3_PARITY_ERROR));
 823                 else
 824                         I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
 825                 dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
 826                 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 827                 POSTING_READ(GTIMR);
 828         }
 829         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 830
 831         return true;
 832 }
 833
 834 static void
 835 gen6_ring_put_irq(struct intel_ring_buffer *ring)
 836 {
 837         struct drm_device *dev = ring->dev;
 838         drm_i915_private_t *dev_priv = dev->dev_private;
 839         unsigned long flags;
 840
 841         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 842         if (--ring->irq_refcount == 0) {
 843                 if (IS_IVYBRIDGE(dev) && ring->id == RCS)
 844                         I915_WRITE_IMR(ring, ~GEN6_RENDER_L3_PARITY_ERROR);
 845                 else
 846                         I915_WRITE_IMR(ring, ~0);
 847                 dev_priv->gt_irq_mask |= ring->irq_enable_mask;
 848                 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 849                 POSTING_READ(GTIMR);
 850         }
 851         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 852
 853         gen6_gt_force_wake_put(dev_priv);
 854 }
 855
 856 static int
 857 i965_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length)
 858 {
 859         int ret;
 860
 861         ret = intel_ring_begin(ring, 2);
 862         if (ret)
 863                 return ret;
 864
 865         intel_ring_emit(ring,
 866                         MI_BATCH_BUFFER_START |
 867                         MI_BATCH_GTT |
 868                         MI_BATCH_NON_SECURE_I965);
 869         intel_ring_emit(ring, offset);
 870         intel_ring_advance(ring);
 871
 872         return 0;
 873 }
 874
 875 static int
 876 i830_dispatch_execbuffer(struct intel_ring_buffer *ring,
 877                                 u32 offset, u32 len)
 878 {
 879         int ret;
 880
 881         ret = intel_ring_begin(ring, 4);
 882         if (ret)
 883                 return ret;
 884
 885         intel_ring_emit(ring, MI_BATCH_BUFFER);
 886         intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
 887         intel_ring_emit(ring, offset + len - 8);
 888         intel_ring_emit(ring, 0);
 889         intel_ring_advance(ring);
 890
 891         return 0;
 892 }
 893
 894 static int
 895 i915_dispatch_execbuffer(struct intel_ring_buffer *ring,
 896                                 u32 offset, u32 len)
 897 {
 898         int ret;
 899
 900         ret = intel_ring_begin(ring, 2);
 901         if (ret)
 902                 return ret;
 903
 904         intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
 905         intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
 906         intel_ring_advance(ring);
 907
 908         return 0;
 909 }
 910
 911 static void cleanup_status_page(struct intel_ring_buffer *ring)
 912 {
 913         struct drm_i915_gem_object *obj;
 914
 915         obj = ring->status_page.obj;
 916         if (obj == NULL)
 917                 return;
 918
 919         kunmap(obj->pages[0]);
 920         i915_gem_object_unpin(obj);
 921         drm_gem_object_unreference(&obj->base);
 922         ring->status_page.obj = NULL;
 923 }
 924
 925 static int init_status_page(struct intel_ring_buffer *ring)
 926 {
 927         struct drm_device *dev = ring->dev;
 928         struct drm_i915_gem_object *obj;
 929         int ret;
 930
 931         obj = i915_gem_alloc_object(dev, 4096);
 932         if (obj == NULL) {
 933                 DRM_ERROR("Failed to allocate status page\n");
 934                 ret = -ENOMEM;
 935                 goto err;
 936         }
 937
 938         i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
 939
 940         ret = i915_gem_object_pin(obj, 4096, true);
 941         if (ret != 0) {
 942                 goto err_unref;
 943         }
 944
 945         ring->status_page.gfx_addr = obj->gtt_offset;
 946         ring->status_page.page_addr = kmap(obj->pages[0]);
 947         if (ring->status_page.page_addr == NULL) {
 948                 goto err_unpin;
 949         }
 950         ring->status_page.obj = obj;
 951         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
 952
 953         intel_ring_setup_status_page(ring);
 954         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
 955                         ring->name, ring->status_page.gfx_addr);
 956
 957         return 0;
 958
 959 err_unpin:
 960         i915_gem_object_unpin(obj);
 961 err_unref:
 962         drm_gem_object_unreference(&obj->base);
 963 err:
 964         return ret;
 965 }
 966
 967 static int intel_init_ring_buffer(struct drm_device *dev,
 968                                   struct intel_ring_buffer *ring)
 969 {
 970         struct drm_i915_gem_object *obj;
 971         int ret;
 972
 973         ring->dev = dev;
 974         INIT_LIST_HEAD(&ring->active_list);
 975         INIT_LIST_HEAD(&ring->request_list);
 976         INIT_LIST_HEAD(&ring->gpu_write_list);
 977         ring->size = 32 * PAGE_SIZE;
 978
 979         init_waitqueue_head(&ring->irq_queue);
 980
 981         if (I915_NEED_GFX_HWS(dev)) {
 982                 ret = init_status_page(ring);
 983                 if (ret)
 984                         return ret;
 985         }
 986
 987         obj = i915_gem_alloc_object(dev, ring->size);
 988         if (obj == NULL) {
 989                 DRM_ERROR("Failed to allocate ringbuffer\n");
 990                 ret = -ENOMEM;
 991                 goto err_hws;
 992         }
 993
 994         ring->obj = obj;
 995
 996         ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
 997         if (ret)
 998                 goto err_unref;
 999
1000         ring->virtual_start = ioremap_wc(dev->agp->base + obj->gtt_offset,
1001                                          ring->size);
1002         if (ring->virtual_start == NULL) {
1003                 DRM_ERROR("Failed to map ringbuffer.\n");
1004                 ret = -EINVAL;
1005                 goto err_unpin;
1006         }
1007
1008         ret = ring->init(ring);
1009         if (ret)
1010                 goto err_unmap;
1011
1012         /* Workaround an erratum on the i830 which causes a hang if
1013          * the TAIL pointer points to within the last 2 cachelines
1014          * of the buffer.
1015          */
1016         ring->effective_size = ring->size;
1017         if (IS_I830(ring->dev) || IS_845G(ring->dev))
1018                 ring->effective_size -= 128;
1019
1020         return 0;
1021
1022 err_unmap:
1023         iounmap(ring->virtual_start);
1024 err_unpin:
1025         i915_gem_object_unpin(obj);
1026 err_unref:
1027         drm_gem_object_unreference(&obj->base);
1028         ring->obj = NULL;
1029 err_hws:
1030         cleanup_status_page(ring);
1031         return ret;
1032 }
1033
1034 void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
1035 {
1036         struct drm_i915_private *dev_priv;
1037         int ret;
1038
1039         if (ring->obj == NULL)
1040                 return;
1041
1042         /* Disable the ring buffer. The ring must be idle at this point */
1043         dev_priv = ring->dev->dev_private;
1044         ret = intel_wait_ring_idle(ring);
1045         if (ret)
1046                 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
1047                           ring->name, ret);
1048
1049         I915_WRITE_CTL(ring, 0);
1050
1051         iounmap(ring->virtual_start);
1052
1053         i915_gem_object_unpin(ring->obj);
1054         drm_gem_object_unreference(&ring->obj->base);
1055         ring->obj = NULL;
1056
1057         if (ring->cleanup)
1058                 ring->cleanup(ring);
1059
1060         cleanup_status_page(ring);
1061 }
1062
1063 static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
1064 {
1065         uint32_t __iomem *virt;
1066         int rem = ring->size - ring->tail;
1067
1068         if (ring->space < rem) {
1069                 int ret = intel_wait_ring_buffer(ring, rem);
1070                 if (ret)
1071                         return ret;
1072         }
1073
1074         virt = ring->virtual_start + ring->tail;
1075         rem /= 4;
1076         while (rem--)
1077                 iowrite32(MI_NOOP, virt++);
1078
1079         ring->tail = 0;
1080         ring->space = ring_space(ring);
1081
1082         return 0;
1083 }
1084
1085 static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno)
1086 {
1087         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1088         bool was_interruptible;
1089         int ret;
1090
1091         /* XXX As we have not yet audited all the paths to check that
1092          * they are ready for ERESTARTSYS from intel_ring_begin, do not
1093          * allow us to be interruptible by a signal.
1094          */
1095         was_interruptible = dev_priv->mm.interruptible;
1096         dev_priv->mm.interruptible = false;
1097
1098         ret = i915_wait_seqno(ring, seqno);
1099
1100         dev_priv->mm.interruptible = was_interruptible;
1101         if (!ret)
1102                 i915_gem_retire_requests_ring(ring);
1103
1104         return ret;
1105 }
1106
1107 static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n)
1108 {
1109         struct drm_i915_gem_request *request;
1110         u32 seqno = 0;
1111         int ret;
1112
1113         i915_gem_retire_requests_ring(ring);
1114
1115         if (ring->last_retired_head != -1) {
1116                 ring->head = ring->last_retired_head;
1117                 ring->last_retired_head = -1;
1118                 ring->space = ring_space(ring);
1119                 if (ring->space >= n)
1120                         return 0;
1121         }
1122
1123         list_for_each_entry(request, &ring->request_list, list) {
1124                 int space;
1125
1126                 if (request->tail == -1)
1127                         continue;
1128
1129                 space = request->tail - (ring->tail + 8);
1130                 if (space < 0)
1131                         space += ring->size;
1132                 if (space >= n) {
1133                         seqno = request->seqno;
1134                         break;
1135                 }
1136
1137                 /* Consume this request in case we need more space than
1138                  * is available and so need to prevent a race between
1139                  * updating last_retired_head and direct reads of
1140                  * I915_RING_HEAD. It also provides a nice sanity check.
1141                  */
1142                 request->tail = -1;
1143         }
1144
1145         if (seqno == 0)
1146                 return -ENOSPC;
1147
1148         ret = intel_ring_wait_seqno(ring, seqno);
1149         if (ret)
1150                 return ret;
1151
1152         if (WARN_ON(ring->last_retired_head == -1))
1153                 return -ENOSPC;
1154
1155         ring->head = ring->last_retired_head;
1156         ring->last_retired_head = -1;
1157         ring->space = ring_space(ring);
1158         if (WARN_ON(ring->space < n))
1159                 return -ENOSPC;
1160
1161         return 0;
1162 }
1163
1164 int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
1165 {
1166         struct drm_device *dev = ring->dev;
1167         struct drm_i915_private *dev_priv = dev->dev_private;
1168         unsigned long end;
1169         int ret;
1170
1171         ret = intel_ring_wait_request(ring, n);
1172         if (ret != -ENOSPC)
1173                 return ret;
1174
1175         trace_i915_ring_wait_begin(ring);
1176         /* With GEM the hangcheck timer should kick us out of the loop,
1177          * leaving it early runs the risk of corrupting GEM state (due
1178          * to running on almost untested codepaths). But on resume
1179          * timers don't work yet, so prevent a complete hang in that
1180          * case by choosing an insanely large timeout. */
1181         end = jiffies + 60 * HZ;
1182
1183         do {
1184                 ring->head = I915_READ_HEAD(ring);
1185                 ring->space = ring_space(ring);
1186                 if (ring->space >= n) {
1187                         trace_i915_ring_wait_end(ring);
1188                         return 0;
1189                 }
1190
1191                 if (dev->primary->master) {
1192                         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
1193                         if (master_priv->sarea_priv)
1194                                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
1195                 }
1196
1197                 msleep(1);
1198                 if (atomic_read(&dev_priv->mm.wedged))
1199                         return -EAGAIN;
1200         } while (!time_after(jiffies, end));
1201         trace_i915_ring_wait_end(ring);
1202         return -EBUSY;
1203 }
1204
1205 int intel_ring_begin(struct intel_ring_buffer *ring,
1206                      int num_dwords)
1207 {
1208         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1209         int n = 4*num_dwords;
1210         int ret;
1211
1212         if (unlikely(atomic_read(&dev_priv->mm.wedged)))
1213                 return -EIO;
1214
1215         if (unlikely(ring->tail + n > ring->effective_size)) {
1216                 ret = intel_wrap_ring_buffer(ring);
1217                 if (unlikely(ret))
1218                         return ret;
1219         }
1220
1221         if (unlikely(ring->space < n)) {
1222                 ret = intel_wait_ring_buffer(ring, n);
1223                 if (unlikely(ret))
1224                         return ret;
1225         }
1226
1227         ring->space -= n;
1228         return 0;
1229 }
1230
1231 void intel_ring_advance(struct intel_ring_buffer *ring)
1232 {
1233         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1234
1235         ring->tail &= ring->size - 1;
1236         if (dev_priv->stop_rings & intel_ring_flag(ring))
1237                 return;
1238         ring->write_tail(ring, ring->tail);
1239 }
1240
1241
1242 static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
1243                                      u32 value)
1244 {
1245         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1246
1247        /* Every tail move must follow the sequence below */
1248         I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1249                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
1250                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
1251         I915_WRITE(GEN6_BSD_RNCID, 0x0);
1252
1253         if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
1254                 GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
1255                 50))
1256         DRM_ERROR("timed out waiting for IDLE Indicator\n");
1257
1258         I915_WRITE_TAIL(ring, value);
1259         I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1260                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
1261                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
1262 }
1263
1264 static int gen6_ring_flush(struct intel_ring_buffer *ring,
1265                            u32 invalidate, u32 flush)
1266 {
1267         uint32_t cmd;
1268         int ret;
1269
1270         ret = intel_ring_begin(ring, 4);
1271         if (ret)
1272                 return ret;
1273
1274         cmd = MI_FLUSH_DW;
1275         if (invalidate & I915_GEM_GPU_DOMAINS)
1276                 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD;
1277         intel_ring_emit(ring, cmd);
1278         intel_ring_emit(ring, 0);
1279         intel_ring_emit(ring, 0);
1280         intel_ring_emit(ring, MI_NOOP);
1281         intel_ring_advance(ring);
1282         return 0;
1283 }
1284
1285 static int
1286 gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
1287                               u32 offset, u32 len)
1288 {
1289         int ret;
1290
1291         ret = intel_ring_begin(ring, 2);
1292         if (ret)
1293                 return ret;
1294
1295         intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
1296         /* bit0-7 is the length on GEN6+ */
1297         intel_ring_emit(ring, offset);
1298         intel_ring_advance(ring);
1299
1300         return 0;
1301 }
1302
1303 /* Blitter support (SandyBridge+) */
1304
1305 static int blt_ring_flush(struct intel_ring_buffer *ring,
1306                           u32 invalidate, u32 flush)
1307 {
1308         uint32_t cmd;
1309         int ret;
1310
1311         ret = intel_ring_begin(ring, 4);
1312         if (ret)
1313                 return ret;
1314
1315         cmd = MI_FLUSH_DW;
1316         if (invalidate & I915_GEM_DOMAIN_RENDER)
1317                 cmd |= MI_INVALIDATE_TLB;
1318         intel_ring_emit(ring, cmd);
1319         intel_ring_emit(ring, 0);
1320         intel_ring_emit(ring, 0);
1321         intel_ring_emit(ring, MI_NOOP);
1322         intel_ring_advance(ring);
1323         return 0;
1324 }
1325
1326 int intel_init_render_ring_buffer(struct drm_device *dev)
1327 {
1328         drm_i915_private_t *dev_priv = dev->dev_private;
1329         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1330
1331         ring->name = "render ring";
1332         ring->id = RCS;
1333         ring->mmio_base = RENDER_RING_BASE;
1334
1335         if (INTEL_INFO(dev)->gen >= 6) {
1336                 ring->add_request = gen6_add_request;
1337                 ring->flush = gen6_render_ring_flush;
1338                 ring->irq_get = gen6_ring_get_irq;
1339                 ring->irq_put = gen6_ring_put_irq;
1340                 ring->irq_enable_mask = GT_USER_INTERRUPT;
1341                 ring->get_seqno = gen6_ring_get_seqno;
1342                 ring->sync_to = gen6_ring_sync;
1343                 ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_INVALID;
1344                 ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_RV;
1345                 ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_RB;
1346                 ring->signal_mbox[0] = GEN6_VRSYNC;
1347                 ring->signal_mbox[1] = GEN6_BRSYNC;
1348         } else if (IS_GEN5(dev)) {
1349                 ring->add_request = pc_render_add_request;
1350                 ring->flush = gen4_render_ring_flush;
1351                 ring->get_seqno = pc_render_get_seqno;
1352                 ring->irq_get = gen5_ring_get_irq;
1353                 ring->irq_put = gen5_ring_put_irq;
1354                 ring->irq_enable_mask = GT_USER_INTERRUPT | GT_PIPE_NOTIFY;
1355         } else {
1356                 ring->add_request = i9xx_add_request;
1357                 if (INTEL_INFO(dev)->gen < 4)
1358                         ring->flush = gen2_render_ring_flush;
1359                 else
1360                         ring->flush = gen4_render_ring_flush;
1361                 ring->get_seqno = ring_get_seqno;
1362                 if (IS_GEN2(dev)) {
1363                         ring->irq_get = i8xx_ring_get_irq;
1364                         ring->irq_put = i8xx_ring_put_irq;
1365                 } else {
1366                         ring->irq_get = i9xx_ring_get_irq;
1367                         ring->irq_put = i9xx_ring_put_irq;
1368                 }
1369                 ring->irq_enable_mask = I915_USER_INTERRUPT;
1370         }
1371         ring->write_tail = ring_write_tail;
1372         if (INTEL_INFO(dev)->gen >= 6)
1373                 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
1374         else if (INTEL_INFO(dev)->gen >= 4)
1375                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
1376         else if (IS_I830(dev) || IS_845G(dev))
1377                 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
1378         else
1379                 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
1380         ring->init = init_render_ring;
1381         ring->cleanup = render_ring_cleanup;
1382
1383
1384         if (!I915_NEED_GFX_HWS(dev)) {
1385                 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1386                 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1387         }
1388
1389         return intel_init_ring_buffer(dev, ring);
1390 }
1391
1392 int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
1393 {
1394         drm_i915_private_t *dev_priv = dev->dev_private;
1395         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1396
1397         ring->name = "render ring";
1398         ring->id = RCS;
1399         ring->mmio_base = RENDER_RING_BASE;
1400
1401         if (INTEL_INFO(dev)->gen >= 6) {
1402                 /* non-kms not supported on gen6+ */
1403                 return -ENODEV;
1404         }
1405
1406         /* Note: gem is not supported on gen5/ilk without kms (the corresponding
1407          * gem_init ioctl returns with -ENODEV). Hence we do not need to set up
1408          * the special gen5 functions. */
1409         ring->add_request = i9xx_add_request;
1410         if (INTEL_INFO(dev)->gen < 4)
1411                 ring->flush = gen2_render_ring_flush;
1412         else
1413                 ring->flush = gen4_render_ring_flush;
1414         ring->get_seqno = ring_get_seqno;
1415         if (IS_GEN2(dev)) {
1416                 ring->irq_get = i8xx_ring_get_irq;
1417                 ring->irq_put = i8xx_ring_put_irq;
1418         } else {
1419                 ring->irq_get = i9xx_ring_get_irq;
1420                 ring->irq_put = i9xx_ring_put_irq;
1421         }
1422         ring->irq_enable_mask = I915_USER_INTERRUPT;
1423         ring->write_tail = ring_write_tail;
1424         if (INTEL_INFO(dev)->gen >= 4)
1425                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
1426         else if (IS_I830(dev) || IS_845G(dev))
1427                 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
1428         else
1429                 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
1430         ring->init = init_render_ring;
1431         ring->cleanup = render_ring_cleanup;
1432
1433         if (!I915_NEED_GFX_HWS(dev))
1434                 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1435
1436         ring->dev = dev;
1437         INIT_LIST_HEAD(&ring->active_list);
1438         INIT_LIST_HEAD(&ring->request_list);
1439         INIT_LIST_HEAD(&ring->gpu_write_list);
1440
1441         ring->size = size;
1442         ring->effective_size = ring->size;
1443         if (IS_I830(ring->dev))
1444                 ring->effective_size -= 128;
1445
1446         ring->virtual_start = ioremap_wc(start, size);
1447         if (ring->virtual_start == NULL) {
1448                 DRM_ERROR("can not ioremap virtual address for"
1449                           " ring buffer\n");
1450                 return -ENOMEM;
1451         }
1452
1453         return 0;
1454 }
1455
1456 int intel_init_bsd_ring_buffer(struct drm_device *dev)
1457 {
1458         drm_i915_private_t *dev_priv = dev->dev_private;
1459         struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
1460
1461         ring->name = "bsd ring";
1462         ring->id = VCS;
1463
1464         ring->write_tail = ring_write_tail;
1465         if (IS_GEN6(dev) || IS_GEN7(dev)) {
1466                 ring->mmio_base = GEN6_BSD_RING_BASE;
1467                 /* gen6 bsd needs a special wa for tail updates */
1468                 if (IS_GEN6(dev))
1469                         ring->write_tail = gen6_bsd_ring_write_tail;
1470                 ring->flush = gen6_ring_flush;
1471                 ring->add_request = gen6_add_request;
1472                 ring->get_seqno = gen6_ring_get_seqno;
1473                 ring->irq_enable_mask = GEN6_BSD_USER_INTERRUPT;
1474                 ring->irq_get = gen6_ring_get_irq;
1475                 ring->irq_put = gen6_ring_put_irq;
1476                 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
1477                 ring->sync_to = gen6_ring_sync;
1478                 ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_VR;
1479                 ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_INVALID;
1480                 ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_VB;
1481                 ring->signal_mbox[0] = GEN6_RVSYNC;
1482                 ring->signal_mbox[1] = GEN6_BVSYNC;
1483         } else {
1484                 ring->mmio_base = BSD_RING_BASE;
1485                 ring->flush = bsd_ring_flush;
1486                 ring->add_request = i9xx_add_request;
1487                 ring->get_seqno = ring_get_seqno;
1488                 if (IS_GEN5(dev)) {
1489                         ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
1490                         ring->irq_get = gen5_ring_get_irq;
1491                         ring->irq_put = gen5_ring_put_irq;
1492                 } else {
1493                         ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
1494                         ring->irq_get = i9xx_ring_get_irq;
1495                         ring->irq_put = i9xx_ring_put_irq;
1496                 }
1497                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
1498         }
1499         ring->init = init_ring_common;
1500
1501
1502         return intel_init_ring_buffer(dev, ring);
1503 }
1504
1505 int intel_init_blt_ring_buffer(struct drm_device *dev)
1506 {
1507         drm_i915_private_t *dev_priv = dev->dev_private;
1508         struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
1509
1510         ring->name = "blitter ring";
1511         ring->id = BCS;
1512
1513         ring->mmio_base = BLT_RING_BASE;
1514         ring->write_tail = ring_write_tail;
1515         ring->flush = blt_ring_flush;
1516         ring->add_request = gen6_add_request;
1517         ring->get_seqno = gen6_ring_get_seqno;
1518         ring->irq_enable_mask = GEN6_BLITTER_USER_INTERRUPT;
1519         ring->irq_get = gen6_ring_get_irq;
1520         ring->irq_put = gen6_ring_put_irq;
1521         ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
1522         ring->sync_to = gen6_ring_sync;
1523         ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_BR;
1524         ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_BV;
1525         ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_INVALID;
1526         ring->signal_mbox[0] = GEN6_RBSYNC;
1527         ring->signal_mbox[1] = GEN6_VBSYNC;
1528         ring->init = init_ring_common;
1529
1530         return intel_init_ring_buffer(dev, ring);
1531 }