Pileus Git - ~andy/linux/blob - drivers/gpu/drm/i915/intel_ringbuffer.c

   1 /*
   2  * Copyright © 2008-2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Zou Nan hai <nanhai.zou@intel.com>
  26  *    Xiang Hai hao<haihao.xiang@intel.com>
  27  *
  28  */
  29
  30 #include "drmP.h"
  31 #include "drm.h"
  32 #include "i915_drv.h"
  33 #include "i915_drm.h"
  34 #include "i915_trace.h"
  35 #include "intel_drv.h"
  36
  37 /*
  38  * 965+ support PIPE_CONTROL commands, which provide finer grained control
  39  * over cache flushing.
  40  */
  41 struct pipe_control {
  42         struct drm_i915_gem_object *obj;
  43         volatile u32 *cpu_page;
  44         u32 gtt_offset;
  45 };
  46
  47 static inline int ring_space(struct intel_ring_buffer *ring)
  48 {
  49         int space = (ring->head & HEAD_ADDR) - (ring->tail + 8);
  50         if (space < 0)
  51                 space += ring->size;
  52         return space;
  53 }
  54
  55 static int
  56 gen2_render_ring_flush(struct intel_ring_buffer *ring,
  57                        u32      invalidate_domains,
  58                        u32      flush_domains)
  59 {
  60         u32 cmd;
  61         int ret;
  62
  63         cmd = MI_FLUSH;
  64         if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
  65                 cmd |= MI_NO_WRITE_FLUSH;
  66
  67         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
  68                 cmd |= MI_READ_FLUSH;
  69
  70         ret = intel_ring_begin(ring, 2);
  71         if (ret)
  72                 return ret;
  73
  74         intel_ring_emit(ring, cmd);
  75         intel_ring_emit(ring, MI_NOOP);
  76         intel_ring_advance(ring);
  77
  78         return 0;
  79 }
  80
  81 static int
  82 gen4_render_ring_flush(struct intel_ring_buffer *ring,
  83                        u32      invalidate_domains,
  84                        u32      flush_domains)
  85 {
  86         struct drm_device *dev = ring->dev;
  87         u32 cmd;
  88         int ret;
  89
  90         /*
  91          * read/write caches:
  92          *
  93          * I915_GEM_DOMAIN_RENDER is always invalidated, but is
  94          * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
  95          * also flushed at 2d versus 3d pipeline switches.
  96          *
  97          * read-only caches:
  98          *
  99          * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
 100          * MI_READ_FLUSH is set, and is always flushed on 965.
 101          *
 102          * I915_GEM_DOMAIN_COMMAND may not exist?
 103          *
 104          * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
 105          * invalidated when MI_EXE_FLUSH is set.
 106          *
 107          * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
 108          * invalidated with every MI_FLUSH.
 109          *
 110          * TLBs:
 111          *
 112          * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
 113          * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
 114          * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
 115          * are flushed at any MI_FLUSH.
 116          */
 117
 118         cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
 119         if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)
 120                 cmd &= ~MI_NO_WRITE_FLUSH;
 121         if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
 122                 cmd |= MI_EXE_FLUSH;
 123
 124         if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
 125             (IS_G4X(dev) || IS_GEN5(dev)))
 126                 cmd |= MI_INVALIDATE_ISP;
 127
 128         ret = intel_ring_begin(ring, 2);
 129         if (ret)
 130                 return ret;
 131
 132         intel_ring_emit(ring, cmd);
 133         intel_ring_emit(ring, MI_NOOP);
 134         intel_ring_advance(ring);
 135
 136         return 0;
 137 }
 138
 139 /**
 140  * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
 141  * implementing two workarounds on gen6.  From section 1.4.7.1
 142  * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
 143  *
 144  * [DevSNB-C+{W/A}] Before any depth stall flush (including those
 145  * produced by non-pipelined state commands), software needs to first
 146  * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
 147  * 0.
 148  *
 149  * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
 150  * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
 151  *
 152  * And the workaround for these two requires this workaround first:
 153  *
 154  * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
 155  * BEFORE the pipe-control with a post-sync op and no write-cache
 156  * flushes.
 157  *
 158  * And this last workaround is tricky because of the requirements on
 159  * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
 160  * volume 2 part 1:
 161  *
 162  *     "1 of the following must also be set:
 163  *      - Render Target Cache Flush Enable ([12] of DW1)
 164  *      - Depth Cache Flush Enable ([0] of DW1)
 165  *      - Stall at Pixel Scoreboard ([1] of DW1)
 166  *      - Depth Stall ([13] of DW1)
 167  *      - Post-Sync Operation ([13] of DW1)
 168  *      - Notify Enable ([8] of DW1)"
 169  *
 170  * The cache flushes require the workaround flush that triggered this
 171  * one, so we can't use it.  Depth stall would trigger the same.
 172  * Post-sync nonzero is what triggered this second workaround, so we
 173  * can't use that one either.  Notify enable is IRQs, which aren't
 174  * really our business.  That leaves only stall at scoreboard.
 175  */
 176 static int
 177 intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring)
 178 {
 179         struct pipe_control *pc = ring->private;
 180         u32 scratch_addr = pc->gtt_offset + 128;
 181         int ret;
 182
 183
 184         ret = intel_ring_begin(ring, 6);
 185         if (ret)
 186                 return ret;
 187
 188         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
 189         intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
 190                         PIPE_CONTROL_STALL_AT_SCOREBOARD);
 191         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
 192         intel_ring_emit(ring, 0); /* low dword */
 193         intel_ring_emit(ring, 0); /* high dword */
 194         intel_ring_emit(ring, MI_NOOP);
 195         intel_ring_advance(ring);
 196
 197         ret = intel_ring_begin(ring, 6);
 198         if (ret)
 199                 return ret;
 200
 201         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
 202         intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
 203         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
 204         intel_ring_emit(ring, 0);
 205         intel_ring_emit(ring, 0);
 206         intel_ring_emit(ring, MI_NOOP);
 207         intel_ring_advance(ring);
 208
 209         return 0;
 210 }
 211
 212 static int
 213 gen6_render_ring_flush(struct intel_ring_buffer *ring,
 214                          u32 invalidate_domains, u32 flush_domains)
 215 {
 216         u32 flags = 0;
 217         struct pipe_control *pc = ring->private;
 218         u32 scratch_addr = pc->gtt_offset + 128;
 219         int ret;
 220
 221         /* Force SNB workarounds for PIPE_CONTROL flushes */
 222         intel_emit_post_sync_nonzero_flush(ring);
 223
 224         /* Just flush everything.  Experiments have shown that reducing the
 225          * number of bits based on the write domains has little performance
 226          * impact.
 227          */
 228         flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
 229         flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
 230         flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
 231         flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
 232         flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
 233         flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
 234         flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
 235
 236         ret = intel_ring_begin(ring, 6);
 237         if (ret)
 238                 return ret;
 239
 240         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
 241         intel_ring_emit(ring, flags);
 242         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
 243         intel_ring_emit(ring, 0); /* lower dword */
 244         intel_ring_emit(ring, 0); /* uppwer dword */
 245         intel_ring_emit(ring, MI_NOOP);
 246         intel_ring_advance(ring);
 247
 248         return 0;
 249 }
 250
 251 static void ring_write_tail(struct intel_ring_buffer *ring,
 252                             u32 value)
 253 {
 254         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 255         I915_WRITE_TAIL(ring, value);
 256 }
 257
 258 u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
 259 {
 260         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 261         u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
 262                         RING_ACTHD(ring->mmio_base) : ACTHD;
 263
 264         return I915_READ(acthd_reg);
 265 }
 266
 267 static int init_ring_common(struct intel_ring_buffer *ring)
 268 {
 269         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 270         struct drm_i915_gem_object *obj = ring->obj;
 271         u32 head;
 272
 273         /* Stop the ring if it's running. */
 274         I915_WRITE_CTL(ring, 0);
 275         I915_WRITE_HEAD(ring, 0);
 276         ring->write_tail(ring, 0);
 277
 278         /* Initialize the ring. */
 279         I915_WRITE_START(ring, obj->gtt_offset);
 280         head = I915_READ_HEAD(ring) & HEAD_ADDR;
 281
 282         /* G45 ring initialization fails to reset head to zero */
 283         if (head != 0) {
 284                 DRM_DEBUG_KMS("%s head not reset to zero "
 285                               "ctl %08x head %08x tail %08x start %08x\n",
 286                               ring->name,
 287                               I915_READ_CTL(ring),
 288                               I915_READ_HEAD(ring),
 289                               I915_READ_TAIL(ring),
 290                               I915_READ_START(ring));
 291
 292                 I915_WRITE_HEAD(ring, 0);
 293
 294                 if (I915_READ_HEAD(ring) & HEAD_ADDR) {
 295                         DRM_ERROR("failed to set %s head to zero "
 296                                   "ctl %08x head %08x tail %08x start %08x\n",
 297                                   ring->name,
 298                                   I915_READ_CTL(ring),
 299                                   I915_READ_HEAD(ring),
 300                                   I915_READ_TAIL(ring),
 301                                   I915_READ_START(ring));
 302                 }
 303         }
 304
 305         I915_WRITE_CTL(ring,
 306                         ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
 307                         | RING_VALID);
 308
 309         /* If the head is still not zero, the ring is dead */
 310         if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
 311                      I915_READ_START(ring) == obj->gtt_offset &&
 312                      (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
 313                 DRM_ERROR("%s initialization failed "
 314                                 "ctl %08x head %08x tail %08x start %08x\n",
 315                                 ring->name,
 316                                 I915_READ_CTL(ring),
 317                                 I915_READ_HEAD(ring),
 318                                 I915_READ_TAIL(ring),
 319                                 I915_READ_START(ring));
 320                 return -EIO;
 321         }
 322
 323         if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
 324                 i915_kernel_lost_context(ring->dev);
 325         else {
 326                 ring->head = I915_READ_HEAD(ring);
 327                 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
 328                 ring->space = ring_space(ring);
 329         }
 330
 331         return 0;
 332 }
 333
 334 static int
 335 init_pipe_control(struct intel_ring_buffer *ring)
 336 {
 337         struct pipe_control *pc;
 338         struct drm_i915_gem_object *obj;
 339         int ret;
 340
 341         if (ring->private)
 342                 return 0;
 343
 344         pc = kmalloc(sizeof(*pc), GFP_KERNEL);
 345         if (!pc)
 346                 return -ENOMEM;
 347
 348         obj = i915_gem_alloc_object(ring->dev, 4096);
 349         if (obj == NULL) {
 350                 DRM_ERROR("Failed to allocate seqno page\n");
 351                 ret = -ENOMEM;
 352                 goto err;
 353         }
 354
 355         i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
 356
 357         ret = i915_gem_object_pin(obj, 4096, true);
 358         if (ret)
 359                 goto err_unref;
 360
 361         pc->gtt_offset = obj->gtt_offset;
 362         pc->cpu_page =  kmap(obj->pages[0]);
 363         if (pc->cpu_page == NULL)
 364                 goto err_unpin;
 365
 366         pc->obj = obj;
 367         ring->private = pc;
 368         return 0;
 369
 370 err_unpin:
 371         i915_gem_object_unpin(obj);
 372 err_unref:
 373         drm_gem_object_unreference(&obj->base);
 374 err:
 375         kfree(pc);
 376         return ret;
 377 }
 378
 379 static void
 380 cleanup_pipe_control(struct intel_ring_buffer *ring)
 381 {
 382         struct pipe_control *pc = ring->private;
 383         struct drm_i915_gem_object *obj;
 384
 385         if (!ring->private)
 386                 return;
 387
 388         obj = pc->obj;
 389         kunmap(obj->pages[0]);
 390         i915_gem_object_unpin(obj);
 391         drm_gem_object_unreference(&obj->base);
 392
 393         kfree(pc);
 394         ring->private = NULL;
 395 }
 396
 397 static int init_render_ring(struct intel_ring_buffer *ring)
 398 {
 399         struct drm_device *dev = ring->dev;
 400         struct drm_i915_private *dev_priv = dev->dev_private;
 401         int ret = init_ring_common(ring);
 402
 403         if (INTEL_INFO(dev)->gen > 3) {
 404                 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
 405                 if (IS_GEN7(dev))
 406                         I915_WRITE(GFX_MODE_GEN7,
 407                                    _MASKED_BIT_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) |
 408                                    _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
 409         }
 410
 411         if (INTEL_INFO(dev)->gen >= 5) {
 412                 ret = init_pipe_control(ring);
 413                 if (ret)
 414                         return ret;
 415         }
 416
 417         if (IS_GEN6(dev)) {
 418                 /* From the Sandybridge PRM, volume 1 part 3, page 24:
 419                  * "If this bit is set, STCunit will have LRA as replacement
 420                  *  policy. [...] This bit must be reset.  LRA replacement
 421                  *  policy is not supported."
 422                  */
 423                 I915_WRITE(CACHE_MODE_0,
 424                            _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
 425         }
 426
 427         if (INTEL_INFO(dev)->gen >= 6)
 428                 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
 429
 430         return ret;
 431 }
 432
 433 static void render_ring_cleanup(struct intel_ring_buffer *ring)
 434 {
 435         if (!ring->private)
 436                 return;
 437
 438         cleanup_pipe_control(ring);
 439 }
 440
 441 static void
 442 update_mboxes(struct intel_ring_buffer *ring,
 443             u32 seqno,
 444             u32 mmio_offset)
 445 {
 446         intel_ring_emit(ring, MI_SEMAPHORE_MBOX |
 447                               MI_SEMAPHORE_GLOBAL_GTT |
 448                               MI_SEMAPHORE_REGISTER |
 449                               MI_SEMAPHORE_UPDATE);
 450         intel_ring_emit(ring, seqno);
 451         intel_ring_emit(ring, mmio_offset);
 452 }
 453
 454 /**
 455  * gen6_add_request - Update the semaphore mailbox registers
 456  *
 457  * @ring - ring that is adding a request
 458  * @seqno - return seqno stuck into the ring
 459  *
 460  * Update the mailbox registers in the *other* rings with the current seqno.
 461  * This acts like a signal in the canonical semaphore.
 462  */
 463 static int
 464 gen6_add_request(struct intel_ring_buffer *ring,
 465                  u32 *seqno)
 466 {
 467         u32 mbox1_reg;
 468         u32 mbox2_reg;
 469         int ret;
 470
 471         ret = intel_ring_begin(ring, 10);
 472         if (ret)
 473                 return ret;
 474
 475         mbox1_reg = ring->signal_mbox[0];
 476         mbox2_reg = ring->signal_mbox[1];
 477
 478         *seqno = i915_gem_next_request_seqno(ring);
 479
 480         update_mboxes(ring, *seqno, mbox1_reg);
 481         update_mboxes(ring, *seqno, mbox2_reg);
 482         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
 483         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 484         intel_ring_emit(ring, *seqno);
 485         intel_ring_emit(ring, MI_USER_INTERRUPT);
 486         intel_ring_advance(ring);
 487
 488         return 0;
 489 }
 490
 491 /**
 492  * intel_ring_sync - sync the waiter to the signaller on seqno
 493  *
 494  * @waiter - ring that is waiting
 495  * @signaller - ring which has, or will signal
 496  * @seqno - seqno which the waiter will block on
 497  */
 498 static int
 499 gen6_ring_sync(struct intel_ring_buffer *waiter,
 500                struct intel_ring_buffer *signaller,
 501                u32 seqno)
 502 {
 503         int ret;
 504         u32 dw1 = MI_SEMAPHORE_MBOX |
 505                   MI_SEMAPHORE_COMPARE |
 506                   MI_SEMAPHORE_REGISTER;
 507
 508         /* Throughout all of the GEM code, seqno passed implies our current
 509          * seqno is >= the last seqno executed. However for hardware the
 510          * comparison is strictly greater than.
 511          */
 512         seqno -= 1;
 513
 514         WARN_ON(signaller->semaphore_register[waiter->id] ==
 515                 MI_SEMAPHORE_SYNC_INVALID);
 516
 517         ret = intel_ring_begin(waiter, 4);
 518         if (ret)
 519                 return ret;
 520
 521         intel_ring_emit(waiter,
 522                         dw1 | signaller->semaphore_register[waiter->id]);
 523         intel_ring_emit(waiter, seqno);
 524         intel_ring_emit(waiter, 0);
 525         intel_ring_emit(waiter, MI_NOOP);
 526         intel_ring_advance(waiter);
 527
 528         return 0;
 529 }
 530
 531 #define PIPE_CONTROL_FLUSH(ring__, addr__)                                      \
 532 do {                                                                    \
 533         intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |                \
 534                  PIPE_CONTROL_DEPTH_STALL);                             \
 535         intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
 536         intel_ring_emit(ring__, 0);                                                     \
 537         intel_ring_emit(ring__, 0);                                                     \
 538 } while (0)
 539
 540 static int
 541 pc_render_add_request(struct intel_ring_buffer *ring,
 542                       u32 *result)
 543 {
 544         u32 seqno = i915_gem_next_request_seqno(ring);
 545         struct pipe_control *pc = ring->private;
 546         u32 scratch_addr = pc->gtt_offset + 128;
 547         int ret;
 548
 549         /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
 550          * incoherent with writes to memory, i.e. completely fubar,
 551          * so we need to use PIPE_NOTIFY instead.
 552          *
 553          * However, we also need to workaround the qword write
 554          * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
 555          * memory before requesting an interrupt.
 556          */
 557         ret = intel_ring_begin(ring, 32);
 558         if (ret)
 559                 return ret;
 560
 561         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
 562                         PIPE_CONTROL_WRITE_FLUSH |
 563                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
 564         intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
 565         intel_ring_emit(ring, seqno);
 566         intel_ring_emit(ring, 0);
 567         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 568         scratch_addr += 128; /* write to separate cachelines */
 569         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 570         scratch_addr += 128;
 571         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 572         scratch_addr += 128;
 573         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 574         scratch_addr += 128;
 575         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 576         scratch_addr += 128;
 577         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 578
 579         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
 580                         PIPE_CONTROL_WRITE_FLUSH |
 581                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
 582                         PIPE_CONTROL_NOTIFY);
 583         intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
 584         intel_ring_emit(ring, seqno);
 585         intel_ring_emit(ring, 0);
 586         intel_ring_advance(ring);
 587
 588         *result = seqno;
 589         return 0;
 590 }
 591
 592 static u32
 593 gen6_ring_get_seqno(struct intel_ring_buffer *ring)
 594 {
 595         struct drm_device *dev = ring->dev;
 596
 597         /* Workaround to force correct ordering between irq and seqno writes on
 598          * ivb (and maybe also on snb) by reading from a CS register (like
 599          * ACTHD) before reading the status page. */
 600         if (IS_GEN6(dev) || IS_GEN7(dev))
 601                 intel_ring_get_active_head(ring);
 602         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 603 }
 604
 605 static u32
 606 ring_get_seqno(struct intel_ring_buffer *ring)
 607 {
 608         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 609 }
 610
 611 static u32
 612 pc_render_get_seqno(struct intel_ring_buffer *ring)
 613 {
 614         struct pipe_control *pc = ring->private;
 615         return pc->cpu_page[0];
 616 }
 617
 618 static bool
 619 gen5_ring_get_irq(struct intel_ring_buffer *ring)
 620 {
 621         struct drm_device *dev = ring->dev;
 622         drm_i915_private_t *dev_priv = dev->dev_private;
 623         unsigned long flags;
 624
 625         if (!dev->irq_enabled)
 626                 return false;
 627
 628         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 629         if (ring->irq_refcount++ == 0) {
 630                 dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
 631                 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 632                 POSTING_READ(GTIMR);
 633         }
 634         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 635
 636         return true;
 637 }
 638
 639 static void
 640 gen5_ring_put_irq(struct intel_ring_buffer *ring)
 641 {
 642         struct drm_device *dev = ring->dev;
 643         drm_i915_private_t *dev_priv = dev->dev_private;
 644         unsigned long flags;
 645
 646         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 647         if (--ring->irq_refcount == 0) {
 648                 dev_priv->gt_irq_mask |= ring->irq_enable_mask;
 649                 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 650                 POSTING_READ(GTIMR);
 651         }
 652         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 653 }
 654
 655 static bool
 656 i9xx_ring_get_irq(struct intel_ring_buffer *ring)
 657 {
 658         struct drm_device *dev = ring->dev;
 659         drm_i915_private_t *dev_priv = dev->dev_private;
 660         unsigned long flags;
 661
 662         if (!dev->irq_enabled)
 663                 return false;
 664
 665         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 666         if (ring->irq_refcount++ == 0) {
 667                 dev_priv->irq_mask &= ~ring->irq_enable_mask;
 668                 I915_WRITE(IMR, dev_priv->irq_mask);
 669                 POSTING_READ(IMR);
 670         }
 671         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 672
 673         return true;
 674 }
 675
 676 static void
 677 i9xx_ring_put_irq(struct intel_ring_buffer *ring)
 678 {
 679         struct drm_device *dev = ring->dev;
 680         drm_i915_private_t *dev_priv = dev->dev_private;
 681         unsigned long flags;
 682
 683         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 684         if (--ring->irq_refcount == 0) {
 685                 dev_priv->irq_mask |= ring->irq_enable_mask;
 686                 I915_WRITE(IMR, dev_priv->irq_mask);
 687                 POSTING_READ(IMR);
 688         }
 689         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 690 }
 691
 692 static bool
 693 i8xx_ring_get_irq(struct intel_ring_buffer *ring)
 694 {
 695         struct drm_device *dev = ring->dev;
 696         drm_i915_private_t *dev_priv = dev->dev_private;
 697         unsigned long flags;
 698
 699         if (!dev->irq_enabled)
 700                 return false;
 701
 702         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 703         if (ring->irq_refcount++ == 0) {
 704                 dev_priv->irq_mask &= ~ring->irq_enable_mask;
 705                 I915_WRITE16(IMR, dev_priv->irq_mask);
 706                 POSTING_READ16(IMR);
 707         }
 708         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 709
 710         return true;
 711 }
 712
 713 static void
 714 i8xx_ring_put_irq(struct intel_ring_buffer *ring)
 715 {
 716         struct drm_device *dev = ring->dev;
 717         drm_i915_private_t *dev_priv = dev->dev_private;
 718         unsigned long flags;
 719
 720         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 721         if (--ring->irq_refcount == 0) {
 722                 dev_priv->irq_mask |= ring->irq_enable_mask;
 723                 I915_WRITE16(IMR, dev_priv->irq_mask);
 724                 POSTING_READ16(IMR);
 725         }
 726         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 727 }
 728
 729 void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
 730 {
 731         struct drm_device *dev = ring->dev;
 732         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 733         u32 mmio = 0;
 734
 735         /* The ring status page addresses are no longer next to the rest of
 736          * the ring registers as of gen7.
 737          */
 738         if (IS_GEN7(dev)) {
 739                 switch (ring->id) {
 740                 case RCS:
 741                         mmio = RENDER_HWS_PGA_GEN7;
 742                         break;
 743                 case BCS:
 744                         mmio = BLT_HWS_PGA_GEN7;
 745                         break;
 746                 case VCS:
 747                         mmio = BSD_HWS_PGA_GEN7;
 748                         break;
 749                 }
 750         } else if (IS_GEN6(ring->dev)) {
 751                 mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
 752         } else {
 753                 mmio = RING_HWS_PGA(ring->mmio_base);
 754         }
 755
 756         I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
 757         POSTING_READ(mmio);
 758 }
 759
 760 static int
 761 bsd_ring_flush(struct intel_ring_buffer *ring,
 762                u32     invalidate_domains,
 763                u32     flush_domains)
 764 {
 765         int ret;
 766
 767         ret = intel_ring_begin(ring, 2);
 768         if (ret)
 769                 return ret;
 770
 771         intel_ring_emit(ring, MI_FLUSH);
 772         intel_ring_emit(ring, MI_NOOP);
 773         intel_ring_advance(ring);
 774         return 0;
 775 }
 776
 777 static int
 778 i9xx_add_request(struct intel_ring_buffer *ring,
 779                  u32 *result)
 780 {
 781         u32 seqno;
 782         int ret;
 783
 784         ret = intel_ring_begin(ring, 4);
 785         if (ret)
 786                 return ret;
 787
 788         seqno = i915_gem_next_request_seqno(ring);
 789
 790         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
 791         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 792         intel_ring_emit(ring, seqno);
 793         intel_ring_emit(ring, MI_USER_INTERRUPT);
 794         intel_ring_advance(ring);
 795
 796         *result = seqno;
 797         return 0;
 798 }
 799
 800 static bool
 801 gen6_ring_get_irq(struct intel_ring_buffer *ring)
 802 {
 803         struct drm_device *dev = ring->dev;
 804         drm_i915_private_t *dev_priv = dev->dev_private;
 805         unsigned long flags;
 806
 807         if (!dev->irq_enabled)
 808                return false;
 809
 810         /* It looks like we need to prevent the gt from suspending while waiting
 811          * for an notifiy irq, otherwise irqs seem to get lost on at least the
 812          * blt/bsd rings on ivb. */
 813         gen6_gt_force_wake_get(dev_priv);
 814
 815         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 816         if (ring->irq_refcount++ == 0) {
 817                 I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
 818                 dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
 819                 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 820                 POSTING_READ(GTIMR);
 821         }
 822         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 823
 824         return true;
 825 }
 826
 827 static void
 828 gen6_ring_put_irq(struct intel_ring_buffer *ring)
 829 {
 830         struct drm_device *dev = ring->dev;
 831         drm_i915_private_t *dev_priv = dev->dev_private;
 832         unsigned long flags;
 833
 834         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 835         if (--ring->irq_refcount == 0) {
 836                 I915_WRITE_IMR(ring, ~0);
 837                 dev_priv->gt_irq_mask |= ring->irq_enable_mask;
 838                 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 839                 POSTING_READ(GTIMR);
 840         }
 841         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 842
 843         gen6_gt_force_wake_put(dev_priv);
 844 }
 845
 846 static int
 847 i965_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length)
 848 {
 849         int ret;
 850
 851         ret = intel_ring_begin(ring, 2);
 852         if (ret)
 853                 return ret;
 854
 855         intel_ring_emit(ring,
 856                         MI_BATCH_BUFFER_START |
 857                         MI_BATCH_GTT |
 858                         MI_BATCH_NON_SECURE_I965);
 859         intel_ring_emit(ring, offset);
 860         intel_ring_advance(ring);
 861
 862         return 0;
 863 }
 864
 865 static int
 866 i830_dispatch_execbuffer(struct intel_ring_buffer *ring,
 867                                 u32 offset, u32 len)
 868 {
 869         int ret;
 870
 871         ret = intel_ring_begin(ring, 4);
 872         if (ret)
 873                 return ret;
 874
 875         intel_ring_emit(ring, MI_BATCH_BUFFER);
 876         intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
 877         intel_ring_emit(ring, offset + len - 8);
 878         intel_ring_emit(ring, 0);
 879         intel_ring_advance(ring);
 880
 881         return 0;
 882 }
 883
 884 static int
 885 i915_dispatch_execbuffer(struct intel_ring_buffer *ring,
 886                                 u32 offset, u32 len)
 887 {
 888         int ret;
 889
 890         ret = intel_ring_begin(ring, 2);
 891         if (ret)
 892                 return ret;
 893
 894         intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
 895         intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
 896         intel_ring_advance(ring);
 897
 898         return 0;
 899 }
 900
 901 static void cleanup_status_page(struct intel_ring_buffer *ring)
 902 {
 903         struct drm_i915_gem_object *obj;
 904
 905         obj = ring->status_page.obj;
 906         if (obj == NULL)
 907                 return;
 908
 909         kunmap(obj->pages[0]);
 910         i915_gem_object_unpin(obj);
 911         drm_gem_object_unreference(&obj->base);
 912         ring->status_page.obj = NULL;
 913 }
 914
 915 static int init_status_page(struct intel_ring_buffer *ring)
 916 {
 917         struct drm_device *dev = ring->dev;
 918         struct drm_i915_gem_object *obj;
 919         int ret;
 920
 921         obj = i915_gem_alloc_object(dev, 4096);
 922         if (obj == NULL) {
 923                 DRM_ERROR("Failed to allocate status page\n");
 924                 ret = -ENOMEM;
 925                 goto err;
 926         }
 927
 928         i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
 929
 930         ret = i915_gem_object_pin(obj, 4096, true);
 931         if (ret != 0) {
 932                 goto err_unref;
 933         }
 934
 935         ring->status_page.gfx_addr = obj->gtt_offset;
 936         ring->status_page.page_addr = kmap(obj->pages[0]);
 937         if (ring->status_page.page_addr == NULL) {
 938                 goto err_unpin;
 939         }
 940         ring->status_page.obj = obj;
 941         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
 942
 943         intel_ring_setup_status_page(ring);
 944         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
 945                         ring->name, ring->status_page.gfx_addr);
 946
 947         return 0;
 948
 949 err_unpin:
 950         i915_gem_object_unpin(obj);
 951 err_unref:
 952         drm_gem_object_unreference(&obj->base);
 953 err:
 954         return ret;
 955 }
 956
 957 static int intel_init_ring_buffer(struct drm_device *dev,
 958                                   struct intel_ring_buffer *ring)
 959 {
 960         struct drm_i915_gem_object *obj;
 961         int ret;
 962
 963         ring->dev = dev;
 964         INIT_LIST_HEAD(&ring->active_list);
 965         INIT_LIST_HEAD(&ring->request_list);
 966         INIT_LIST_HEAD(&ring->gpu_write_list);
 967         ring->size = 32 * PAGE_SIZE;
 968
 969         init_waitqueue_head(&ring->irq_queue);
 970
 971         if (I915_NEED_GFX_HWS(dev)) {
 972                 ret = init_status_page(ring);
 973                 if (ret)
 974                         return ret;
 975         }
 976
 977         obj = i915_gem_alloc_object(dev, ring->size);
 978         if (obj == NULL) {
 979                 DRM_ERROR("Failed to allocate ringbuffer\n");
 980                 ret = -ENOMEM;
 981                 goto err_hws;
 982         }
 983
 984         ring->obj = obj;
 985
 986         ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
 987         if (ret)
 988                 goto err_unref;
 989
 990         ring->virtual_start = ioremap_wc(dev->agp->base + obj->gtt_offset,
 991                                          ring->size);
 992         if (ring->virtual_start == NULL) {
 993                 DRM_ERROR("Failed to map ringbuffer.\n");
 994                 ret = -EINVAL;
 995                 goto err_unpin;
 996         }
 997
 998         ret = ring->init(ring);
 999         if (ret)
1000                 goto err_unmap;
1001
1002         /* Workaround an erratum on the i830 which causes a hang if
1003          * the TAIL pointer points to within the last 2 cachelines
1004          * of the buffer.
1005          */
1006         ring->effective_size = ring->size;
1007         if (IS_I830(ring->dev) || IS_845G(ring->dev))
1008                 ring->effective_size -= 128;
1009
1010         return 0;
1011
1012 err_unmap:
1013         iounmap(ring->virtual_start);
1014 err_unpin:
1015         i915_gem_object_unpin(obj);
1016 err_unref:
1017         drm_gem_object_unreference(&obj->base);
1018         ring->obj = NULL;
1019 err_hws:
1020         cleanup_status_page(ring);
1021         return ret;
1022 }
1023
1024 void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
1025 {
1026         struct drm_i915_private *dev_priv;
1027         int ret;
1028
1029         if (ring->obj == NULL)
1030                 return;
1031
1032         /* Disable the ring buffer. The ring must be idle at this point */
1033         dev_priv = ring->dev->dev_private;
1034         ret = intel_wait_ring_idle(ring);
1035         if (ret)
1036                 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
1037                           ring->name, ret);
1038
1039         I915_WRITE_CTL(ring, 0);
1040
1041         iounmap(ring->virtual_start);
1042
1043         i915_gem_object_unpin(ring->obj);
1044         drm_gem_object_unreference(&ring->obj->base);
1045         ring->obj = NULL;
1046
1047         if (ring->cleanup)
1048                 ring->cleanup(ring);
1049
1050         cleanup_status_page(ring);
1051 }
1052
1053 static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
1054 {
1055         uint32_t __iomem *virt;
1056         int rem = ring->size - ring->tail;
1057
1058         if (ring->space < rem) {
1059                 int ret = intel_wait_ring_buffer(ring, rem);
1060                 if (ret)
1061                         return ret;
1062         }
1063
1064         virt = ring->virtual_start + ring->tail;
1065         rem /= 4;
1066         while (rem--)
1067                 iowrite32(MI_NOOP, virt++);
1068
1069         ring->tail = 0;
1070         ring->space = ring_space(ring);
1071
1072         return 0;
1073 }
1074
1075 static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno)
1076 {
1077         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1078         bool was_interruptible;
1079         int ret;
1080
1081         /* XXX As we have not yet audited all the paths to check that
1082          * they are ready for ERESTARTSYS from intel_ring_begin, do not
1083          * allow us to be interruptible by a signal.
1084          */
1085         was_interruptible = dev_priv->mm.interruptible;
1086         dev_priv->mm.interruptible = false;
1087
1088         ret = i915_wait_request(ring, seqno);
1089
1090         dev_priv->mm.interruptible = was_interruptible;
1091         if (!ret)
1092                 i915_gem_retire_requests_ring(ring);
1093
1094         return ret;
1095 }
1096
1097 static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n)
1098 {
1099         struct drm_i915_gem_request *request;
1100         u32 seqno = 0;
1101         int ret;
1102
1103         i915_gem_retire_requests_ring(ring);
1104
1105         if (ring->last_retired_head != -1) {
1106                 ring->head = ring->last_retired_head;
1107                 ring->last_retired_head = -1;
1108                 ring->space = ring_space(ring);
1109                 if (ring->space >= n)
1110                         return 0;
1111         }
1112
1113         list_for_each_entry(request, &ring->request_list, list) {
1114                 int space;
1115
1116                 if (request->tail == -1)
1117                         continue;
1118
1119                 space = request->tail - (ring->tail + 8);
1120                 if (space < 0)
1121                         space += ring->size;
1122                 if (space >= n) {
1123                         seqno = request->seqno;
1124                         break;
1125                 }
1126
1127                 /* Consume this request in case we need more space than
1128                  * is available and so need to prevent a race between
1129                  * updating last_retired_head and direct reads of
1130                  * I915_RING_HEAD. It also provides a nice sanity check.
1131                  */
1132                 request->tail = -1;
1133         }
1134
1135         if (seqno == 0)
1136                 return -ENOSPC;
1137
1138         ret = intel_ring_wait_seqno(ring, seqno);
1139         if (ret)
1140                 return ret;
1141
1142         if (WARN_ON(ring->last_retired_head == -1))
1143                 return -ENOSPC;
1144
1145         ring->head = ring->last_retired_head;
1146         ring->last_retired_head = -1;
1147         ring->space = ring_space(ring);
1148         if (WARN_ON(ring->space < n))
1149                 return -ENOSPC;
1150
1151         return 0;
1152 }
1153
1154 int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
1155 {
1156         struct drm_device *dev = ring->dev;
1157         struct drm_i915_private *dev_priv = dev->dev_private;
1158         unsigned long end;
1159         int ret;
1160
1161         ret = intel_ring_wait_request(ring, n);
1162         if (ret != -ENOSPC)
1163                 return ret;
1164
1165         trace_i915_ring_wait_begin(ring);
1166         /* With GEM the hangcheck timer should kick us out of the loop,
1167          * leaving it early runs the risk of corrupting GEM state (due
1168          * to running on almost untested codepaths). But on resume
1169          * timers don't work yet, so prevent a complete hang in that
1170          * case by choosing an insanely large timeout. */
1171         end = jiffies + 60 * HZ;
1172
1173         do {
1174                 ring->head = I915_READ_HEAD(ring);
1175                 ring->space = ring_space(ring);
1176                 if (ring->space >= n) {
1177                         trace_i915_ring_wait_end(ring);
1178                         return 0;
1179                 }
1180
1181                 if (dev->primary->master) {
1182                         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
1183                         if (master_priv->sarea_priv)
1184                                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
1185                 }
1186
1187                 msleep(1);
1188                 if (atomic_read(&dev_priv->mm.wedged))
1189                         return -EAGAIN;
1190         } while (!time_after(jiffies, end));
1191         trace_i915_ring_wait_end(ring);
1192         return -EBUSY;
1193 }
1194
1195 int intel_ring_begin(struct intel_ring_buffer *ring,
1196                      int num_dwords)
1197 {
1198         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1199         int n = 4*num_dwords;
1200         int ret;
1201
1202         if (unlikely(atomic_read(&dev_priv->mm.wedged)))
1203                 return -EIO;
1204
1205         if (unlikely(ring->tail + n > ring->effective_size)) {
1206                 ret = intel_wrap_ring_buffer(ring);
1207                 if (unlikely(ret))
1208                         return ret;
1209         }
1210
1211         if (unlikely(ring->space < n)) {
1212                 ret = intel_wait_ring_buffer(ring, n);
1213                 if (unlikely(ret))
1214                         return ret;
1215         }
1216
1217         ring->space -= n;
1218         return 0;
1219 }
1220
1221 void intel_ring_advance(struct intel_ring_buffer *ring)
1222 {
1223         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1224
1225         ring->tail &= ring->size - 1;
1226         if (dev_priv->stop_rings & intel_ring_flag(ring))
1227                 return;
1228         ring->write_tail(ring, ring->tail);
1229 }
1230
1231
1232 static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
1233                                      u32 value)
1234 {
1235         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1236
1237        /* Every tail move must follow the sequence below */
1238         I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1239                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
1240                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
1241         I915_WRITE(GEN6_BSD_RNCID, 0x0);
1242
1243         if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
1244                 GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
1245                 50))
1246         DRM_ERROR("timed out waiting for IDLE Indicator\n");
1247
1248         I915_WRITE_TAIL(ring, value);
1249         I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1250                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
1251                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
1252 }
1253
1254 static int gen6_ring_flush(struct intel_ring_buffer *ring,
1255                            u32 invalidate, u32 flush)
1256 {
1257         uint32_t cmd;
1258         int ret;
1259
1260         ret = intel_ring_begin(ring, 4);
1261         if (ret)
1262                 return ret;
1263
1264         cmd = MI_FLUSH_DW;
1265         if (invalidate & I915_GEM_GPU_DOMAINS)
1266                 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD;
1267         intel_ring_emit(ring, cmd);
1268         intel_ring_emit(ring, 0);
1269         intel_ring_emit(ring, 0);
1270         intel_ring_emit(ring, MI_NOOP);
1271         intel_ring_advance(ring);
1272         return 0;
1273 }
1274
1275 static int
1276 gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
1277                               u32 offset, u32 len)
1278 {
1279         int ret;
1280
1281         ret = intel_ring_begin(ring, 2);
1282         if (ret)
1283                 return ret;
1284
1285         intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
1286         /* bit0-7 is the length on GEN6+ */
1287         intel_ring_emit(ring, offset);
1288         intel_ring_advance(ring);
1289
1290         return 0;
1291 }
1292
1293 /* Blitter support (SandyBridge+) */
1294
1295 static int blt_ring_flush(struct intel_ring_buffer *ring,
1296                           u32 invalidate, u32 flush)
1297 {
1298         uint32_t cmd;
1299         int ret;
1300
1301         ret = intel_ring_begin(ring, 4);
1302         if (ret)
1303                 return ret;
1304
1305         cmd = MI_FLUSH_DW;
1306         if (invalidate & I915_GEM_DOMAIN_RENDER)
1307                 cmd |= MI_INVALIDATE_TLB;
1308         intel_ring_emit(ring, cmd);
1309         intel_ring_emit(ring, 0);
1310         intel_ring_emit(ring, 0);
1311         intel_ring_emit(ring, MI_NOOP);
1312         intel_ring_advance(ring);
1313         return 0;
1314 }
1315
1316 int intel_init_render_ring_buffer(struct drm_device *dev)
1317 {
1318         drm_i915_private_t *dev_priv = dev->dev_private;
1319         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1320
1321         ring->name = "render ring";
1322         ring->id = RCS;
1323         ring->mmio_base = RENDER_RING_BASE;
1324
1325         if (INTEL_INFO(dev)->gen >= 6) {
1326                 ring->add_request = gen6_add_request;
1327                 ring->flush = gen6_render_ring_flush;
1328                 ring->irq_get = gen6_ring_get_irq;
1329                 ring->irq_put = gen6_ring_put_irq;
1330                 ring->irq_enable_mask = GT_USER_INTERRUPT;
1331                 ring->get_seqno = gen6_ring_get_seqno;
1332                 ring->sync_to = gen6_ring_sync;
1333                 ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_INVALID;
1334                 ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_RV;
1335                 ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_RB;
1336                 ring->signal_mbox[0] = GEN6_VRSYNC;
1337                 ring->signal_mbox[1] = GEN6_BRSYNC;
1338         } else if (IS_GEN5(dev)) {
1339                 ring->add_request = pc_render_add_request;
1340                 ring->flush = gen4_render_ring_flush;
1341                 ring->get_seqno = pc_render_get_seqno;
1342                 ring->irq_get = gen5_ring_get_irq;
1343                 ring->irq_put = gen5_ring_put_irq;
1344                 ring->irq_enable_mask = GT_USER_INTERRUPT | GT_PIPE_NOTIFY;
1345         } else {
1346                 ring->add_request = i9xx_add_request;
1347                 if (INTEL_INFO(dev)->gen < 4)
1348                         ring->flush = gen2_render_ring_flush;
1349                 else
1350                         ring->flush = gen4_render_ring_flush;
1351                 ring->get_seqno = ring_get_seqno;
1352                 if (IS_GEN2(dev)) {
1353                         ring->irq_get = i8xx_ring_get_irq;
1354                         ring->irq_put = i8xx_ring_put_irq;
1355                 } else {
1356                         ring->irq_get = i9xx_ring_get_irq;
1357                         ring->irq_put = i9xx_ring_put_irq;
1358                 }
1359                 ring->irq_enable_mask = I915_USER_INTERRUPT;
1360         }
1361         ring->write_tail = ring_write_tail;
1362         if (INTEL_INFO(dev)->gen >= 6)
1363                 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
1364         else if (INTEL_INFO(dev)->gen >= 4)
1365                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
1366         else if (IS_I830(dev) || IS_845G(dev))
1367                 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
1368         else
1369                 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
1370         ring->init = init_render_ring;
1371         ring->cleanup = render_ring_cleanup;
1372
1373
1374         if (!I915_NEED_GFX_HWS(dev)) {
1375                 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1376                 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1377         }
1378
1379         return intel_init_ring_buffer(dev, ring);
1380 }
1381
1382 int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
1383 {
1384         drm_i915_private_t *dev_priv = dev->dev_private;
1385         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1386
1387         ring->name = "render ring";
1388         ring->id = RCS;
1389         ring->mmio_base = RENDER_RING_BASE;
1390
1391         if (INTEL_INFO(dev)->gen >= 6) {
1392                 /* non-kms not supported on gen6+ */
1393                 return -ENODEV;
1394         }
1395
1396         /* Note: gem is not supported on gen5/ilk without kms (the corresponding
1397          * gem_init ioctl returns with -ENODEV). Hence we do not need to set up
1398          * the special gen5 functions. */
1399         ring->add_request = i9xx_add_request;
1400         if (INTEL_INFO(dev)->gen < 4)
1401                 ring->flush = gen2_render_ring_flush;
1402         else
1403                 ring->flush = gen4_render_ring_flush;
1404         ring->get_seqno = ring_get_seqno;
1405         if (IS_GEN2(dev)) {
1406                 ring->irq_get = i8xx_ring_get_irq;
1407                 ring->irq_put = i8xx_ring_put_irq;
1408         } else {
1409                 ring->irq_get = i9xx_ring_get_irq;
1410                 ring->irq_put = i9xx_ring_put_irq;
1411         }
1412         ring->irq_enable_mask = I915_USER_INTERRUPT;
1413         ring->write_tail = ring_write_tail;
1414         if (INTEL_INFO(dev)->gen >= 4)
1415                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
1416         else if (IS_I830(dev) || IS_845G(dev))
1417                 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
1418         else
1419                 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
1420         ring->init = init_render_ring;
1421         ring->cleanup = render_ring_cleanup;
1422
1423         if (!I915_NEED_GFX_HWS(dev))
1424                 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1425
1426         ring->dev = dev;
1427         INIT_LIST_HEAD(&ring->active_list);
1428         INIT_LIST_HEAD(&ring->request_list);
1429         INIT_LIST_HEAD(&ring->gpu_write_list);
1430
1431         ring->size = size;
1432         ring->effective_size = ring->size;
1433         if (IS_I830(ring->dev))
1434                 ring->effective_size -= 128;
1435
1436         ring->virtual_start = ioremap_wc(start, size);
1437         if (ring->virtual_start == NULL) {
1438                 DRM_ERROR("can not ioremap virtual address for"
1439                           " ring buffer\n");
1440                 return -ENOMEM;
1441         }
1442
1443         return 0;
1444 }
1445
1446 int intel_init_bsd_ring_buffer(struct drm_device *dev)
1447 {
1448         drm_i915_private_t *dev_priv = dev->dev_private;
1449         struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
1450
1451         ring->name = "bsd ring";
1452         ring->id = VCS;
1453
1454         ring->write_tail = ring_write_tail;
1455         if (IS_GEN6(dev) || IS_GEN7(dev)) {
1456                 ring->mmio_base = GEN6_BSD_RING_BASE;
1457                 /* gen6 bsd needs a special wa for tail updates */
1458                 if (IS_GEN6(dev))
1459                         ring->write_tail = gen6_bsd_ring_write_tail;
1460                 ring->flush = gen6_ring_flush;
1461                 ring->add_request = gen6_add_request;
1462                 ring->get_seqno = gen6_ring_get_seqno;
1463                 ring->irq_enable_mask = GEN6_BSD_USER_INTERRUPT;
1464                 ring->irq_get = gen6_ring_get_irq;
1465                 ring->irq_put = gen6_ring_put_irq;
1466                 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
1467                 ring->sync_to = gen6_ring_sync;
1468                 ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_VR;
1469                 ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_INVALID;
1470                 ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_VB;
1471                 ring->signal_mbox[0] = GEN6_RVSYNC;
1472                 ring->signal_mbox[1] = GEN6_BVSYNC;
1473         } else {
1474                 ring->mmio_base = BSD_RING_BASE;
1475                 ring->flush = bsd_ring_flush;
1476                 ring->add_request = i9xx_add_request;
1477                 ring->get_seqno = ring_get_seqno;
1478                 if (IS_GEN5(dev)) {
1479                         ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
1480                         ring->irq_get = gen5_ring_get_irq;
1481                         ring->irq_put = gen5_ring_put_irq;
1482                 } else {
1483                         ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
1484                         ring->irq_get = i9xx_ring_get_irq;
1485                         ring->irq_put = i9xx_ring_put_irq;
1486                 }
1487                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
1488         }
1489         ring->init = init_ring_common;
1490
1491
1492         return intel_init_ring_buffer(dev, ring);
1493 }
1494
1495 int intel_init_blt_ring_buffer(struct drm_device *dev)
1496 {
1497         drm_i915_private_t *dev_priv = dev->dev_private;
1498         struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
1499
1500         ring->name = "blitter ring";
1501         ring->id = BCS;
1502
1503         ring->mmio_base = BLT_RING_BASE;
1504         ring->write_tail = ring_write_tail;
1505         ring->flush = blt_ring_flush;
1506         ring->add_request = gen6_add_request;
1507         ring->get_seqno = gen6_ring_get_seqno;
1508         ring->irq_enable_mask = GEN6_BLITTER_USER_INTERRUPT;
1509         ring->irq_get = gen6_ring_get_irq;
1510         ring->irq_put = gen6_ring_put_irq;
1511         ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
1512         ring->sync_to = gen6_ring_sync;
1513         ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_BR;
1514         ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_BV;
1515         ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_INVALID;
1516         ring->signal_mbox[0] = GEN6_RBSYNC;
1517         ring->signal_mbox[1] = GEN6_VBSYNC;
1518         ring->init = init_ring_common;
1519
1520         return intel_init_ring_buffer(dev, ring);
1521 }