Pileus Git - ~andy/linux/blob - drivers/gpu/drm/i915/intel_uncore.c

   1 /*
   2  * Copyright © 2013 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "i915_drv.h"
  25 #include "intel_drv.h"
  26
  27 #define FORCEWAKE_ACK_TIMEOUT_MS 2
  28
  29 #define __raw_i915_read8(dev_priv__, reg__) readb((dev_priv__)->regs + (reg__))
  30 #define __raw_i915_write8(dev_priv__, reg__, val__) writeb(val__, (dev_priv__)->regs + (reg__))
  31
  32 #define __raw_i915_read16(dev_priv__, reg__) readw((dev_priv__)->regs + (reg__))
  33 #define __raw_i915_write16(dev_priv__, reg__, val__) writew(val__, (dev_priv__)->regs + (reg__))
  34
  35 #define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))
  36 #define __raw_i915_write32(dev_priv__, reg__, val__) writel(val__, (dev_priv__)->regs + (reg__))
  37
  38 #define __raw_i915_read64(dev_priv__, reg__) readq((dev_priv__)->regs + (reg__))
  39 #define __raw_i915_write64(dev_priv__, reg__, val__) writeq(val__, (dev_priv__)->regs + (reg__))
  40
  41 #define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32(dev_priv__, reg__)
  42
  43
  44 static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv)
  45 {
  46         u32 gt_thread_status_mask;
  47
  48         if (IS_HASWELL(dev_priv->dev))
  49                 gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK_HSW;
  50         else
  51                 gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK;
  52
  53         /* w/a for a sporadic read returning 0 by waiting for the GT
  54          * thread to wake up.
  55          */
  56         if (wait_for_atomic_us((__raw_i915_read32(dev_priv, GEN6_GT_THREAD_STATUS_REG) & gt_thread_status_mask) == 0, 500))
  57                 DRM_ERROR("GT thread status wait timed out\n");
  58 }
  59
  60 static void __gen6_gt_force_wake_reset(struct drm_i915_private *dev_priv)
  61 {
  62         __raw_i915_write32(dev_priv, FORCEWAKE, 0);
  63         /* something from same cacheline, but !FORCEWAKE */
  64         __raw_posting_read(dev_priv, ECOBUS);
  65 }
  66
  67 static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv,
  68                                                         int fw_engine)
  69 {
  70         if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1) == 0,
  71                             FORCEWAKE_ACK_TIMEOUT_MS))
  72                 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
  73
  74         __raw_i915_write32(dev_priv, FORCEWAKE, 1);
  75         /* something from same cacheline, but !FORCEWAKE */
  76         __raw_posting_read(dev_priv, ECOBUS);
  77
  78         if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1),
  79                             FORCEWAKE_ACK_TIMEOUT_MS))
  80                 DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
  81
  82         /* WaRsForcewakeWaitTC0:snb */
  83         __gen6_gt_wait_for_thread_c0(dev_priv);
  84 }
  85
  86 static void __gen6_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv)
  87 {
  88         __raw_i915_write32(dev_priv, FORCEWAKE_MT, _MASKED_BIT_DISABLE(0xffff));
  89         /* something from same cacheline, but !FORCEWAKE_MT */
  90         __raw_posting_read(dev_priv, ECOBUS);
  91 }
  92
  93 static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv,
  94                                                         int fw_engine)
  95 {
  96         u32 forcewake_ack;
  97
  98         if (IS_HASWELL(dev_priv->dev) || IS_GEN8(dev_priv->dev))
  99                 forcewake_ack = FORCEWAKE_ACK_HSW;
 100         else
 101                 forcewake_ack = FORCEWAKE_MT_ACK;
 102
 103         if (wait_for_atomic((__raw_i915_read32(dev_priv, forcewake_ack) & FORCEWAKE_KERNEL) == 0,
 104                             FORCEWAKE_ACK_TIMEOUT_MS))
 105                 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
 106
 107         __raw_i915_write32(dev_priv, FORCEWAKE_MT,
 108                            _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
 109         /* something from same cacheline, but !FORCEWAKE_MT */
 110         __raw_posting_read(dev_priv, ECOBUS);
 111
 112         if (wait_for_atomic((__raw_i915_read32(dev_priv, forcewake_ack) & FORCEWAKE_KERNEL),
 113                             FORCEWAKE_ACK_TIMEOUT_MS))
 114                 DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
 115
 116         /* WaRsForcewakeWaitTC0:ivb,hsw */
 117         if (INTEL_INFO(dev_priv->dev)->gen < 8)
 118                 __gen6_gt_wait_for_thread_c0(dev_priv);
 119 }
 120
 121 static void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv)
 122 {
 123         u32 gtfifodbg;
 124
 125         gtfifodbg = __raw_i915_read32(dev_priv, GTFIFODBG);
 126         if (WARN(gtfifodbg, "GT wake FIFO error 0x%x\n", gtfifodbg))
 127                 __raw_i915_write32(dev_priv, GTFIFODBG, gtfifodbg);
 128 }
 129
 130 static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv,
 131                                                         int fw_engine)
 132 {
 133         __raw_i915_write32(dev_priv, FORCEWAKE, 0);
 134         /* something from same cacheline, but !FORCEWAKE */
 135         __raw_posting_read(dev_priv, ECOBUS);
 136         gen6_gt_check_fifodbg(dev_priv);
 137 }
 138
 139 static void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv,
 140                                                         int fw_engine)
 141 {
 142         __raw_i915_write32(dev_priv, FORCEWAKE_MT,
 143                            _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
 144         /* something from same cacheline, but !FORCEWAKE_MT */
 145         __raw_posting_read(dev_priv, ECOBUS);
 146         gen6_gt_check_fifodbg(dev_priv);
 147 }
 148
 149 static int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
 150 {
 151         int ret = 0;
 152
 153         /* On VLV, FIFO will be shared by both SW and HW.
 154          * So, we need to read the FREE_ENTRIES everytime */
 155         if (IS_VALLEYVIEW(dev_priv->dev))
 156                 dev_priv->uncore.fifo_count =
 157                         __raw_i915_read32(dev_priv, GTFIFOCTL) &
 158                                                 GT_FIFO_FREE_ENTRIES_MASK;
 159
 160         if (dev_priv->uncore.fifo_count < GT_FIFO_NUM_RESERVED_ENTRIES) {
 161                 int loop = 500;
 162                 u32 fifo = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK;
 163                 while (fifo <= GT_FIFO_NUM_RESERVED_ENTRIES && loop--) {
 164                         udelay(10);
 165                         fifo = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK;
 166                 }
 167                 if (WARN_ON(loop < 0 && fifo <= GT_FIFO_NUM_RESERVED_ENTRIES))
 168                         ++ret;
 169                 dev_priv->uncore.fifo_count = fifo;
 170         }
 171         dev_priv->uncore.fifo_count--;
 172
 173         return ret;
 174 }
 175
 176 static void vlv_force_wake_reset(struct drm_i915_private *dev_priv)
 177 {
 178         __raw_i915_write32(dev_priv, FORCEWAKE_VLV,
 179                            _MASKED_BIT_DISABLE(0xffff));
 180         /* something from same cacheline, but !FORCEWAKE_VLV */
 181         __raw_posting_read(dev_priv, FORCEWAKE_ACK_VLV);
 182 }
 183
 184 static void __vlv_force_wake_get(struct drm_i915_private *dev_priv,
 185                                                 int fw_engine)
 186 {
 187         /* Check for Render Engine */
 188         if (FORCEWAKE_RENDER & fw_engine) {
 189                 if (wait_for_atomic((__raw_i915_read32(dev_priv,
 190                                                 FORCEWAKE_ACK_VLV) &
 191                                                 FORCEWAKE_KERNEL) == 0,
 192                                         FORCEWAKE_ACK_TIMEOUT_MS))
 193                         DRM_ERROR("Timed out: Render forcewake old ack to clear.\n");
 194
 195                 __raw_i915_write32(dev_priv, FORCEWAKE_VLV,
 196                                    _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
 197
 198                 if (wait_for_atomic((__raw_i915_read32(dev_priv,
 199                                                 FORCEWAKE_ACK_VLV) &
 200                                                 FORCEWAKE_KERNEL),
 201                                         FORCEWAKE_ACK_TIMEOUT_MS))
 202                         DRM_ERROR("Timed out: waiting for Render to ack.\n");
 203         }
 204
 205         /* Check for Media Engine */
 206         if (FORCEWAKE_MEDIA & fw_engine) {
 207                 if (wait_for_atomic((__raw_i915_read32(dev_priv,
 208                                                 FORCEWAKE_ACK_MEDIA_VLV) &
 209                                                 FORCEWAKE_KERNEL) == 0,
 210                                         FORCEWAKE_ACK_TIMEOUT_MS))
 211                         DRM_ERROR("Timed out: Media forcewake old ack to clear.\n");
 212
 213                 __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV,
 214                                    _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
 215
 216                 if (wait_for_atomic((__raw_i915_read32(dev_priv,
 217                                                 FORCEWAKE_ACK_MEDIA_VLV) &
 218                                                 FORCEWAKE_KERNEL),
 219                                         FORCEWAKE_ACK_TIMEOUT_MS))
 220                         DRM_ERROR("Timed out: waiting for media to ack.\n");
 221         }
 222
 223         /* WaRsForcewakeWaitTC0:vlv */
 224         __gen6_gt_wait_for_thread_c0(dev_priv);
 225
 226 }
 227
 228 static void __vlv_force_wake_put(struct drm_i915_private *dev_priv,
 229                                         int fw_engine)
 230 {
 231
 232         /* Check for Render Engine */
 233         if (FORCEWAKE_RENDER & fw_engine)
 234                 __raw_i915_write32(dev_priv, FORCEWAKE_VLV,
 235                                         _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
 236
 237
 238         /* Check for Media Engine */
 239         if (FORCEWAKE_MEDIA & fw_engine)
 240                 __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV,
 241                                 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
 242
 243         /* The below doubles as a POSTING_READ */
 244         gen6_gt_check_fifodbg(dev_priv);
 245
 246 }
 247
 248 void vlv_force_wake_get(struct drm_i915_private *dev_priv,
 249                                                 int fw_engine)
 250 {
 251         unsigned long irqflags;
 252
 253         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 254         if (FORCEWAKE_RENDER & fw_engine) {
 255                 if (dev_priv->uncore.fw_rendercount++ == 0)
 256                         dev_priv->uncore.funcs.force_wake_get(dev_priv,
 257                                                         FORCEWAKE_RENDER);
 258         }
 259         if (FORCEWAKE_MEDIA & fw_engine) {
 260                 if (dev_priv->uncore.fw_mediacount++ == 0)
 261                         dev_priv->uncore.funcs.force_wake_get(dev_priv,
 262                                                         FORCEWAKE_MEDIA);
 263         }
 264
 265         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 266 }
 267
 268 void vlv_force_wake_put(struct drm_i915_private *dev_priv,
 269                                                 int fw_engine)
 270 {
 271         unsigned long irqflags;
 272
 273         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 274
 275         if (FORCEWAKE_RENDER & fw_engine) {
 276                 WARN_ON(dev_priv->uncore.fw_rendercount == 0);
 277                 if (--dev_priv->uncore.fw_rendercount == 0)
 278                         dev_priv->uncore.funcs.force_wake_put(dev_priv,
 279                                                         FORCEWAKE_RENDER);
 280         }
 281
 282         if (FORCEWAKE_MEDIA & fw_engine) {
 283                 WARN_ON(dev_priv->uncore.fw_mediacount == 0);
 284                 if (--dev_priv->uncore.fw_mediacount == 0)
 285                         dev_priv->uncore.funcs.force_wake_put(dev_priv,
 286                                                         FORCEWAKE_MEDIA);
 287         }
 288
 289         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 290 }
 291
 292 static void gen6_force_wake_work(struct work_struct *work)
 293 {
 294         struct drm_i915_private *dev_priv =
 295                 container_of(work, typeof(*dev_priv), uncore.force_wake_work.work);
 296         unsigned long irqflags;
 297
 298         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 299         if (--dev_priv->uncore.forcewake_count == 0)
 300                 dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL);
 301         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 302 }
 303
 304 static void intel_uncore_forcewake_reset(struct drm_device *dev)
 305 {
 306         struct drm_i915_private *dev_priv = dev->dev_private;
 307
 308         if (IS_VALLEYVIEW(dev)) {
 309                 vlv_force_wake_reset(dev_priv);
 310         } else if (INTEL_INFO(dev)->gen >= 6) {
 311                 __gen6_gt_force_wake_reset(dev_priv);
 312                 if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
 313                         __gen6_gt_force_wake_mt_reset(dev_priv);
 314         }
 315 }
 316
 317 void intel_uncore_early_sanitize(struct drm_device *dev)
 318 {
 319         struct drm_i915_private *dev_priv = dev->dev_private;
 320
 321         if (HAS_FPGA_DBG_UNCLAIMED(dev))
 322                 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 323
 324         if (IS_HASWELL(dev) &&
 325             (__raw_i915_read32(dev_priv, HSW_EDRAM_PRESENT) == 1)) {
 326                 /* The docs do not explain exactly how the calculation can be
 327                  * made. It is somewhat guessable, but for now, it's always
 328                  * 128MB.
 329                  * NB: We can't write IDICR yet because we do not have gt funcs
 330                  * set up */
 331                 dev_priv->ellc_size = 128;
 332                 DRM_INFO("Found %zuMB of eLLC\n", dev_priv->ellc_size);
 333         }
 334
 335         intel_uncore_forcewake_reset(dev);
 336 }
 337
 338 void intel_uncore_sanitize(struct drm_device *dev)
 339 {
 340         struct drm_i915_private *dev_priv = dev->dev_private;
 341         u32 reg_val;
 342
 343         intel_uncore_forcewake_reset(dev);
 344
 345         /* BIOS often leaves RC6 enabled, but disable it for hw init */
 346         intel_disable_gt_powersave(dev);
 347
 348         /* Turn off power gate, require especially for the BIOS less system */
 349         if (IS_VALLEYVIEW(dev)) {
 350
 351                 mutex_lock(&dev_priv->rps.hw_lock);
 352                 reg_val = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS);
 353
 354                 if (reg_val & (RENDER_PWRGT | MEDIA_PWRGT | DISP2D_PWRGT))
 355                         vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, 0x0);
 356
 357                 mutex_unlock(&dev_priv->rps.hw_lock);
 358
 359         }
 360 }
 361
 362 /*
 363  * Generally this is called implicitly by the register read function. However,
 364  * if some sequence requires the GT to not power down then this function should
 365  * be called at the beginning of the sequence followed by a call to
 366  * gen6_gt_force_wake_put() at the end of the sequence.
 367  */
 368 void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine)
 369 {
 370         unsigned long irqflags;
 371
 372         if (!dev_priv->uncore.funcs.force_wake_get)
 373                 return;
 374
 375         /* Redirect to VLV specific routine */
 376         if (IS_VALLEYVIEW(dev_priv->dev))
 377                 return vlv_force_wake_get(dev_priv, fw_engine);
 378
 379         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 380         if (dev_priv->uncore.forcewake_count++ == 0)
 381                 dev_priv->uncore.funcs.force_wake_get(dev_priv, FORCEWAKE_ALL);
 382         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 383 }
 384
 385 /*
 386  * see gen6_gt_force_wake_get()
 387  */
 388 void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
 389 {
 390         unsigned long irqflags;
 391
 392         if (!dev_priv->uncore.funcs.force_wake_put)
 393                 return;
 394
 395         /* Redirect to VLV specific routine */
 396         if (IS_VALLEYVIEW(dev_priv->dev))
 397                 return vlv_force_wake_put(dev_priv, fw_engine);
 398
 399
 400         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 401         if (--dev_priv->uncore.forcewake_count == 0) {
 402                 dev_priv->uncore.forcewake_count++;
 403                 mod_delayed_work(dev_priv->wq,
 404                                  &dev_priv->uncore.force_wake_work,
 405                                  1);
 406         }
 407         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 408 }
 409
 410 /* We give fast paths for the really cool registers */
 411 #define NEEDS_FORCE_WAKE(dev_priv, reg) \
 412          ((reg) < 0x40000 && (reg) != FORCEWAKE)
 413
 414 static void
 415 ilk_dummy_write(struct drm_i915_private *dev_priv)
 416 {
 417         /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up
 418          * the chip from rc6 before touching it for real. MI_MODE is masked,
 419          * hence harmless to write 0 into. */
 420         __raw_i915_write32(dev_priv, MI_MODE, 0);
 421 }
 422
 423 static void
 424 hsw_unclaimed_reg_clear(struct drm_i915_private *dev_priv, u32 reg)
 425 {
 426         if (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM) {
 427                 DRM_ERROR("Unknown unclaimed register before writing to %x\n",
 428                           reg);
 429                 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 430         }
 431 }
 432
 433 static void
 434 hsw_unclaimed_reg_check(struct drm_i915_private *dev_priv, u32 reg)
 435 {
 436         if (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM) {
 437                 DRM_ERROR("Unclaimed write to %x\n", reg);
 438                 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 439         }
 440 }
 441
 442 #define REG_READ_HEADER(x) \
 443         unsigned long irqflags; \
 444         u##x val = 0; \
 445         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags)
 446
 447 #define REG_READ_FOOTER \
 448         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
 449         trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
 450         return val
 451
 452 #define __gen4_read(x) \
 453 static u##x \
 454 gen4_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 455         REG_READ_HEADER(x); \
 456         val = __raw_i915_read##x(dev_priv, reg); \
 457         REG_READ_FOOTER; \
 458 }
 459
 460 #define __gen5_read(x) \
 461 static u##x \
 462 gen5_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 463         REG_READ_HEADER(x); \
 464         ilk_dummy_write(dev_priv); \
 465         val = __raw_i915_read##x(dev_priv, reg); \
 466         REG_READ_FOOTER; \
 467 }
 468
 469 #define __gen6_read(x) \
 470 static u##x \
 471 gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 472         REG_READ_HEADER(x); \
 473         if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
 474                 if (dev_priv->uncore.forcewake_count == 0) \
 475                         dev_priv->uncore.funcs.force_wake_get(dev_priv, \
 476                                                         FORCEWAKE_ALL); \
 477                 val = __raw_i915_read##x(dev_priv, reg); \
 478                 if (dev_priv->uncore.forcewake_count == 0) \
 479                         dev_priv->uncore.funcs.force_wake_put(dev_priv, \
 480                                                         FORCEWAKE_ALL); \
 481         } else { \
 482                 val = __raw_i915_read##x(dev_priv, reg); \
 483         } \
 484         REG_READ_FOOTER; \
 485 }
 486
 487 #define __vlv_read(x) \
 488 static u##x \
 489 vlv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 490         unsigned fwengine = 0; \
 491         unsigned *fwcount; \
 492         REG_READ_HEADER(x); \
 493         if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) {   \
 494                 fwengine = FORCEWAKE_RENDER;            \
 495                 fwcount = &dev_priv->uncore.fw_rendercount;    \
 496         }                                               \
 497         else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) {       \
 498                 fwengine = FORCEWAKE_MEDIA;             \
 499                 fwcount = &dev_priv->uncore.fw_mediacount;     \
 500         }  \
 501         if (fwengine != 0) {            \
 502                 if ((*fwcount)++ == 0) \
 503                         (dev_priv)->uncore.funcs.force_wake_get(dev_priv, \
 504                                                                 fwengine); \
 505                 val = __raw_i915_read##x(dev_priv, reg); \
 506                 if (--(*fwcount) == 0) \
 507                         (dev_priv)->uncore.funcs.force_wake_put(dev_priv, \
 508                                                         fwengine); \
 509         } else { \
 510                 val = __raw_i915_read##x(dev_priv, reg); \
 511         } \
 512         REG_READ_FOOTER; \
 513 }
 514
 515
 516 __vlv_read(8)
 517 __vlv_read(16)
 518 __vlv_read(32)
 519 __vlv_read(64)
 520 __gen6_read(8)
 521 __gen6_read(16)
 522 __gen6_read(32)
 523 __gen6_read(64)
 524 __gen5_read(8)
 525 __gen5_read(16)
 526 __gen5_read(32)
 527 __gen5_read(64)
 528 __gen4_read(8)
 529 __gen4_read(16)
 530 __gen4_read(32)
 531 __gen4_read(64)
 532
 533 #undef __vlv_read
 534 #undef __gen6_read
 535 #undef __gen5_read
 536 #undef __gen4_read
 537 #undef REG_READ_FOOTER
 538 #undef REG_READ_HEADER
 539
 540 #define REG_WRITE_HEADER \
 541         unsigned long irqflags; \
 542         trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
 543         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags)
 544
 545 #define REG_WRITE_FOOTER \
 546         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags)
 547
 548 #define __gen4_write(x) \
 549 static void \
 550 gen4_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 551         REG_WRITE_HEADER; \
 552         __raw_i915_write##x(dev_priv, reg, val); \
 553         REG_WRITE_FOOTER; \
 554 }
 555
 556 #define __gen5_write(x) \
 557 static void \
 558 gen5_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 559         REG_WRITE_HEADER; \
 560         ilk_dummy_write(dev_priv); \
 561         __raw_i915_write##x(dev_priv, reg, val); \
 562         REG_WRITE_FOOTER; \
 563 }
 564
 565 #define __gen6_write(x) \
 566 static void \
 567 gen6_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 568         u32 __fifo_ret = 0; \
 569         REG_WRITE_HEADER; \
 570         if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
 571                 __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
 572         } \
 573         __raw_i915_write##x(dev_priv, reg, val); \
 574         if (unlikely(__fifo_ret)) { \
 575                 gen6_gt_check_fifodbg(dev_priv); \
 576         } \
 577         REG_WRITE_FOOTER; \
 578 }
 579
 580 #define __hsw_write(x) \
 581 static void \
 582 hsw_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 583         u32 __fifo_ret = 0; \
 584         REG_WRITE_HEADER; \
 585         if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
 586                 __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
 587         } \
 588         hsw_unclaimed_reg_clear(dev_priv, reg); \
 589         __raw_i915_write##x(dev_priv, reg, val); \
 590         if (unlikely(__fifo_ret)) { \
 591                 gen6_gt_check_fifodbg(dev_priv); \
 592         } \
 593         hsw_unclaimed_reg_check(dev_priv, reg); \
 594         REG_WRITE_FOOTER; \
 595 }
 596
 597 static const u32 gen8_shadowed_regs[] = {
 598         FORCEWAKE_MT,
 599         GEN6_RPNSWREQ,
 600         GEN6_RC_VIDEO_FREQ,
 601         RING_TAIL(RENDER_RING_BASE),
 602         RING_TAIL(GEN6_BSD_RING_BASE),
 603         RING_TAIL(VEBOX_RING_BASE),
 604         RING_TAIL(BLT_RING_BASE),
 605         /* TODO: Other registers are not yet used */
 606 };
 607
 608 static bool is_gen8_shadowed(struct drm_i915_private *dev_priv, u32 reg)
 609 {
 610         int i;
 611         for (i = 0; i < ARRAY_SIZE(gen8_shadowed_regs); i++)
 612                 if (reg == gen8_shadowed_regs[i])
 613                         return true;
 614
 615         return false;
 616 }
 617
 618 #define __gen8_write(x) \
 619 static void \
 620 gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 621         bool __needs_put = !is_gen8_shadowed(dev_priv, reg); \
 622         REG_WRITE_HEADER; \
 623         if (__needs_put) { \
 624                 dev_priv->uncore.funcs.force_wake_get(dev_priv, \
 625                                                         FORCEWAKE_ALL); \
 626         } \
 627         __raw_i915_write##x(dev_priv, reg, val); \
 628         if (__needs_put) { \
 629                 dev_priv->uncore.funcs.force_wake_put(dev_priv, \
 630                                                         FORCEWAKE_ALL); \
 631         } \
 632         REG_WRITE_FOOTER; \
 633 }
 634
 635 __gen8_write(8)
 636 __gen8_write(16)
 637 __gen8_write(32)
 638 __gen8_write(64)
 639 __hsw_write(8)
 640 __hsw_write(16)
 641 __hsw_write(32)
 642 __hsw_write(64)
 643 __gen6_write(8)
 644 __gen6_write(16)
 645 __gen6_write(32)
 646 __gen6_write(64)
 647 __gen5_write(8)
 648 __gen5_write(16)
 649 __gen5_write(32)
 650 __gen5_write(64)
 651 __gen4_write(8)
 652 __gen4_write(16)
 653 __gen4_write(32)
 654 __gen4_write(64)
 655
 656 #undef __gen8_write
 657 #undef __hsw_write
 658 #undef __gen6_write
 659 #undef __gen5_write
 660 #undef __gen4_write
 661 #undef REG_WRITE_FOOTER
 662 #undef REG_WRITE_HEADER
 663
 664 void intel_uncore_init(struct drm_device *dev)
 665 {
 666         struct drm_i915_private *dev_priv = dev->dev_private;
 667
 668         INIT_DELAYED_WORK(&dev_priv->uncore.force_wake_work,
 669                           gen6_force_wake_work);
 670
 671         if (IS_VALLEYVIEW(dev)) {
 672                 dev_priv->uncore.funcs.force_wake_get = __vlv_force_wake_get;
 673                 dev_priv->uncore.funcs.force_wake_put = __vlv_force_wake_put;
 674         } else if (IS_HASWELL(dev) || IS_GEN8(dev)) {
 675                 dev_priv->uncore.funcs.force_wake_get = __gen6_gt_force_wake_mt_get;
 676                 dev_priv->uncore.funcs.force_wake_put = __gen6_gt_force_wake_mt_put;
 677         } else if (IS_IVYBRIDGE(dev)) {
 678                 u32 ecobus;
 679
 680                 /* IVB configs may use multi-threaded forcewake */
 681
 682                 /* A small trick here - if the bios hasn't configured
 683                  * MT forcewake, and if the device is in RC6, then
 684                  * force_wake_mt_get will not wake the device and the
 685                  * ECOBUS read will return zero. Which will be
 686                  * (correctly) interpreted by the test below as MT
 687                  * forcewake being disabled.
 688                  */
 689                 mutex_lock(&dev->struct_mutex);
 690                 __gen6_gt_force_wake_mt_get(dev_priv, FORCEWAKE_ALL);
 691                 ecobus = __raw_i915_read32(dev_priv, ECOBUS);
 692                 __gen6_gt_force_wake_mt_put(dev_priv, FORCEWAKE_ALL);
 693                 mutex_unlock(&dev->struct_mutex);
 694
 695                 if (ecobus & FORCEWAKE_MT_ENABLE) {
 696                         dev_priv->uncore.funcs.force_wake_get =
 697                                 __gen6_gt_force_wake_mt_get;
 698                         dev_priv->uncore.funcs.force_wake_put =
 699                                 __gen6_gt_force_wake_mt_put;
 700                 } else {
 701                         DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n");
 702                         DRM_INFO("when using vblank-synced partial screen updates.\n");
 703                         dev_priv->uncore.funcs.force_wake_get =
 704                                 __gen6_gt_force_wake_get;
 705                         dev_priv->uncore.funcs.force_wake_put =
 706                                 __gen6_gt_force_wake_put;
 707                 }
 708         } else if (IS_GEN6(dev)) {
 709                 dev_priv->uncore.funcs.force_wake_get =
 710                         __gen6_gt_force_wake_get;
 711                 dev_priv->uncore.funcs.force_wake_put =
 712                         __gen6_gt_force_wake_put;
 713         }
 714
 715         switch (INTEL_INFO(dev)->gen) {
 716         default:
 717                 dev_priv->uncore.funcs.mmio_writeb  = gen8_write8;
 718                 dev_priv->uncore.funcs.mmio_writew  = gen8_write16;
 719                 dev_priv->uncore.funcs.mmio_writel  = gen8_write32;
 720                 dev_priv->uncore.funcs.mmio_writeq  = gen8_write64;
 721                 dev_priv->uncore.funcs.mmio_readb  = gen6_read8;
 722                 dev_priv->uncore.funcs.mmio_readw  = gen6_read16;
 723                 dev_priv->uncore.funcs.mmio_readl  = gen6_read32;
 724                 dev_priv->uncore.funcs.mmio_readq  = gen6_read64;
 725                 break;
 726         case 7:
 727         case 6:
 728                 if (IS_HASWELL(dev)) {
 729                         dev_priv->uncore.funcs.mmio_writeb  = hsw_write8;
 730                         dev_priv->uncore.funcs.mmio_writew  = hsw_write16;
 731                         dev_priv->uncore.funcs.mmio_writel  = hsw_write32;
 732                         dev_priv->uncore.funcs.mmio_writeq  = hsw_write64;
 733                 } else {
 734                         dev_priv->uncore.funcs.mmio_writeb  = gen6_write8;
 735                         dev_priv->uncore.funcs.mmio_writew  = gen6_write16;
 736                         dev_priv->uncore.funcs.mmio_writel  = gen6_write32;
 737                         dev_priv->uncore.funcs.mmio_writeq  = gen6_write64;
 738                 }
 739
 740                 if (IS_VALLEYVIEW(dev)) {
 741                         dev_priv->uncore.funcs.mmio_readb  = vlv_read8;
 742                         dev_priv->uncore.funcs.mmio_readw  = vlv_read16;
 743                         dev_priv->uncore.funcs.mmio_readl  = vlv_read32;
 744                         dev_priv->uncore.funcs.mmio_readq  = vlv_read64;
 745                 } else {
 746                         dev_priv->uncore.funcs.mmio_readb  = gen6_read8;
 747                         dev_priv->uncore.funcs.mmio_readw  = gen6_read16;
 748                         dev_priv->uncore.funcs.mmio_readl  = gen6_read32;
 749                         dev_priv->uncore.funcs.mmio_readq  = gen6_read64;
 750                 }
 751                 break;
 752         case 5:
 753                 dev_priv->uncore.funcs.mmio_writeb  = gen5_write8;
 754                 dev_priv->uncore.funcs.mmio_writew  = gen5_write16;
 755                 dev_priv->uncore.funcs.mmio_writel  = gen5_write32;
 756                 dev_priv->uncore.funcs.mmio_writeq  = gen5_write64;
 757                 dev_priv->uncore.funcs.mmio_readb  = gen5_read8;
 758                 dev_priv->uncore.funcs.mmio_readw  = gen5_read16;
 759                 dev_priv->uncore.funcs.mmio_readl  = gen5_read32;
 760                 dev_priv->uncore.funcs.mmio_readq  = gen5_read64;
 761                 break;
 762         case 4:
 763         case 3:
 764         case 2:
 765                 dev_priv->uncore.funcs.mmio_writeb  = gen4_write8;
 766                 dev_priv->uncore.funcs.mmio_writew  = gen4_write16;
 767                 dev_priv->uncore.funcs.mmio_writel  = gen4_write32;
 768                 dev_priv->uncore.funcs.mmio_writeq  = gen4_write64;
 769                 dev_priv->uncore.funcs.mmio_readb  = gen4_read8;
 770                 dev_priv->uncore.funcs.mmio_readw  = gen4_read16;
 771                 dev_priv->uncore.funcs.mmio_readl  = gen4_read32;
 772                 dev_priv->uncore.funcs.mmio_readq  = gen4_read64;
 773                 break;
 774         }
 775 }
 776
 777 void intel_uncore_fini(struct drm_device *dev)
 778 {
 779         struct drm_i915_private *dev_priv = dev->dev_private;
 780
 781         flush_delayed_work(&dev_priv->uncore.force_wake_work);
 782
 783         /* Paranoia: make sure we have disabled everything before we exit. */
 784         intel_uncore_sanitize(dev);
 785 }
 786
 787 static const struct register_whitelist {
 788         uint64_t offset;
 789         uint32_t size;
 790         uint32_t gen_bitmask; /* support gens, 0x10 for 4, 0x30 for 4 and 5, etc. */
 791 } whitelist[] = {
 792         { RING_TIMESTAMP(RENDER_RING_BASE), 8, 0xF0 },
 793 };
 794
 795 int i915_reg_read_ioctl(struct drm_device *dev,
 796                         void *data, struct drm_file *file)
 797 {
 798         struct drm_i915_private *dev_priv = dev->dev_private;
 799         struct drm_i915_reg_read *reg = data;
 800         struct register_whitelist const *entry = whitelist;
 801         int i;
 802
 803         for (i = 0; i < ARRAY_SIZE(whitelist); i++, entry++) {
 804                 if (entry->offset == reg->offset &&
 805                     (1 << INTEL_INFO(dev)->gen & entry->gen_bitmask))
 806                         break;
 807         }
 808
 809         if (i == ARRAY_SIZE(whitelist))
 810                 return -EINVAL;
 811
 812         switch (entry->size) {
 813         case 8:
 814                 reg->val = I915_READ64(reg->offset);
 815                 break;
 816         case 4:
 817                 reg->val = I915_READ(reg->offset);
 818                 break;
 819         case 2:
 820                 reg->val = I915_READ16(reg->offset);
 821                 break;
 822         case 1:
 823                 reg->val = I915_READ8(reg->offset);
 824                 break;
 825         default:
 826                 WARN_ON(1);
 827                 return -EINVAL;
 828         }
 829
 830         return 0;
 831 }
 832
 833 int i915_get_reset_stats_ioctl(struct drm_device *dev,
 834                                void *data, struct drm_file *file)
 835 {
 836         struct drm_i915_private *dev_priv = dev->dev_private;
 837         struct drm_i915_reset_stats *args = data;
 838         struct i915_ctx_hang_stats *hs;
 839         int ret;
 840
 841         if (args->flags || args->pad)
 842                 return -EINVAL;
 843
 844         if (args->ctx_id == DEFAULT_CONTEXT_ID && !capable(CAP_SYS_ADMIN))
 845                 return -EPERM;
 846
 847         ret = mutex_lock_interruptible(&dev->struct_mutex);
 848         if (ret)
 849                 return ret;
 850
 851         hs = i915_gem_context_get_hang_stats(dev, file, args->ctx_id);
 852         if (IS_ERR(hs)) {
 853                 mutex_unlock(&dev->struct_mutex);
 854                 return PTR_ERR(hs);
 855         }
 856
 857         if (capable(CAP_SYS_ADMIN))
 858                 args->reset_count = i915_reset_count(&dev_priv->gpu_error);
 859         else
 860                 args->reset_count = 0;
 861
 862         args->batch_active = hs->batch_active;
 863         args->batch_pending = hs->batch_pending;
 864
 865         mutex_unlock(&dev->struct_mutex);
 866
 867         return 0;
 868 }
 869
 870 static int i965_reset_complete(struct drm_device *dev)
 871 {
 872         u8 gdrst;
 873         pci_read_config_byte(dev->pdev, I965_GDRST, &gdrst);
 874         return (gdrst & GRDOM_RESET_ENABLE) == 0;
 875 }
 876
 877 static int i965_do_reset(struct drm_device *dev)
 878 {
 879         int ret;
 880
 881         /*
 882          * Set the domains we want to reset (GRDOM/bits 2 and 3) as
 883          * well as the reset bit (GR/bit 0).  Setting the GR bit
 884          * triggers the reset; when done, the hardware will clear it.
 885          */
 886         pci_write_config_byte(dev->pdev, I965_GDRST,
 887                               GRDOM_RENDER | GRDOM_RESET_ENABLE);
 888         ret =  wait_for(i965_reset_complete(dev), 500);
 889         if (ret)
 890                 return ret;
 891
 892         /* We can't reset render&media without also resetting display ... */
 893         pci_write_config_byte(dev->pdev, I965_GDRST,
 894                               GRDOM_MEDIA | GRDOM_RESET_ENABLE);
 895
 896         ret =  wait_for(i965_reset_complete(dev), 500);
 897         if (ret)
 898                 return ret;
 899
 900         pci_write_config_byte(dev->pdev, I965_GDRST, 0);
 901
 902         return 0;
 903 }
 904
 905 static int ironlake_do_reset(struct drm_device *dev)
 906 {
 907         struct drm_i915_private *dev_priv = dev->dev_private;
 908         u32 gdrst;
 909         int ret;
 910
 911         gdrst = I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR);
 912         gdrst &= ~GRDOM_MASK;
 913         I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR,
 914                    gdrst | GRDOM_RENDER | GRDOM_RESET_ENABLE);
 915         ret = wait_for(I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1, 500);
 916         if (ret)
 917                 return ret;
 918
 919         /* We can't reset render&media without also resetting display ... */
 920         gdrst = I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR);
 921         gdrst &= ~GRDOM_MASK;
 922         I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR,
 923                    gdrst | GRDOM_MEDIA | GRDOM_RESET_ENABLE);
 924         return wait_for(I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1, 500);
 925 }
 926
 927 static int gen6_do_reset(struct drm_device *dev)
 928 {
 929         struct drm_i915_private *dev_priv = dev->dev_private;
 930         int     ret;
 931         unsigned long irqflags;
 932
 933         /* Hold uncore.lock across reset to prevent any register access
 934          * with forcewake not set correctly
 935          */
 936         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 937
 938         /* Reset the chip */
 939
 940         /* GEN6_GDRST is not in the gt power well, no need to check
 941          * for fifo space for the write or forcewake the chip for
 942          * the read
 943          */
 944         __raw_i915_write32(dev_priv, GEN6_GDRST, GEN6_GRDOM_FULL);
 945
 946         /* Spin waiting for the device to ack the reset request */
 947         ret = wait_for((__raw_i915_read32(dev_priv, GEN6_GDRST) & GEN6_GRDOM_FULL) == 0, 500);
 948
 949         intel_uncore_forcewake_reset(dev);
 950
 951         /* If reset with a user forcewake, try to restore, otherwise turn it off */
 952         if (dev_priv->uncore.forcewake_count)
 953                 dev_priv->uncore.funcs.force_wake_get(dev_priv, FORCEWAKE_ALL);
 954         else
 955                 dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL);
 956
 957         /* Restore fifo count */
 958         dev_priv->uncore.fifo_count = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK;
 959
 960         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 961         return ret;
 962 }
 963
 964 int intel_gpu_reset(struct drm_device *dev)
 965 {
 966         switch (INTEL_INFO(dev)->gen) {
 967         case 8:
 968         case 7:
 969         case 6: return gen6_do_reset(dev);
 970         case 5: return ironlake_do_reset(dev);
 971         case 4: return i965_do_reset(dev);
 972         default: return -ENODEV;
 973         }
 974 }
 975
 976 void intel_uncore_check_errors(struct drm_device *dev)
 977 {
 978         struct drm_i915_private *dev_priv = dev->dev_private;
 979
 980         if (HAS_FPGA_DBG_UNCLAIMED(dev) &&
 981             (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM)) {
 982                 DRM_ERROR("Unclaimed register before interrupt\n");
 983                 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 984         }
 985 }